Changeset 10 for trunk


Ignore:
Timestamp:
Jun 4, 2015, 9:48:05 PM (10 years ago)
Author:
gegorbet
Message:

fixes to allow simultaneous production/development jobs

Location:
trunk
Files:
3 added
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/cleanup.php

    r9 r10  
    44 *
    55 * functions relating to copying results and cleaning up the gfac DB
     6 *  where the job used an Airavata interface.
    67 *
    78 */
    89
    9 $us3bin = exec( "ls -d ~us3/bin" );
    10 include_once "$us3bin/listen-config.php";
    11 $me              = 'cleanup.php';
    1210$email_address   = '';
    1311$queuestatus     = '';
    … …  
    1715$status          = '';
    1816
    19 function gfac_cleanup( $us3_db, $reqID, $gfac_link )
     17function aira_cleanup( $us3_db, $reqID, $gfac_link )
    2018{
    2119   global $dbhost;
    … …  
    3432   global $submittime;
    3533   global $status;
     34   global $stderr;
    3635   global $stdout;
     36   global $tarfile;
    3737   global $requestID;
     38   global $submit_dir;
     39   $me        = 'cleanup_aira.php';
    3840
    3941   $requestID = $reqID;
    … …  
    7981   list( $personID ) = mysql_fetch_array( $result );
    8082
    81    /*
    82    $query  = "SELECT clusterName, submitTime, queueStatus, method "              .
    83              "FROM HPCAnalysisRequest h LEFT JOIN HPCAnalysisResult "            .
    84              "ON h.HPCAnalysisRequestID=HPCAnalysisResult.HPCAnalysisRequestID " .
    85              "WHERE h.HPCAnalysisRequestID=$requestID";
    86    */
    8783   $query  = "SELECT clusterName, submitTime, queueStatus, method "              .
    8884             "FROM HPCAnalysisRequest h, HPCAnalysisResult r "                   .
    … …  
    121117   list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
    122118
    123    ////////
    124    // Get data from global GFAC DB and insert it into US3 DB
    125    // $gfac_link = mysql_connect( $dbhost, $guser, $gpasswd );
     119   // Get data from global GFAC DB then insert it into US3 DB
    126120
    127121   $result = mysql_select_db( $gDB, $gfac_link );
    … …  
    154148   }
    155149
    156    $query = "SELECT id, stderr, stdout, tarfile FROM analysis " .
     150
     151   $query = "SELECT id FROM analysis " .
    157152            "WHERE gfacID='$gfacID'";
    158153
    … …  
    166161   }
    167162
    168    list( $analysisID, $stderr, $stdout, $tarfile ) = mysql_fetch_array( $result );
     163   list( $analysisID ) = mysql_fetch_array( $result );
     164
     165   // Get the request guid (LIMS submit dir name)
     166   $query  = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
     167             "WHERE HPCAnalysisRequestID = $requestID ";
     168   $result = mysql_query( $query, $us3_link );
     169   
     170   if ( ! $result )
     171   {
     172      write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
     173   }
     174
     175   list( $requestGUID ) = mysql_fetch_array( $result );
     176   $output_dir = "$submit_dir/$requestGUID";
     177
     178   // Get stderr,stdout,tarfile from work directory
     179   if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
     180   chdir( "$output_dir" );
     181//write_log( "$me: gfacID=$gfacID" );
     182//write_log( "$me: submit_dir=$submit_dir" );
     183//write_log( "$me: requestGUID=$requestGUID" );
     184write_log( "$me: output_dir=$output_dir" );
     185
     186   $stderr     = "";
     187   $stdout     = "";
     188   $tarfile    = "";
     189   $fn_stderr  = "Ultrascan.stderr";
     190   $fn_stdout  = "Ultrascan.stdout";
     191   $fn_tarfile = "analysis-results.tar";
     192   $num_try    = 0;
     193   while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
     194   {
     195      sleep( 10 );
     196      $num_try++;
     197   }
     198
     199   $ofiles     = scandir( $output_dir );
     200   foreach ( $ofiles as $ofile )
     201   {
     202      if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
     203         $fn_stderr  = $ofile;
     204      if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
     205         $fn_stdout  = $ofile;
     206//write_log( "$me:    ofile=$ofile" );
     207   }
     208write_log( "$me: fn_stderr=$fn_stderr" );
     209write_log( "$me: fn_stdout=$fn_stdout" );
     210if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
     211else                          write_log( "$me: NOT FOUND: $fn_tarfile" );
     212
     213   if ( file_exists( $fn_stderr  ) ) $stderr   = file_get_contents( $fn_stderr  );
     214   if ( file_exists( $fn_stdout  ) ) $stdout   = file_get_contents( $fn_stdout  );
     215   if ( file_exists( $fn_tarfile ) ) $tarfile  = file_get_contents( $fn_tarfile );
     216
     217   if ( $cluster == 'alamo'  || $cluster == 'alamo-local' )
     218   {  // Filter "ipath_userinit" lines out of alamo stdout lines
     219      $prefln = strlen( $stdout );
     220      $output = array();
     221      exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
     222      $stdout = implode( "\n", $output );
     223      $posfln = strlen( $stdout );
     224write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
     225   }
    169226
    170227   // Save queue messages for post-mortem analysis
    … …  
    203260   }
    204261
    205    // Save stdout, stderr, etc. for message log
    206    $query  = "SELECT stdout, stderr, status, queue_msg FROM analysis " .
    207              "WHERE gfacID='$gfacID' ";
     262   $query = "SELECT queue_msg FROM analysis " .
     263            "WHERE gfacID='$gfacID' ";
     264
    208265   $result = mysql_query( $query, $gfac_link );
    209    try
    210    {
    211       // What if this is too large?
    212       list( $stdout, $stderr, $status, $queue_msg ) = mysql_fetch_array( $result );
    213    }
    214    catch ( Exception $e )
    215    {
    216       write_log( "$me: stdout + stderr larger than 128M - $gfacID\n" . mysql_error( $gfac_link ) );
    217       // Just go ahead and clean up
    218    }
     266   list( $queue_msg ) = mysql_fetch_array( $result );
    219267
    220268   // But let's allow for investigation of other large stdout and/or stderr
    … …  
    241289   }
    242290
    243    // Copy queue messages to LIMS submit directory (files there are deleted after 7 days)
    244    global $submit_dir;
    245    
    246    // Get the request guid (LIMS submit dir name)
    247    $query  = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
    248              "WHERE HPCAnalysisRequestID = $requestID ";
    249    $result = mysql_query( $query, $us3_link );
    250    
    251    if ( ! $result )
    252    {
    253       write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    254    }
    255    
    256    list( $requestGUID ) = mysql_fetch_array( $result );
    257    $output_dir = "$submit_dir/$requestGUID";
    258291
    259292   // Try to create it if necessary, and write the file
    … …  
    271304   $query = "UPDATE HPCAnalysisResult SET "                              .
    272305            "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
    273             "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "' "  .
     306            "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
     307            "queueStatus='completed' " .
    274308            "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
    275309
    … …  
    283317   }
    284318
    285    // Save the tarfile and expand it
     319   // Delete data from GFAC DB
     320   $query = "DELETE from analysis WHERE gfacID='$gfacID'";
     321
     322   $result = mysql_query( $query, $gfac_link );
     323
     324   if ( ! $result )
     325   {
     326      // Just log it and continue
     327      write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
     328   }
     329
     330   // Expand the tar file
    286331
    287332   if ( strlen( $tarfile ) == 0 )
    … …  
    292337   }
    293338
    294    // Shouldn't happen
    295    if ( ! is_dir( "$work" ) )
    296    {
    297       write_log( "$me: $work directory does not exist" );
    298       mail_to_user( "fail", "$work directory does not exist" );
    299       return( -1 );
    300    }
    301 
    302    if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
    303    chdir( "$work/$gfacID" );
    304 
    305    $f = fopen( "analysis.tar", "w" );
    306    fwrite( $f, $tarfile );
    307    fclose( $f );
    308 
    309339   $tar_out = array();
    310    exec( "tar -xf analysis.tar 2>&1", $tar_out, $err );
    311 
    312    if ( $err != 0 )
    313    {
    314       chdir( $work );
    315       exec( "rm -r $gfacID" );
    316       $output = implode( "\n", $tar_out );
    317 
    318       write_log( "$me: Bad output tarfile: $output" );
    319       mail_to_user( "fail", "Bad output file" );
    320       return( -1 );
    321    }
     340   exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
    322341
    323342   // Insert the model files and noise files
    … …  
    341360         $fn = $file;
    342361
     362      if ( preg_match( "/mdl.tmp$/", $fn ) )
     363         continue;
     364
    343365      if ( filesize( $fn ) < 100 )
    344366      {
    345          write_log( "$me:fn is invalid $fn" );
     367         write_log( "$me:fn is invalid $fn size filesize($fn)" );
    346368         mail_to_user( "fail", "Internal error\n$fn is invalid" );
    347369         return( -1 );
    … …  
    352374         $xml         = file_get_contents( $fn );
    353375         $statistics  = parse_xml( $xml, 'statistics' );
     376//         $ntries      = 0;
     377//
     378//         while ( $statistics['cpucount'] < 1  &&  $ntries < 3 )
     379//         {  // job_statistics file not totally copied, so retry
     380//            sleep( 10 );
     381//            $xml         = file_get_contents( $fn );
     382//            $statistics  = parse_xml( $xml, 'statistics' );
     383//            $ntries++;
     384//write_log( "$me:jobstats retry $ntries" );
     385//         }
     386//write_log( "$me:cputime=$statistics['cputime']" );
     387
    354388         $otherdata   = parse_xml( $xml, 'id' );
    355389
    … …  
    411445         
    412446      }
    413       else  // It's a model file
     447
     448      else if ( preg_match( "/\.mrecs/", $fn ) > 0 )  // It's an mrecs file
     449      {
     450         $xml         = file_get_contents( $fn );
     451         $mrecs_data  = parse_xml( $xml, "modelrecords" );
     452         $desc        = $mrecs_data[ 'description' ];
     453         $editGUID    = $mrecs_data[ 'editGUID' ];
     454write_log( "$me:   mrecs file editGUID=$editGUID" );
     455         if ( strlen( $editGUID ) < 36 )
     456            $editGUID    = "12345678-0123-5678-0123-567890123456";
     457         $mrecGUID    = $mrecs_data[ 'mrecGUID' ];
     458         $modelGUID   = $mrecs_data[ 'modelGUID' ];
     459
     460         $query = "INSERT INTO pcsa_modelrecs SET "  .
     461                  "editedDataID="                .
     462                  "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
     463                  "modelID=0, "             .
     464                  "mrecsGUID='$mrecGUID'," .
     465                  "description='$desc',"    .
     466                  "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
     467
     468         // Add later after all files are processed: editDataID, modelID
     469
     470         $result = mysql_query( $query, $us3_link );
     471
     472         if ( ! $result )
     473         {
     474            write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
     475            mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
     476            return( -1 );
     477         }
     478
     479         $id         = mysql_insert_id( $us3_link );
     480         $file_type  = "mrecs";
     481         $mrecsIDs[] = $id;
     482
     483         // Keep track of modelGUIDs for later, when we replace them
     484         $rmodlGUIDs[ $id ] = $modelGUID;
     485//write_log( "$me:   mrecs file inserted into DB : id=$id" );
     486      }
     487
     488      else                                           // It's a model file
    414489      {
    415490         $xml         = file_get_contents( $fn );
    … …  
    418493         $modelGUID   = $model_data[ 'modelGUID' ];
    419494         $editGUID    = $model_data[ 'editGUID' ];
     495
     496         if ( $mc_iteration > 1 )
     497         {
     498            $miter       = sprintf( "_mcN%03d", $mc_iteration );
     499            $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
     500write_log( "$me:   MODELUpd: O:description=$description" );
     501         }
    420502
    421503         $query = "INSERT INTO model SET "       .
    … …  
    499581   }
    500582   
    501    list( $requestGUID ) = mysql_fetch_array( $result );
    502    
    503    chdir( "$submit_dir/$requestGUID" );
    504    $f = fopen( "analysis.tar", "w" );
    505    fwrite( $f, $tarfile );
    506    fclose( $f );
     583//   list( $requestGUID ) = mysql_fetch_array( $result );
     584//   
     585//   chdir( "$submit_dir/$requestGUID" );
     586//   $f = fopen( "analysis-results.tar", "w" );
     587//   fwrite( $f, $tarfile );
     588//   fclose( $f );
    507589
    508590   // Clean up
    509    chdir ( $work );
     591//   chdir ( $work );
    510592   // exec( "rm -rf $gfacID" );
    511593
    … …  
    545627   // function get_gfac_message() also sets global $status
    546628   $gfac_message = get_gfac_message( $gfacID );
    547    if ( $gfac_message === false ) $gfac_message = "";
     629   if ( $gfac_message === false ) $gfac_message = "Job Finished";
    548630     
    549631   // Create a status to put in the subject line
    … …  
    561643      case "FAILED":
    562644         $subj_status = 'failed';
     645         if ( preg_match( "/^US3-A/i", $gfacID ) )
     646         {  // For A/Thrift FAIL, get error message
     647            $gfac_message = getExperimentErrors( $gfacID );
     648//$gfac_message .= "Test ERROR MESSAGE";
     649         }
    563650         break;
    564651
    … …  
    572659
    573660   }
     661
     662   $queuestatus = $subj_status;
     663   $limshost    = $dbhost;
     664   if ( $limshost == 'localhost' )
     665      $limshost    = gethostname();
    574666
    575667   // Parse the editXMLFilename
    … …  
    596688
    597689   Submission Time : $submittime
     690   LIMS Host       : $limshost
    598691   Analysis ID     : $gfacID
    599692   Request ID      : $requestID  ( $db )

  • trunk/cleanup_aira.php

    r8 r10  
    44 *
    55 * functions relating to copying results and cleaning up the gfac DB
    6  *  where the job used an Airavata interface.
     6 *  where the job used an Airavata interface (production version).
    77 *
    88 */
    … …  
    1010$us3bin = exec( "ls -d ~us3/bin" );
    1111include_once "$us3bin/listen-config.php";
    12 include "/srv/www/htdocs/common/class/experiment_status.php";
    13 include "/srv/www/htdocs/common/class/experiment_errors.php";
     12include $class_dir_p . "experiment_status.php";
     13include $class_dir_p . "experiment_errors.php";
    1414$me              = 'cleanup_aira.php';
    15 $email_address   = '';
    16 $queuestatus     = '';
    17 $jobtype         = '';
    18 $db              = '';
    19 $editXMLFilename = '';
    20 $status          = '';
     15$class_dir       = $class_dir_p;
     16include_once "cleanup.php";
    2117
    22 function aira_cleanup( $us3_db, $reqID, $gfac_link )
    23 {
    24    global $dbhost;
    25    global $user;
    26    global $passwd;
    27    global $db;
    28    global $guser;
    29    global $gpasswd;
    30    global $gDB;
    31    global $me;
    32    global $work;
    33    global $email_address;
    34    global $queuestatus;
    35    global $jobtype;
    36    global $editXMLFilename;
    37    global $submittime;
    38    global $status;
    39    global $stderr;
    40    global $stdout;
    41    global $tarfile;
    42    global $requestID;
    43    global $submit_dir;
    44    $me        = 'cleanup_aira.php';
    45 
    46    $requestID = $reqID;
    47    $db = $us3_db;
    48    write_log( "$me: debug db=$db; requestID=$requestID" );
    49 
    50    $us3_link = mysql_connect( $dbhost, $user, $passwd );
    51 
    52    if ( ! $us3_link )
    53    {
    54       write_log( "$me: could not connect: $dbhost, $user, $passwd" );
    55       mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
    56       return( -1 );
    57    }
    58 
    59    $result = mysql_select_db( $db, $us3_link );
    60 
    61    if ( ! $result )
    62    {
    63       write_log( "$me: could not select DB $db" );
    64       mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
    65       return( -1 );
    66    }
    67 
    68    // First get basic info for email messages
    69    $query  = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
    70              "WHERE HPCAnalysisRequestID=$requestID";
    71    $result = mysql_query( $query, $us3_link );
    72 
    73    if ( ! $result )
    74    {
    75       write_log( "$me: Bad query: $query" );
    76       mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
    77       return( -1 );
    78    }
    79 
    80    list( $email_address, $investigatorGUID, $editXMLFilename ) =  mysql_fetch_array( $result );
    81 
    82    $query  = "SELECT personID FROM people " .
    83              "WHERE personGUID='$investigatorGUID'";
    84    $result = mysql_query( $query, $us3_link );
    85 
    86    list( $personID ) = mysql_fetch_array( $result );
    87 
    88    $query  = "SELECT clusterName, submitTime, queueStatus, method "              .
    89              "FROM HPCAnalysisRequest h, HPCAnalysisResult r "                   .
    90              "WHERE h.HPCAnalysisRequestID=$requestID "                          .
    91              "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
    92 
    93    $result = mysql_query( $query, $us3_link );
    94 
    95    if ( ! $result )
    96    {
    97       write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    98       return( -1 );
    99    }
    100 
    101    if ( mysql_num_rows( $result ) == 0 )
    102    {
    103       write_log( "$me: US3 Table error - No records for requestID: $requestID" );
    104       return( -1 );
    105    }
    106 
    107    list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
    108 
    109    // Get the GFAC ID
    110    $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
    111             "WHERE HPCAnalysisRequestID=$requestID";
    112 
    113    $result = mysql_query( $query, $us3_link );
    114 
    115    if ( ! $result )
    116    {
    117       write_log( "$me: Bad query: $query" );
    118       mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
    119       return( -1 );
    120    }
    121 
    122    list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
    123 
    124    // Get data from global GFAC DB then insert it into US3 DB
    125 
    126    $result = mysql_select_db( $gDB, $gfac_link );
    127 
    128    if ( ! $result )
    129    {
    130       write_log( "$me: Could not connect to DB $gDB" );
    131       mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
    132       return( -1 );
    133    }
    134 
    135    $query = "SELECT status, cluster, id FROM analysis " .
    136             "WHERE gfacID='$gfacID'";
    137 
    138    $result = mysql_query( $query, $gfac_link );
    139    if ( ! $result )
    140    {
    141       write_log( "$me: Could not select GFAC status for $gfacID" );
    142       mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
    143       return( -1 );
    144    }
    145    
    146    list( $status, $cluster, $id ) = mysql_fetch_array( $result );
    147 
    148    if ( $cluster == 'bcf-local'  || $cluster == 'alamo-local' )
    149    {
    150          $clushost = $cluster;
    151          $clushost = preg_replace( "/\-local/", "", $clushost );
    152          get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
    153    }
    154 
    155 
    156    $query = "SELECT id FROM analysis " .
    157             "WHERE gfacID='$gfacID'";
    158 
    159    $result = mysql_query( $query, $gfac_link );
    160 
    161    if ( ! $result )
    162    {
    163       write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
    164       mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
    165       return( -1 );
    166    }
    167 
    168    list( $analysisID ) = mysql_fetch_array( $result );
    169 
    170    // Get the request guid (LIMS submit dir name)
    171    $query  = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
    172              "WHERE HPCAnalysisRequestID = $requestID ";
    173    $result = mysql_query( $query, $us3_link );
    174    
    175    if ( ! $result )
    176    {
    177       write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    178    }
    179 
    180    list( $requestGUID ) = mysql_fetch_array( $result );
    181    $output_dir = "$submit_dir/$requestGUID";
    182 
    183    // Get stderr,stdout,tarfile from work directory
    184    if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
    185    chdir( "$output_dir" );
    186 //write_log( "$me: gfacID=$gfacID" );
    187 //write_log( "$me: submit_dir=$submit_dir" );
    188 //write_log( "$me: requestGUID=$requestGUID" );
    189 write_log( "$me: output_dir=$output_dir" );
    190 
    191    $stderr     = "";
    192    $stdout     = "";
    193    $tarfile    = "";
    194    $fn_stderr  = "Ultrascan.stderr";
    195    $fn_stdout  = "Ultrascan.stdout";
    196    $fn_tarfile = "analysis-results.tar";
    197    $num_try    = 0;
    198    while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
    199    {
    200       sleep( 10 );
    201       $num_try++;
    202    }
    203 
    204    $ofiles     = scandir( $output_dir );
    205    foreach ( $ofiles as $ofile )
    206    {
    207       if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
    208          $fn_stderr  = $ofile;
    209       if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
    210          $fn_stdout  = $ofile;
    211 //write_log( "$me:    ofile=$ofile" );
    212    }
    213 write_log( "$me: fn_stderr=$fn_stderr" );
    214 write_log( "$me: fn_stdout=$fn_stdout" );
    215 if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
    216 else                          write_log( "$me: NOT FOUND: $fn_tarfile" );
    217 
    218    if ( file_exists( $fn_stderr  ) ) $stderr   = file_get_contents( $fn_stderr  );
    219    if ( file_exists( $fn_stdout  ) ) $stdout   = file_get_contents( $fn_stdout  );
    220    if ( file_exists( $fn_tarfile ) ) $tarfile  = file_get_contents( $fn_tarfile );
    221 
    222    if ( $cluster == 'alamo'  || $cluster == 'alamo-local' )
    223    {  // Filter "ipath_userinit" lines out of alamo stdout lines
    224       $prefln = strlen( $stdout );
    225       $output = array();
    226       exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
    227       $stdout = implode( "\n", $output );
    228       $posfln = strlen( $stdout );
    229 write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
    230    }
    231 
    232    // Save queue messages for post-mortem analysis
    233    $query = "SELECT message, time FROM queue_messages " .
    234             "WHERE analysisID = $analysisID " .
    235             "ORDER BY time ";
    236    $result = mysql_query( $query, $gfac_link );
    237 
    238    if ( ! $result )
    239    {
    240       // Just log it and continue
    241       write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
    242    }
    243 
    244    $now = date( 'Y-m-d H:i:s' );
    245    $message_log = "US3 DB: $db\n" .
    246                   "RequestID: $requestID\n" .
    247                   "GFAC ID: $gfacID\n" .
    248                   "Processed: $now\n\n" .
    249                   "Queue Messages\n\n" ;
    250    if ( mysql_num_rows( $result ) > 0 )
    251    {
    252       while ( list( $message, $time ) = mysql_fetch_array( $result ) )
    253          $message_log .= "$time $message\n";
    254    }
    255 
    256    $query = "DELETE FROM queue_messages " .
    257             "WHERE analysisID = $analysisID ";
    258 
    259    $result = mysql_query( $query, $gfac_link );
    260 
    261    if ( ! $result )
    262    {
    263       // Just log it and continue
    264       write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
    265    }
    266 
    267    $query = "SELECT queue_msg FROM analysis " .
    268             "WHERE gfacID='$gfacID' ";
    269 
    270    $result = mysql_query( $query, $gfac_link );
    271    list( $queue_msg ) = mysql_fetch_array( $result );
    272 
    273    // But let's allow for investigation of other large stdout and/or stderr
    274    if ( strlen( $stdout ) > 20480000 ||
    275         strlen( $stderr ) > 20480000 )
    276       write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
    277 
    278    $message_log .= "\n\n\nStdout Contents\n\n" .
    279                    $stdout .
    280                    "\n\n\nStderr Contents\n\n" .
    281                    $stderr .
    282                    "\n\n\nGFAC Status: $status\n" .
    283                    "GFAC message field: $queue_msg\n";
    284 
    285    // Delete data from GFAC DB
    286    $query = "DELETE from analysis WHERE gfacID='$gfacID'";
    287 
    288    $result = mysql_query( $query, $gfac_link );
    289 
    290    if ( ! $result )
    291    {
    292       // Just log it and continue
    293       write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
    294    }
    295 
    296 
    297    // Try to create it if necessary, and write the file
    298    // Let's use FILE_APPEND, in case this is the second time around and the
    299    //  GFAC job status was INSERTed, rather than UPDATEd
    300    if ( ! is_dir( $output_dir ) )
    301       mkdir( $output_dir, 0775, true );
    302    $message_filename = "$output_dir/$db-$requestID-messages.txt";
    303    file_put_contents( $message_filename, $message_log, FILE_APPEND );
    304   // mysql_close( $gfac_link );
    305 
    306    /////////
    307    // Insert data into HPCAnalysis
    308 
    309    $query = "UPDATE HPCAnalysisResult SET "                              .
    310             "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
    311             "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
    312             "queueStatus='completed' " .
    313             "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
    314 
    315    $result = mysql_query( $query, $us3_link );
    316 
    317    if ( ! $result )
    318    {
    319       write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    320       mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
    321       return( -1 );
    322    }
    323 
    324    // Delete data from GFAC DB
    325    $query = "DELETE from analysis WHERE gfacID='$gfacID'";
    326 
    327    $result = mysql_query( $query, $gfac_link );
    328 
    329    if ( ! $result )
    330    {
    331       // Just log it and continue
    332       write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
    333    }
    334 
    335    // Expand the tar file
    336 
    337    if ( strlen( $tarfile ) == 0 )
    338    {
    339       write_log( "$me: No tarfile" );
    340       mail_to_user( "fail", "No results" );
    341       return( -1 );
    342    }
    343 
    344    $tar_out = array();
    345    exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
    346 
    347    // Insert the model files and noise files
    348    $files      = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
    349    $noiseIDs   = array();
    350    $modelGUIDs = array();
    351 
    352    foreach ( $files as $file )
    353    {
    354       $split = explode( ";", $file );
    355 
    356       if ( count( $split ) > 1 )
    357       {
    358          list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
    359      
    360          list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
    361          list( $other, $variance     ) = explode( "=", $variance );
    362          list( $other, $meniscus     ) = explode( "=", $meniscus );
    363       }
    364       else
    365          $fn = $file;
    366 
    367       if ( preg_match( "/mdl.tmp$/", $fn ) )
    368          continue;
    369 
    370       if ( filesize( $fn ) < 100 )
    371       {
    372          write_log( "$me:fn is invalid $fn size filesize($fn)" );
    373          mail_to_user( "fail", "Internal error\n$fn is invalid" );
    374          return( -1 );
    375       }
    376 
    377       if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
    378       {
    379          $xml         = file_get_contents( $fn );
    380          $statistics  = parse_xml( $xml, 'statistics' );
    381 //         $ntries      = 0;
    382 //
    383 //         while ( $statistics['cpucount'] < 1  &&  $ntries < 3 )
    384 //         {  // job_statistics file not totally copied, so retry
    385 //            sleep( 10 );
    386 //            $xml         = file_get_contents( $fn );
    387 //            $statistics  = parse_xml( $xml, 'statistics' );
    388 //            $ntries++;
    389 //write_log( "$me:jobstats retry $ntries" );
    390 //         }
    391 //write_log( "$me:cputime=$statistics['cputime']" );
    392 
    393          $otherdata   = parse_xml( $xml, 'id' );
    394 
    395          $query = "UPDATE HPCAnalysisResult SET "   .
    396                   "wallTime = {$statistics['walltime']}, " .
    397                   "CPUTime = {$statistics['cputime']}, " .
    398                   "CPUCount = {$statistics['cpucount']}, " .
    399                   "max_rss = {$statistics['maxmemory']}, " .
    400                   "startTime = '{$otherdata['starttime']}', " .
    401                   "endTime = '{$otherdata['endtime']}', " .
    402                   "mgroupcount = {$otherdata['groupcount']} " .
    403                   "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
    404          $result = mysql_query( $query, $us3_link );
    405 
    406          if ( ! $result )
    407          {
    408             write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    409          }
    410 
    411          file_put_contents( "$output_dir/$fn", $xml );    // Copy to submit dir
    412 
    413       }
    414 
    415       else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
    416       {
    417          $xml        = file_get_contents( $fn );
    418          $noise_data = parse_xml( $xml, "noise" );
    419          $type       = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
    420          $desc       = $noise_data[ 'description' ];
    421          $modelGUID  = $noise_data[ 'modelGUID' ];
    422          $noiseGUID  = $noise_data[ 'noiseGUID' ];
    423 
    424          $query = "INSERT INTO noise SET "  .
    425                   "noiseGUID='$noiseGUID'," .
    426                   "modelGUID='$modelGUID'," .
    427                   "editedDataID=1, "        .
    428                   "modelID=1, "             .
    429                   "noiseType='$type',"      .
    430                   "description='$desc',"    .
    431                   "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
    432 
    433          // Add later after all files are processed: editDataID, modelID
    434 
    435          $result = mysql_query( $query, $us3_link );
    436 
    437          if ( ! $result )
    438          {
    439             write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    440             mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
    441             return( -1 );
    442          }
    443 
    444          $id        = mysql_insert_id( $us3_link );
    445          $file_type = "noise";
    446          $noiseIDs[] = $id;
    447 
    448          // Keep track of modelGUIDs for later, when we replace them
    449          $modelGUIDs[ $id ] = $modelGUID;
    450          
    451       }
    452 
    453       else if ( preg_match( "/\.mrecs/", $fn ) > 0 )  // It's an mrecs file
    454       {
    455          $xml         = file_get_contents( $fn );
    456          $mrecs_data  = parse_xml( $xml, "modelrecords" );
    457          $desc        = $mrecs_data[ 'description' ];
    458          $editGUID    = $mrecs_data[ 'editGUID' ];
    459 write_log( "$me:   mrecs file editGUID=$editGUID" );
    460          if ( strlen( $editGUID ) < 36 )
    461             $editGUID    = "12345678-0123-5678-0123-567890123456";
    462          $mrecGUID    = $mrecs_data[ 'mrecGUID' ];
    463          $modelGUID   = $mrecs_data[ 'modelGUID' ];
    464 
    465          $query = "INSERT INTO pcsa_modelrecs SET "  .
    466                   "editedDataID="                .
    467                   "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
    468                   "modelID=0, "             .
    469                   "mrecsGUID='$mrecGUID'," .
    470                   "description='$desc',"    .
    471                   "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
    472 
    473          // Add later after all files are processed: editDataID, modelID
    474 
    475          $result = mysql_query( $query, $us3_link );
    476 
    477          if ( ! $result )
    478          {
    479             write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    480             mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
    481             return( -1 );
    482          }
    483 
    484          $id         = mysql_insert_id( $us3_link );
    485          $file_type  = "mrecs";
    486          $mrecsIDs[] = $id;
    487 
    488          // Keep track of modelGUIDs for later, when we replace them
    489          $rmodlGUIDs[ $id ] = $modelGUID;
    490 //write_log( "$me:   mrecs file inserted into DB : id=$id" );
    491       }
    492 
    493       else                                           // It's a model file
    494       {
    495          $xml         = file_get_contents( $fn );
    496          $model_data  = parse_xml( $xml, "model" );
    497          $description = $model_data[ 'description' ];
    498          $modelGUID   = $model_data[ 'modelGUID' ];
    499          $editGUID    = $model_data[ 'editGUID' ];
    500 
    501          if ( $mc_iteration > 1 )
    502          {
    503             $miter       = sprintf( "_mcN%03d", $mc_iteration );
    504             $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
    505 write_log( "$me:   MODELUpd: O:description=$description" );
    506          }
    507 
    508          $query = "INSERT INTO model SET "       .
    509                   "modelGUID='$modelGUID',"      .
    510                   "editedDataID="                .
    511                   "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
    512                   "description='$description',"  .
    513                   "MCIteration='$mc_iteration'," .
    514                   "meniscus='$meniscus'," .
    515                   "variance='$variance'," .
    516                   "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
    517 
    518          $result = mysql_query( $query, $us3_link );
    519 
    520          if ( ! $result )
    521          {
    522             write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
    523             mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
    524             return( -1 );
    525          }
    526 
    527          $modelID   = mysql_insert_id( $us3_link );
    528          $id        = $modelID;
    529          $file_type = "model";
    530 
    531          $query = "INSERT INTO modelPerson SET " .
    532                   "modelID=$modelID, personID=$personID";
    533          $result = mysql_query( $query, $us3_link );
    534       }
    535 
    536       $query = "INSERT INTO HPCAnalysisResultData SET "       .
    537                "HPCAnalysisResultID='$HPCAnalysisResultID', " .
    538                "HPCAnalysisResultType='$file_type', "         .
    539                "resultID=$id";
    540 
    541       $result = mysql_query( $query, $us3_link );
    542 
    543       if ( ! $result )
    544       {
    545          write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    546          mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
    547          return( -1 );
    548       }
    549    }
    550 
    551    // Now fix up noise entries
    552    // For noise files, there is, at most two: ti_noise and ri_noise
    553    // In this case there will only be one modelID
    554 
    555    foreach ( $noiseIDs as $noiseID )
    556    {
    557       $modelGUID = $modelGUIDs[ $noiseID ];
    558       $query = "UPDATE noise SET "                                                 .
    559                "editedDataID="                                                     .
    560                "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID'),"    .
    561                "modelID="                                                          .
    562                "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')"          .
    563                "WHERE noiseID=$noiseID";
    564 
    565       $result = mysql_query( $query, $us3_link );
    566 
    567       if ( ! $result )
    568       {
    569          write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    570          mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
    571          return( -1 );
    572       }
    573    }
    574 
    575    // Copy results to LIMS submit directory (files there are deleted after 7 days)
    576    global $submit_dir; // LIMS submit files dir
    577    
    578    // Get the request guid (LIMS submit dir name)
    579    $query  = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
    580              "WHERE HPCAnalysisRequestID = $requestID ";
    581    $result = mysql_query( $query, $us3_link );
    582    
    583    if ( ! $result )
    584    {
    585       write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    586    }
    587    
    588 //   list( $requestGUID ) = mysql_fetch_array( $result );
    589 //   
    590 //   chdir( "$submit_dir/$requestGUID" );
    591 //   $f = fopen( "analysis-results.tar", "w" );
    592 //   fwrite( $f, $tarfile );
    593 //   fclose( $f );
    594 
    595    // Clean up
    596 //   chdir ( $work );
    597    // exec( "rm -rf $gfacID" );
    598 
    599    mysql_close( $us3_link );
    600 
    601    /////////
    602    // Send email
    603 
    604    mail_to_user( "success", "" );
    605 }
    606 
    607 function mail_to_user( $type, $msg )
    608 {
    609    // Note to me. Just changed subject line to include a modified $status instead
    610    // of the $type variable passed. More informative than just "fail" or "success."
    611    // See how it works for awhile and then consider removing $type parameter from
    612    // function.
    613    global $email_address;
    614    global $submittime;
    615    global $queuestatus;
    616    global $status;
    617    global $cluster;
    618    global $jobtype;
    619    global $org_name;
    620    global $admin_email;
    621    global $db;
    622    global $dbhost;
    623    global $requestID;
    624    global $gfacID;
    625    global $editXMLFilename;
    626    global $stdout;
    627 
    628 global $me;
    629 write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
    630 
    631    // Get GFAC status and message
    632    // function get_gfac_message() also sets global $status
    633    $gfac_message = get_gfac_message( $gfacID );
    634    if ( $gfac_message === false ) $gfac_message = "Job Finished";
    635      
    636    // Create a status to put in the subject line
    637    switch ( $status )
    638    {
    639       case "COMPLETE":
    640          $subj_status = 'completed';
    641          break;
    642 
    643       case "CANCELLED":
    644       case "CANCELED":
    645          $subj_status = 'canceled';
    646          break;
    647 
    648       case "FAILED":
    649          $subj_status = 'failed';
    650          if ( preg_match( "/^US3-AIRA/i", $gfacID ) )
    651          {  // For A/Thrift FAIL, get error message
    652             $gfac_message = getExperimentErrors( $gfacID );
    653 //$gfac_message .= "Test ERROR MESSAGE";
    654          }
    655          break;
    656 
    657       case "ERROR":
    658          $subj_status = 'unknown error';
    659          break;
    660 
    661       default:
    662          $subj_status = $status;       // For now
    663          break;
    664 
    665    }
    666 
    667    $queuestatus = $subj_status;
    668    $limshost    = $dbhost;
    669    if ( $limshost == 'localhost' )
    670       $limshost    = gethostname();
    671 
    672    // Parse the editXMLFilename
    673    list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
    674       explode( ".", $editXMLFilename );
    675 
    676    $headers  = "From: $org_name Admin<$admin_email>"     . "\n";
    677    $headers .= "Cc: $org_name Admin<$admin_email>"       . "\n";
    678 
    679    // Set the reply address
    680    $headers .= "Reply-To: $org_name<$admin_email>"      . "\n";
    681    $headers .= "Return-Path: $org_name<$admin_email>"   . "\n";
    682 
    683    // Try to avoid spam filters
    684    $now = time();
    685    $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
    686    $headers .= "X-Mailer: PHP v" . phpversion()         . "\n";
    687    $headers .= "MIME-Version: 1.0"                      . "\n";
    688    $headers .= "Content-Transfer-Encoding: 8bit"        . "\n";
    689 
    690    $subject       = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
    691    $message       = "
    692    Your UltraScan job is complete:
    693 
    694    Submission Time : $submittime
    695    LIMS Host       : $limshost
    696    Analysis ID     : $gfacID
    697    Request ID      : $requestID  ( $db )
    698    RunID           : $runID
    699    EditID          : $editID
    700    Data Type       : $dataType
    701    Cell/Channel/Wl : $cell / $channel / $wl
    702    Status          : $queuestatus
    703    Cluster         : $cluster
    704    Job Type        : $jobtype
    705    GFAC Status     : $status
    706    GFAC Message    : $gfac_message
    707    Stdout          : $stdout
    708    ";
    709 
    710    if ( $type != "success" ) $message .= "Grid Ctrl Error :  $msg\n";
    711 
    712    // Handle the error case where an error occurs before fetching the
    713    // user's email address
    714    if ( $email_address == "" ) $email_address = $admin_email;
    715 
    716    mail( $email_address, $subject, $message, $headers );
    717 }
    718 
    719 function parse_xml( $xml, $type )
    720 {
    721    $parser = new XMLReader();
    722    $parser->xml( $xml );
    723 
    724    $results = array();
    725 
    726    while ( $parser->read() )
    727    {
    728       if ( $parser->name == $type )
    729       {
    730          while ( $parser->moveToNextAttribute() )
    731          {
    732             $results[ $parser->name ] = $parser->value;
    733          }
    734 
    735          break;
    736       }
    737    }
    738 
    739    $parser->close();
    740    return $results;
    741 }
    742 
    743 // Function to get information about the current job GFAC
    744 function get_gfac_message( $gfacID )
    745 {
    746   global $serviceURL;
    747   global $me;
    748 
    749   $hex = "[0-9a-fA-F]";
    750   if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
    751        ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
    752    {
    753       // Then it's not a GFAC job
    754       return false;
    755    }
    756 
    757    $url = "$serviceURL/jobstatus/$gfacID";
    758    try
    759    {
    760       $post = new HttpRequest( $url, HttpRequest::METH_GET );
    761       $http = $post->send();
    762       $xml  = $post->getResponseBody();     
    763    }
    764    catch ( HttpException $e )
    765    {
    766       write_log( "$me: Job status not available - $gfacID" );
    767       return false;
    768    }
    769 
    770    // Parse the result
    771    $gfac_message = parse_message( $xml );
    772 
    773    return $gfac_message;
    774 }
    775 
    776 function parse_message( $xml )
    777 {
    778    global $status;
    779    $status       = "";
    780    $gfac_message = "";
    781 
    782    $parser = new XMLReader();
    783    $parser->xml( $xml );
    784 
    785    $results = array();
    786 
    787    while( $parser->read() )
    788    {
    789       $type = $parser->nodeType;
    790 
    791       if ( $type == XMLReader::ELEMENT )
    792          $name = $parser->name;
    793 
    794       else if ( $type == XMLReader::TEXT )
    795       {
    796          if ( $name == "status" )
    797             $status       = $parser->value;
    798          else
    799             $gfac_message = $parser->value;
    800       }
    801    }
    802      
    803    $parser->close();
    804    return $gfac_message;
    805 }
    806 
    807 function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
    808 {
    809    global $work;
    810    global $work_remote;
    811    global $me;
    812    global $db;
    813    global $status;
    814 
    815    // Figure out local working directory
    816    if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
    817    $pwd = chdir( "$work/$gfacID" );
    818 
    819    // Figure out remote directory
    820    $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
    821 
    822    // Get stdout, stderr, output/analysis-results.tar
    823    $output = array();
    824 //   $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
    825 //
    826 //   exec( $cmd, $output, $stat );
    827 //   if ( $stat != 0 )
    828 //      write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
    829 //     
    830 //   $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
    831 //   exec( $cmd, $output, $stat );
    832 //   if ( $stat != 0 )
    833 //      write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
    834 
    835    $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
    836    exec( $cmd, $output, $stat );
    837    if ( $stat != 0 )
    838       write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
    839 
    840    $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
    841    exec( $cmd, $output, $stat );
    842    if ( $stat != 0 )
    843    {
    844       write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
    845       sleep( 10 );
    846       write_log( "$me: RETRY" );
    847       exec( $cmd, $output, $stat );
    848       if ( $stat != 0 )
    849          write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
    850    }
    851      
    852    $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
    853    exec( $cmd, $output, $stat );
    854    if ( $stat != 0 )
    855    {
    856       write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
    857       sleep( 10 );
    858       write_log( "$me: RETRY" );
    859       exec( $cmd, $output, $stat );
    860       if ( $stat != 0 )
    861          write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
    862    }
    863 
    864    // Write the files to gfacDB
    865 
    866    if ( file_exists( "stderr" ) ) $stderr  = file_get_contents( "stderr" );
    867    if ( file_exists( "stdout" ) ) $stdout  = file_get_contents( "stdout" );
    868    if ( file_exists( "analysis-results.tar" ) )
    869       $tarfile = file_get_contents( "analysis-results.tar" );
    870 
    871    $query = "UPDATE analysis SET " .
    872             "stderr='"  . mysql_real_escape_string( $stderr,  $gfac_link ) . "'," .
    873             "stdout='"  . mysql_real_escape_string( $stdout,  $gfac_link ) . "'," .
    874             "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
    875 
    876    $result = mysql_query( $query, $gfac_link );
    877 
    878    if ( ! $result )
    879    {
    880       write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
    881       echo "Bad query\n";
    882       return( -1 );
    883    }
    884 }
    88518?>

  • trunk/gridctl.php

    r6 r10  
    11<?php
    2 
    3 $us3bin = exec( "ls -d ~us3/bin" );
    4 include_once "$us3bin/listen-config.php";
    5 include "$us3bin/cleanup_aira.php";
    6 include "$us3bin/cleanup_gfac.php";
    72
    83// Global variables
    … …  
    7469   {
    7570      $status_in  = $status;
    76 //write_log( "$loghdr status_in=$status_in" );
    7771      $status     = aira_status( $gfacID, $status_in );
    7872if($status != $status_in )
    … …  
    155149      case "FINISHED":
    156150      case "DONE":
    157          if ( is_aira_job( $gfacID ) )
    158          {
    159             complete();
    160          }
     151//         if ( is_aira_job( $gfacID ) )
     152//         {
     153//            complete();
     154//         }
    161155      case "PROCESSING":
    162156      default:
    … …  
    437431   global $us3_db;
    438432   global $loghdr;
     433   global $class_dir;
    439434
    440435   // Double check that the gfacID exists
    … …  
    460455   if ( $requestID == 0 ) return;
    461456
    462    if ( preg_match( "/US3-AIRA/i", $gfacID ) )
    463    {
     457   if ( preg_match( "/US3-A/i", $gfacID ) )
     458   {
     459      $me_devel  = preg_match( "/class_devel/", $class_dir );
     460      $job_devel = preg_match( "/US3-ADEV/i", $gfacID );
     461      if ( ( !$me_devel  &&  !$job_devel )  ||
     462           (  $me_devel  &&   $job_devel ) )
     463      {  // If job from appropriate Airavata server, process it
    464464//write_log( "$loghdr CALLING aira_cleanup()" );
    465       aira_cleanup( $us3_db, $requestID, $gLink );
     465         aira_cleanup( $us3_db, $requestID, $gLink );
     466      }
    466467//write_log( "$loghdr RTN FR aira_cleanup()" );
    467468   }
    … …  
    625626   global $cluster;
    626627
    627    if ( preg_match( "/US3-AIRA/i", $gfacID )  &&
     628   if ( preg_match( "/US3-A/i", $gfacID )  &&
    628629        ! preg_match( "/juropa/i", $cluster ) )
    629630   {
    … …  
    997998   global $self;
    998999   global $loghdr;
     1000   global $class_dir;
    9991001//echo "a_st: st_in$status_in : $gfacID\n";
    10001002   //$status_gw = standard_status( $status_in );
    … …  
    10021004//echo "a_st:  st_db=$status_gw\n";
    10031005   $status    = $status_gw;
    1004 
    1005    if ( preg_match( "/US3-AIRA/i", $gfacID ) )
    1006    {
     1006   $me_devel  = preg_match( "/class_devel/", $class_dir );
     1007   $job_devel = preg_match( "/US3-ADEV/i", $gfacID );
     1008   $devmatch  = ( ( !$me_devel  &&  !$job_devel )  ||
     1009                  (  $me_devel  &&   $job_devel ) );
     1010
     1011//write_log( "$loghdr  gfacID=$gfacID  devmatch=$devmatch" );
     1012//write_log( "$loghdr   me_d=$me_devel  jo_d=$job_devel  cd=$class_dir" );
     1013   if ( preg_match( "/US3-A/i", $gfacID )  &&  $devmatch )
     1014   {  // Airavata job and development/production type is right
    10071015      $status_ex = getExperimentStatus( $gfacID );
     1016//write_log( "$loghdr status_ex $status_ex" );
    10081017
    10091018      if ( $status_ex == 'COMPLETED' )
    … …  
    10471056      }
    10481057
    1049 write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
     1058//write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
     1059//write_log( "  me_d=$me_devel jo_d=$job_devel dm=$devmatch cd=$class_dir" );
    10501060      if ( $status != $status_gw )
    10511061      {

  • trunk/manage-us3-pipe.php

    r6 r10  
    8989   mysql_close( $resource );
    9090
    91    $is_athrift  = preg_match( "/^US3-AIRA/i", $gfacID );
     91   $is_athrift  = preg_match( "/^US3-A/i", $gfacID );
    9292   $is_finished = preg_match( "/^Finished/i", $message );
    9393
Note: See TracChangeset for help on using the changeset viewer.