Changeset 10 for trunk/cleanup.php


Ignore:
Timestamp:
Jun 4, 2015, 9:48:05 PM (9 years ago)
Author:
gegorbet
Message:

fixes to allow simultaneous production/development jobs

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/cleanup.php

    r9 r10  
    44 *
    55 * functions relating to copying results and cleaning up the gfac DB
     6 *  where the job used an Airavata interface.
    67 *
    78 */
    89
    9 $us3bin = exec( "ls -d ~us3/bin" );
    10 include_once "$us3bin/listen-config.php";
    11 $me              = 'cleanup.php';
    1210$email_address   = '';
    1311$queuestatus     = '';
    … …  
    1715$status          = '';
    1816
    19 function gfac_cleanup( $us3_db, $reqID, $gfac_link )
     17function aira_cleanup( $us3_db, $reqID, $gfac_link )
    2018{
    2119   global $dbhost;
    … …  
    3432   global $submittime;
    3533   global $status;
     34   global $stderr;
    3635   global $stdout;
     36   global $tarfile;
    3737   global $requestID;
     38   global $submit_dir;
     39   $me        = 'cleanup_aira.php';
    3840
    3941   $requestID = $reqID;
    … …  
    7981   list( $personID ) = mysql_fetch_array( $result );
    8082
    81    /*
    82    $query  = "SELECT clusterName, submitTime, queueStatus, method "              .
    83              "FROM HPCAnalysisRequest h LEFT JOIN HPCAnalysisResult "            .
    84              "ON h.HPCAnalysisRequestID=HPCAnalysisResult.HPCAnalysisRequestID " .
    85              "WHERE h.HPCAnalysisRequestID=$requestID";
    86    */
    8783   $query  = "SELECT clusterName, submitTime, queueStatus, method "              .
    8884             "FROM HPCAnalysisRequest h, HPCAnalysisResult r "                   .
    … …  
    121117   list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
    122118
    123    ////////
    124    // Get data from global GFAC DB and insert it into US3 DB
    125    // $gfac_link = mysql_connect( $dbhost, $guser, $gpasswd );
     119   // Get data from global GFAC DB then insert it into US3 DB
    126120
    127121   $result = mysql_select_db( $gDB, $gfac_link );
    … …  
    154148   }
    155149
    156    $query = "SELECT id, stderr, stdout, tarfile FROM analysis " .
     150
     151   $query = "SELECT id FROM analysis " .
    157152            "WHERE gfacID='$gfacID'";
    158153
    … …  
    166161   }
    167162
    168    list( $analysisID, $stderr, $stdout, $tarfile ) = mysql_fetch_array( $result );
     163   list( $analysisID ) = mysql_fetch_array( $result );
     164
     165   // Get the request guid (LIMS submit dir name)
     166   $query  = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
     167             "WHERE HPCAnalysisRequestID = $requestID ";
     168   $result = mysql_query( $query, $us3_link );
     169   
     170   if ( ! $result )
     171   {
     172      write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
     173   }
     174
     175   list( $requestGUID ) = mysql_fetch_array( $result );
     176   $output_dir = "$submit_dir/$requestGUID";
     177
     178   // Get stderr,stdout,tarfile from work directory
     179   if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
     180   chdir( "$output_dir" );
     181//write_log( "$me: gfacID=$gfacID" );
     182//write_log( "$me: submit_dir=$submit_dir" );
     183//write_log( "$me: requestGUID=$requestGUID" );
     184write_log( "$me: output_dir=$output_dir" );
     185
     186   $stderr     = "";
     187   $stdout     = "";
     188   $tarfile    = "";
     189   $fn_stderr  = "Ultrascan.stderr";
     190   $fn_stdout  = "Ultrascan.stdout";
     191   $fn_tarfile = "analysis-results.tar";
     192   $num_try    = 0;
     193   while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
     194   {
     195      sleep( 10 );
     196      $num_try++;
     197   }
     198
     199   $ofiles     = scandir( $output_dir );
     200   foreach ( $ofiles as $ofile )
     201   {
     202      if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
     203         $fn_stderr  = $ofile;
     204      if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
     205         $fn_stdout  = $ofile;
     206//write_log( "$me:    ofile=$ofile" );
     207   }
     208write_log( "$me: fn_stderr=$fn_stderr" );
     209write_log( "$me: fn_stdout=$fn_stdout" );
     210if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
     211else                          write_log( "$me: NOT FOUND: $fn_tarfile" );
     212
     213   if ( file_exists( $fn_stderr  ) ) $stderr   = file_get_contents( $fn_stderr  );
     214   if ( file_exists( $fn_stdout  ) ) $stdout   = file_get_contents( $fn_stdout  );
     215   if ( file_exists( $fn_tarfile ) ) $tarfile  = file_get_contents( $fn_tarfile );
     216
     217   if ( $cluster == 'alamo'  || $cluster == 'alamo-local' )
     218   {  // Filter "ipath_userinit" lines out of alamo stdout lines
     219      $prefln = strlen( $stdout );
     220      $output = array();
     221      exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
     222      $stdout = implode( "\n", $output );
     223      $posfln = strlen( $stdout );
     224write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
     225   }
    169226
    170227   // Save queue messages for post-mortem analysis
    … …  
    203260   }
    204261
    205    // Save stdout, stderr, etc. for message log
    206    $query  = "SELECT stdout, stderr, status, queue_msg FROM analysis " .
    207              "WHERE gfacID='$gfacID' ";
     262   $query = "SELECT queue_msg FROM analysis " .
     263            "WHERE gfacID='$gfacID' ";
     264
    208265   $result = mysql_query( $query, $gfac_link );
    209    try
    210    {
    211       // What if this is too large?
    212       list( $stdout, $stderr, $status, $queue_msg ) = mysql_fetch_array( $result );
    213    }
    214    catch ( Exception $e )
    215    {
    216       write_log( "$me: stdout + stderr larger than 128M - $gfacID\n" . mysql_error( $gfac_link ) );
    217       // Just go ahead and clean up
    218    }
     266   list( $queue_msg ) = mysql_fetch_array( $result );
    219267
    220268   // But let's allow for investigation of other large stdout and/or stderr
    … …  
    241289   }
    242290
    243    // Copy queue messages to LIMS submit directory (files there are deleted after 7 days)
    244    global $submit_dir;
    245    
    246    // Get the request guid (LIMS submit dir name)
    247    $query  = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
    248              "WHERE HPCAnalysisRequestID = $requestID ";
    249    $result = mysql_query( $query, $us3_link );
    250    
    251    if ( ! $result )
    252    {
    253       write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
    254    }
    255    
    256    list( $requestGUID ) = mysql_fetch_array( $result );
    257    $output_dir = "$submit_dir/$requestGUID";
    258291
    259292   // Try to create it if necessary, and write the file
    … …  
    271304   $query = "UPDATE HPCAnalysisResult SET "                              .
    272305            "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
    273             "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "' "  .
     306            "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
     307            "queueStatus='completed' " .
    274308            "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
    275309
    … …  
    283317   }
    284318
    285    // Save the tarfile and expand it
     319   // Delete data from GFAC DB
     320   $query = "DELETE from analysis WHERE gfacID='$gfacID'";
     321
     322   $result = mysql_query( $query, $gfac_link );
     323
     324   if ( ! $result )
     325   {
     326      // Just log it and continue
     327      write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
     328   }
     329
     330   // Expand the tar file
    286331
    287332   if ( strlen( $tarfile ) == 0 )
    … …  
    292337   }
    293338
    294    // Shouldn't happen
    295    if ( ! is_dir( "$work" ) )
    296    {
    297       write_log( "$me: $work directory does not exist" );
    298       mail_to_user( "fail", "$work directory does not exist" );
    299       return( -1 );
    300    }
    301 
    302    if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
    303    chdir( "$work/$gfacID" );
    304 
    305    $f = fopen( "analysis.tar", "w" );
    306    fwrite( $f, $tarfile );
    307    fclose( $f );
    308 
    309339   $tar_out = array();
    310    exec( "tar -xf analysis.tar 2>&1", $tar_out, $err );
    311 
    312    if ( $err != 0 )
    313    {
    314       chdir( $work );
    315       exec( "rm -r $gfacID" );
    316       $output = implode( "\n", $tar_out );
    317 
    318       write_log( "$me: Bad output tarfile: $output" );
    319       mail_to_user( "fail", "Bad output file" );
    320       return( -1 );
    321    }
     340   exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
    322341
    323342   // Insert the model files and noise files
    … …  
    341360         $fn = $file;
    342361
     362      if ( preg_match( "/mdl.tmp$/", $fn ) )
     363         continue;
     364
    343365      if ( filesize( $fn ) < 100 )
    344366      {
    345          write_log( "$me:fn is invalid $fn" );
     367         write_log( "$me:fn is invalid $fn size filesize($fn)" );
    346368         mail_to_user( "fail", "Internal error\n$fn is invalid" );
    347369         return( -1 );
    … …  
    352374         $xml         = file_get_contents( $fn );
    353375         $statistics  = parse_xml( $xml, 'statistics' );
     376//         $ntries      = 0;
     377//
     378//         while ( $statistics['cpucount'] < 1  &&  $ntries < 3 )
     379//         {  // job_statistics file not totally copied, so retry
     380//            sleep( 10 );
     381//            $xml         = file_get_contents( $fn );
     382//            $statistics  = parse_xml( $xml, 'statistics' );
     383//            $ntries++;
     384//write_log( "$me:jobstats retry $ntries" );
     385//         }
     386//write_log( "$me:cputime=$statistics['cputime']" );
     387
    354388         $otherdata   = parse_xml( $xml, 'id' );
    355389
    … …  
    411445         
    412446      }
    413       else  // It's a model file
     447
     448      else if ( preg_match( "/\.mrecs/", $fn ) > 0 )  // It's an mrecs file
     449      {
     450         $xml         = file_get_contents( $fn );
     451         $mrecs_data  = parse_xml( $xml, "modelrecords" );
     452         $desc        = $mrecs_data[ 'description' ];
     453         $editGUID    = $mrecs_data[ 'editGUID' ];
     454write_log( "$me:   mrecs file editGUID=$editGUID" );
     455         if ( strlen( $editGUID ) < 36 )
     456            $editGUID    = "12345678-0123-5678-0123-567890123456";
     457         $mrecGUID    = $mrecs_data[ 'mrecGUID' ];
     458         $modelGUID   = $mrecs_data[ 'modelGUID' ];
     459
     460         $query = "INSERT INTO pcsa_modelrecs SET "  .
     461                  "editedDataID="                .
     462                  "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
     463                  "modelID=0, "             .
     464                  "mrecsGUID='$mrecGUID'," .
     465                  "description='$desc',"    .
     466                  "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
     467
     468         // Add later after all files are processed: editDataID, modelID
     469
     470         $result = mysql_query( $query, $us3_link );
     471
     472         if ( ! $result )
     473         {
     474            write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
     475            mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
     476            return( -1 );
     477         }
     478
     479         $id         = mysql_insert_id( $us3_link );
     480         $file_type  = "mrecs";
     481         $mrecsIDs[] = $id;
     482
     483         // Keep track of modelGUIDs for later, when we replace them
     484         $rmodlGUIDs[ $id ] = $modelGUID;
     485//write_log( "$me:   mrecs file inserted into DB : id=$id" );
     486      }
     487
     488      else                                           // It's a model file
    414489      {
    415490         $xml         = file_get_contents( $fn );
    … …  
    418493         $modelGUID   = $model_data[ 'modelGUID' ];
    419494         $editGUID    = $model_data[ 'editGUID' ];
     495
     496         if ( $mc_iteration > 1 )
     497         {
     498            $miter       = sprintf( "_mcN%03d", $mc_iteration );
     499            $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
     500write_log( "$me:   MODELUpd: O:description=$description" );
     501         }
    420502
    421503         $query = "INSERT INTO model SET "       .
    … …  
    499581   }
    500582   
    501    list( $requestGUID ) = mysql_fetch_array( $result );
    502    
    503    chdir( "$submit_dir/$requestGUID" );
    504    $f = fopen( "analysis.tar", "w" );
    505    fwrite( $f, $tarfile );
    506    fclose( $f );
     583//   list( $requestGUID ) = mysql_fetch_array( $result );
     584//   
     585//   chdir( "$submit_dir/$requestGUID" );
     586//   $f = fopen( "analysis-results.tar", "w" );
     587//   fwrite( $f, $tarfile );
     588//   fclose( $f );
    507589
    508590   // Clean up
    509    chdir ( $work );
     591//   chdir ( $work );
    510592   // exec( "rm -rf $gfacID" );
    511593
    … …  
    545627   // function get_gfac_message() also sets global $status
    546628   $gfac_message = get_gfac_message( $gfacID );
    547    if ( $gfac_message === false ) $gfac_message = "";
     629   if ( $gfac_message === false ) $gfac_message = "Job Finished";
    548630     
    549631   // Create a status to put in the subject line
    … …  
    561643      case "FAILED":
    562644         $subj_status = 'failed';
     645         if ( preg_match( "/^US3-A/i", $gfacID ) )
     646         {  // For A/Thrift FAIL, get error message
     647            $gfac_message = getExperimentErrors( $gfacID );
     648//$gfac_message .= "Test ERROR MESSAGE";
     649         }
    563650         break;
    564651
    … …  
    572659
    573660   }
     661
     662   $queuestatus = $subj_status;
     663   $limshost    = $dbhost;
     664   if ( $limshost == 'localhost' )
     665      $limshost    = gethostname();
    574666
    575667   // Parse the editXMLFilename
    … …  
    596688
    597689   Submission Time : $submittime
     690   LIMS Host       : $limshost
    598691   Analysis ID     : $gfacID
    599692   Request ID      : $requestID  ( $db )
Note: See TracChangeset for help on using the changeset viewer.