0 ) { while ( list( $message, $time ) = mysql_fetch_array( $result ) ) $message_log .= "$time $message\n"; } $query = "DELETE FROM queue_messages " . "WHERE analysisID = $analysisID "; $result = mysql_query( $query, $gfac_link ); if ( ! $result ) { // Just log it and continue write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); } // Save stdout, stderr, etc. for message log $query = "SELECT stdout, stderr, status, queue_msg FROM analysis " . "WHERE gfacID='$gfacID' "; $result = mysql_query( $query, $gfac_link ); try { // What if this is too large? list( $stdout, $stderr, $status, $queue_msg ) = mysql_fetch_array( $result ); } catch ( Exception $e ) { write_log( "$me: stdout + stderr larger than 128M - $gfacID\n" . mysql_error( $gfac_link ) ); // Just go ahead and clean up } // But let's allow for investigation of other large stdout and/or stderr if ( strlen( $stdout ) > 20480000 || strlen( $stderr ) > 20480000 ) write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" ); $message_log .= "\n\n\nStdout Contents\n\n" . $stdout . "\n\n\nStderr Contents\n\n" . $stderr . "\n\n\nGFAC Status: $status\n" . "GFAC message field: $queue_msg\n"; // Delete data from GFAC DB $query = "DELETE from analysis WHERE gfacID='$gfacID'"; $result = mysql_query( $query, $gfac_link ); if ( ! $result ) { // Just log it and continue write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); } // Copy queue messages to LIMS submit directory (files there are deleted after 7 days) global $submit_dir; // Get the request guid (LIMS submit dir name) $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " . "WHERE HPCAnalysisRequestID = $requestID "; $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); } list( $requestGUID ) = mysql_fetch_array( $result ); $output_dir = "$submit_dir/$requestGUID"; // Try to create it if necessary, and write the file // Let's use FILE_APPEND, in case this is the second time around and the // GFAC job status was INSERTed, rather than UPDATEd if ( ! is_dir( $output_dir ) ) mkdir( $output_dir, 0775, true ); $message_filename = "$output_dir/$db-$requestID-messages.txt"; file_put_contents( $message_filename, $message_log, FILE_APPEND ); // mysql_close( $gfac_link ); ///////// // Insert data into HPCAnalysis $query = "UPDATE HPCAnalysisResult SET " . "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " . "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "' " . "WHERE HPCAnalysisResultID=$HPCAnalysisResultID"; $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } // Save the tarfile and expand it if ( strlen( $tarfile ) == 0 ) { write_log( "$me: No tarfile" ); mail_to_user( "fail", "No results" ); return( -1 ); } // Shouldn't happen if ( ! is_dir( "$work" ) ) { write_log( "$me: $work directory does not exist" ); mail_to_user( "fail", "$work directory does not exist" ); return( -1 ); } if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 ); chdir( "$work/$gfacID" ); $f = fopen( "analysis.tar", "w" ); fwrite( $f, $tarfile ); fclose( $f ); $tar_out = array(); exec( "tar -xf analysis.tar 2>&1", $tar_out, $err ); if ( $err != 0 ) { chdir( $work ); exec( "rm -r $gfacID" ); $output = implode( "\n", $tar_out ); write_log( "$me: Bad output tarfile: $output" ); mail_to_user( "fail", "Bad output file" ); return( -1 ); } // Insert the model files and noise files $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES ); $noiseIDs = array(); $modelGUIDs = array(); foreach ( $files as $file ) { $split = explode( ";", $file ); if ( count( $split ) > 1 ) { list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file ); list( $other, $mc_iteration ) = explode( "=", $mc_iteration ); list( $other, $variance ) = explode( "=", $variance ); list( $other, $meniscus ) = explode( "=", $meniscus ); } else $fn = $file; if ( filesize( $fn ) < 100 ) { write_log( "$me:fn is invalid $fn" ); mail_to_user( "fail", "Internal error\n$fn is invalid" ); return( -1 ); } if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file { $xml = file_get_contents( $fn ); $statistics = parse_xml( $xml, 'statistics' ); $otherdata = parse_xml( $xml, 'id' ); $query = "UPDATE HPCAnalysisResult SET " . "wallTime = {$statistics['walltime']}, " . "CPUTime = {$statistics['cputime']}, " . "CPUCount = {$statistics['cpucount']}, " . "max_rss = {$statistics['maxmemory']}, " . "startTime = '{$otherdata['starttime']}', " . "endTime = '{$otherdata['endtime']}', " . "mgroupcount = {$otherdata['groupcount']} " . "WHERE HPCAnalysisResultID=$HPCAnalysisResultID"; $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); } file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir } else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file { $xml = file_get_contents( $fn ); $noise_data = parse_xml( $xml, "noise" ); $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise"; $desc = $noise_data[ 'description' ]; $modelGUID = $noise_data[ 'modelGUID' ]; $noiseGUID = $noise_data[ 'noiseGUID' ]; $query = "INSERT INTO noise SET " . "noiseGUID='$noiseGUID'," . "modelGUID='$modelGUID'," . "editedDataID=1, " . "modelID=1, " . "noiseType='$type'," . "description='$desc'," . "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'"; // Add later after all files are processed: editDataID, modelID $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } $id = mysql_insert_id( $us3_link ); $file_type = "noise"; $noiseIDs[] = $id; // Keep track of modelGUIDs for later, when we replace them $modelGUIDs[ $id ] = $modelGUID; } else // It's a model file { $xml = file_get_contents( $fn ); $model_data = parse_xml( $xml, "model" ); $description = $model_data[ 'description' ]; $modelGUID = $model_data[ 'modelGUID' ]; $editGUID = $model_data[ 'editGUID' ]; $query = "INSERT INTO model SET " . "modelGUID='$modelGUID'," . "editedDataID=" . "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . "description='$description'," . "MCIteration='$mc_iteration'," . "meniscus='$meniscus'," . "variance='$variance'," . "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'"; $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) ); mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } $modelID = mysql_insert_id( $us3_link ); $id = $modelID; $file_type = "model"; $query = "INSERT INTO modelPerson SET " . "modelID=$modelID, personID=$personID"; $result = mysql_query( $query, $us3_link ); } $query = "INSERT INTO HPCAnalysisResultData SET " . "HPCAnalysisResultID='$HPCAnalysisResultID', " . "HPCAnalysisResultType='$file_type', " . "resultID=$id"; $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } } // Now fix up noise entries // For noise files, there is, at most two: ti_noise and ri_noise // In this case there will only be one modelID foreach ( $noiseIDs as $noiseID ) { $modelGUID = $modelGUIDs[ $noiseID ]; $query = "UPDATE noise SET " . "editedDataID=" . "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . "modelID=" . "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" . "WHERE noiseID=$noiseID"; $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } } // Copy results to LIMS submit directory (files there are deleted after 7 days) global $submit_dir; // LIMS submit files dir // Get the request guid (LIMS submit dir name) $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " . "WHERE HPCAnalysisRequestID = $requestID "; $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); } list( $requestGUID ) = mysql_fetch_array( $result ); chdir( "$submit_dir/$requestGUID" ); $f = fopen( "analysis.tar", "w" ); fwrite( $f, $tarfile ); fclose( $f ); // Clean up chdir ( $work ); // exec( "rm -rf $gfacID" ); mysql_close( $us3_link ); ///////// // Send email mail_to_user( "success", "" ); } function mail_to_user( $type, $msg ) { // Note to me. Just changed subject line to include a modified $status instead // of the $type variable passed. More informative than just "fail" or "success." // See how it works for awhile and then consider removing $type parameter from // function. global $email_address; global $submittime; global $queuestatus; global $status; global $cluster; global $jobtype; global $org_name; global $admin_email; global $dbhost; global $requestID; global $gfacID; global $editXMLFilename; global $me; write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" ); // Get GFAC status and message // function get_gfac_message() also sets global $status $gfac_message = get_gfac_message( $gfacID ); if ( $gfac_message === false ) $gfac_message = ""; // Create a status to put in the subject line switch ( $status ) { case "COMPLETE": $subj_status = 'completed'; break; case "CANCELLED": case "CANCELED": $subj_status = 'canceled'; break; case "FAILED": $subj_status = 'failed'; break; case "ERROR": $subj_status = 'unknown error'; break; default: $subj_status = $status; // For now break; } // Parse the editXMLFilename list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) = explode( ".", $editXMLFilename ); $headers = "From: $org_name Admin<$admin_email>" . "\n"; $headers .= "Cc: $org_name Admin<$admin_email>" . "\n"; // Set the reply address $headers .= "Reply-To: $org_name<$admin_email>" . "\n"; $headers .= "Return-Path: $org_name<$admin_email>" . "\n"; // Try to avoid spam filters $now = time(); $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n"; $headers .= "X-Mailer: PHP v" . phpversion() . "\n"; $headers .= "MIME-Version: 1.0" . "\n"; $headers .= "Content-Transfer-Encoding: 8bit" . "\n"; $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 ); $message = " Your UltraScan job is complete: Submission Time : $submittime Analysis ID : $gfacID RunID : $runID EditID : $editID Data Type : $dataType Cell/Channel/Wl : $cell / $channel / $wl Status : $queuestatus Cluster : $cluster Job Type : $jobtype GFAC Status : $status GFAC Message : $gfac_message "; if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n"; // Handle the error case where an error occurs before fetching the // user's email address if ( $email_address == "" ) $email_address = $admin_email; mail( $email_address, $subject, $message, $headers ); } function parse_xml( $xml, $type ) { $parser = new XMLReader(); $parser->xml( $xml ); $results = array(); while ( $parser->read() ) { if ( $parser->name == $type ) { while ( $parser->moveToNextAttribute() ) { $results[ $parser->name ] = $parser->value; } break; } } $parser->close(); return $results; } // Function to get information about the current job GFAC function get_gfac_message( $gfacID ) { global $serviceURL; global $me; $hex = "[0-9a-fA-F]"; if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) && ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) ) { // Then it's not a GFAC job return false; } $url = "$serviceURL/jobstatus/$gfacID"; try { $post = new HttpRequest( $url, HttpRequest::METH_GET ); $http = $post->send(); $xml = $post->getResponseBody(); } catch ( HttpException $e ) { write_log( "$me: Job status not available - $gfacID" ); return false; } // Parse the result $gfac_message = parse_message( $xml ); return $gfac_message; } function parse_message( $xml ) { global $status; $status = ""; $gfac_message = ""; $parser = new XMLReader(); $parser->xml( $xml ); $results = array(); while( $parser->read() ) { $type = $parser->nodeType; if ( $type == XMLReader::ELEMENT ) $name = $parser->name; else if ( $type == XMLReader::TEXT ) { if ( $name == "status" ) $status = $parser->value; else $gfac_message = $parser->value; } } $parser->close(); return $gfac_message; } function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID ) { global $work; global $work_remote; global $me; global $db; global $status; // Figure out local working directory if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 ); $pwd = chdir( "$work/$gfacID" ); // Figure out remote directory $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID ); // Get stdout, stderr, output/analysis-results.tar $output = array(); // $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1"; // // exec( $cmd, $output, $stat ); // if ( $stat != 0 ) // write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); // // $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1"; // exec( $cmd, $output, $stat ); // if ( $stat != 0 ) // write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1"; exec( $cmd, $output, $stat ); if ( $stat != 0 ) write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1"; exec( $cmd, $output, $stat ); if ( $stat != 0 ) { write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); sleep( 10 ); write_log( "$me: RETRY" ); exec( $cmd, $output, $stat ); if ( $stat != 0 ) write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); } $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1"; exec( $cmd, $output, $stat ); if ( $stat != 0 ) { write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); sleep( 10 ); write_log( "$me: RETRY" ); exec( $cmd, $output, $stat ); if ( $stat != 0 ) write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); } // Write the files to gfacDB if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" ); if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" ); if ( file_exists( "analysis-results.tar" ) ) $tarfile = file_get_contents( "analysis-results.tar" ); $query = "UPDATE analysis SET " . "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," . "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," . "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'"; $result = mysql_query( $query, $gfac_link ); if ( ! $result ) { write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); echo "Bad query\n"; return( -1 ); } } ?>