Index: /trunk/cleanup.php =================================================================== --- /trunk/cleanup.php (revision 41) +++ /trunk/cleanup.php (revision 1) @@ -4,8 +4,9 @@ * * functions relating to copying results and cleaning up the gfac DB - * where the job used an Airavata interface. * */ +include_once "/home/us3/bin/listen-config.php"; +$me = 'cleanup.php'; $email_address = ''; $queuestatus = ''; @@ -13,9 +14,7 @@ $db = ''; $editXMLFilename = ''; -$status = ''; - -function aira_cleanup( $us3_db, $reqID, $gfac_link ) + +function gfac_cleanup( $us3_db, $requestID, $gfac_link ) { - global $org_domain; global $dbhost; global $user; @@ -32,23 +31,23 @@ global $editXMLFilename; global $submittime; - global $endtime; - global $status; - global $stderr; - global $stdout; - global $tarfile; - global $requestID; - global $submit_dir; - $me = 'cleanup_aira.php'; - - $requestID = $reqID; + $db = $us3_db; write_log( "$me: debug db=$db; requestID=$requestID" ); - $us3_link = mysqli_connect( $dbhost, $user, $passwd, $db ); + $us3_link = mysql_connect( $dbhost, $user, $passwd ); if ( ! $us3_link ) { - write_log( "$me: could not connect: $dbhost, $user, $passwd, $db" ); + write_log( "$me: could not connect: $dbhost, $user, $passwd" ); mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" ); + return( -1 ); + } + + $result = mysql_select_db( $db, $us3_link ); + + if ( ! $result ) + { + write_log( "$me: could not select DB $db" ); + mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" ); return( -1 ); } @@ -57,35 +56,41 @@ $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " . "WHERE HPCAnalysisRequestID=$requestID"; - $result = mysqli_query( $us3_link, $query ); + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query: $query" ); - mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - list( $email_address, $investigatorGUID, $editXMLFilename ) = mysqli_fetch_array( $result ); + mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) ); + return( -1 ); + } + + list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result ); $query = "SELECT personID FROM people " . "WHERE personGUID='$investigatorGUID'"; - $result = mysqli_query( $us3_link, $query ); - - list( $personID ) = mysqli_fetch_array( $result ); - - $query = "SELECT clusterName, submitTime, queueStatus, analType " . - "FROM HPCAnalysisRequest h, HPCAnalysisResult r " . - "WHERE h.HPCAnalysisRequestID=$requestID " . + $result = mysql_query( $query, $us3_link ); + + list( $personID ) = mysql_fetch_array( $result ); + + /* + $query = "SELECT clusterName, submitTime, queueStatus, method " . + "FROM HPCAnalysisRequest h LEFT JOIN HPCAnalysisResult " . + "ON h.HPCAnalysisRequestID=HPCAnalysisResult.HPCAnalysisRequestID " . + "WHERE h.HPCAnalysisRequestID=$requestID"; + */ + $query = "SELECT clusterName, submitTime, queueStatus, method " . + "FROM HPCAnalysisRequest h, HPCAnalysisResult r " . + "WHERE h.HPCAnalysisRequestID=$requestID " . "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID"; - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - if ( mysqli_num_rows( $result ) == 0 ) + $result = mysql_query( $query, $us3_link ); + + if ( ! $result ) + { + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); + return( -1 ); + } + + if ( mysql_num_rows( $result ) == 0 ) { write_log( "$me: US3 Table error - No records for requestID: $requestID" ); @@ -93,25 +98,26 @@ } - list( $cluster, $submittime, $queuestatus, $jobtype ) = mysqli_fetch_array( $result ); + list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result ); // Get the GFAC ID - $query = "SELECT HPCAnalysisResultID, gfacID, endTime FROM HPCAnalysisResult " . + $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " . "WHERE HPCAnalysisRequestID=$requestID"; - $result = mysqli_query( $us3_link, $query ); + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { write_log( "$me: Bad query: $query" ); - mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - list( $HPCAnalysisResultID, $gfacID, $endtime ) = mysqli_fetch_array( $result ); - - // Get data from global GFAC DB then insert it into US3 DB - -/* - $result = mysqli_select_db( $gfac_link, $gDB ); + mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) ); + return( -1 ); + } + + list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result ); + + //////// + // Get data from global GFAC DB and insert it into US3 DB + // $gfac_link = mysql_connect( $dbhost, $guser, $gpasswd ); + + $result = mysql_select_db( $gDB, $gfac_link ); if ( ! $result ) @@ -121,10 +127,9 @@ return( -1 ); } - */ $query = "SELECT status, cluster, id FROM analysis " . "WHERE gfacID='$gfacID'"; - $result = mysqli_query( $gfac_link, $query ); + $result = mysql_query( $query, $gfac_link ); if ( ! $result ) { @@ -134,10 +139,7 @@ } - list( $status, $cluster, $id ) = mysqli_fetch_array( $result ); - - $is_us3iab = preg_match( "/us3iab/", $cluster ); - $is_local = preg_match( "/-local/", $cluster ); - - if ( $is_us3iab || $is_local ) + list( $status, $cluster, $id ) = mysql_fetch_array( $result ); + + if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' ) { $clushost = $cluster; @@ -146,101 +148,17 @@ } - - $query = "SELECT id FROM analysis " . + $query = "SELECT id, stderr, stdout, tarfile FROM analysis " . "WHERE gfacID='$gfacID'"; - $result = mysqli_query( $gfac_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); - mail_to_user( "fail", "Internal error " . mysqli_error( $gfac_link ) ); - return( -1 ); - } - - list( $analysisID ) = mysqli_fetch_array( $result ); - - // Get the request guid (LIMS submit dir name) - $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " . - "WHERE HPCAnalysisRequestID = $requestID "; - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - } - - list( $requestGUID ) = mysqli_fetch_array( $result ); - $output_dir = "$submit_dir/$requestGUID"; - - // Get stderr,stdout,tarfile from work directory - if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 ); - chdir( "$output_dir" ); -//write_log( "$me: gfacID=$gfacID" ); -//write_log( "$me: submit_dir=$submit_dir" ); -//write_log( "$me: requestGUID=$requestGUID" ); -write_log( "$me: output_dir=$output_dir" ); - - $stderr = ""; - $stdout = ""; - $tarfile = ""; - $fn_stderr = "Ultrascan.stderr"; - $fn_stdout = "Ultrascan.stdout"; - $fn_tarfile = "analysis-results.tar"; - $secwait = 10; - $num_try = 0; -write_log( "$me: fn_tarfile=$fn_tarfile" ); - while ( ! file_exists( $fn_tarfile ) && $num_try < 3 ) - { - sleep( $secwait ); - $num_try++; - $secwait *= 2; -write_log( "$me: tar-exists: num_try=$num_try" ); - } - - $ofiles = scandir( $output_dir ); - foreach ( $ofiles as $ofile ) - { - if ( preg_match( "/^.*stderr$/", $ofile ) ) - $fn_stderr = $ofile; - if ( preg_match( "/^.*stdout$/", $ofile ) ) - $fn_stdout = $ofile; -//write_log( "$me: ofile=$ofile" ); - } -write_log( "$me: fn_stderr=$fn_stderr" ); -write_log( "$me: fn_stdout=$fn_stdout" ); -if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" ); -else write_log( "$me: NOT FOUND: $fn_tarfile" ); - - $stderr = ''; - $stdout = ''; - $tarfile = ''; - if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr ); - if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout ); - if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile ); -write_log( "$me(0): length contents stderr,stdout,tarfile -- " - . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) ); - // If stdout,stderr have no content, retry after delay - if ( strlen( $stdout ) == 0 || strlen( $stderr ) == 0 ) - { - sleep( 20 ); - if ( file_exists( $fn_stderr ) ) - $stderr = file_get_contents( $fn_stderr ); - if ( file_exists( $fn_stdout ) ) - $stdout = file_get_contents( $fn_stdout ); - } - -write_log( "$me: length contents stderr,stdout,tarfile -- " - . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) ); - - if ( $cluster == 'alamo' || $cluster == 'alamo-local' ) - { // Filter "ipath_userinit" lines out of alamo stdout lines - $prefln = strlen( $stdout ); - $output = array(); - exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err ); - $stdout = implode( "\n", $output ); - $posfln = strlen( $stdout ); -write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." ); - } + $result = mysql_query( $query, $gfac_link ); + + if ( ! $result ) + { + write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); + mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) ); + return( -1 ); + } + + list( $analysisID, $stderr, $stdout, $tarfile ) = mysql_fetch_array( $result ); // Save queue messages for post-mortem analysis @@ -248,10 +166,10 @@ "WHERE analysisID = $analysisID " . "ORDER BY time "; - $result = mysqli_query( $gfac_link, $query ); + $result = mysql_query( $query, $gfac_link ); if ( ! $result ) { // Just log it and continue - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); + write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); } @@ -262,7 +180,7 @@ "Processed: $now\n\n" . "Queue Messages\n\n" ; - if ( mysqli_num_rows( $result ) > 0 ) - { - while ( list( $message, $time ) = mysqli_fetch_array( $result ) ) + if ( mysql_num_rows( $result ) > 0 ) + { + while ( list( $message, $time ) = mysql_fetch_array( $result ) ) $message_log .= "$time $message\n"; } @@ -271,17 +189,26 @@ "WHERE analysisID = $analysisID "; - $result = mysqli_query( $gfac_link, $query ); + $result = mysql_query( $query, $gfac_link ); if ( ! $result ) { // Just log it and continue - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); - } - - $query = "SELECT queue_msg FROM analysis " . - "WHERE gfacID='$gfacID' "; - - $result = mysqli_query( $gfac_link, $query ); - list( $queue_msg ) = mysqli_fetch_array( $result ); + write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); + } + + // Save stdout, stderr, etc. for message log + $query = "SELECT stdout, stderr, status, queue_msg FROM analysis " . + "WHERE gfacID='$gfacID' "; + $result = mysql_query( $query, $gfac_link ); + try + { + // What if this is too large? + list( $stdout, $stderr, $status, $queue_msg ) = mysql_fetch_array( $result ); + } + catch ( Exception $e ) + { + write_log( "$me: stdout + stderr larger than 128M - $gfacID\n" . mysql_error( $gfac_link ) ); + // Just go ahead and clean up + } // But let's allow for investigation of other large stdout and/or stderr @@ -300,12 +227,27 @@ $query = "DELETE from analysis WHERE gfacID='$gfacID'"; - $result = mysqli_query( $gfac_link, $query ); + $result = mysql_query( $query, $gfac_link ); if ( ! $result ) { // Just log it and continue - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); - } - + write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); + } + + // Copy queue messages to LIMS submit directory (files there are deleted after 7 days) + global $submit_dir; + + // Get the request guid (LIMS submit dir name) + $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " . + "WHERE HPCAnalysisRequestID = $requestID "; + $result = mysql_query( $query, $us3_link ); + + if ( ! $result ) + { + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); + } + + list( $requestGUID ) = mysql_fetch_array( $result ); + $output_dir = "$submit_dir/$requestGUID"; // Try to create it if necessary, and write the file @@ -316,5 +258,5 @@ $message_filename = "$output_dir/$db-$requestID-messages.txt"; file_put_contents( $message_filename, $message_log, FILE_APPEND ); - // mysqli_close( $gfac_link ); + // mysql_close( $gfac_link ); ///////// @@ -322,30 +264,18 @@ $query = "UPDATE HPCAnalysisResult SET " . - "stderr='" . mysqli_real_escape_string( $us3_link, $stderr ) . "', " . - "stdout='" . mysqli_real_escape_string( $us3_link, $stdout ) . "', " . - "queueStatus='completed' " . + "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " . + "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "' " . "WHERE HPCAnalysisResultID=$HPCAnalysisResultID"; - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - // Delete data from GFAC DB - $query = "DELETE from analysis WHERE gfacID='$gfacID'"; - - $result = mysqli_query( $gfac_link, $query ); - - if ( ! $result ) - { - // Just log it and continue - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); - } - - // Expand the tar file + $result = mysql_query( $query, $us3_link ); + + if ( ! $result ) + { + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); + mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) ); + return( -1 ); + } + + // Save the tarfile and expand it if ( strlen( $tarfile ) == 0 ) @@ -356,6 +286,32 @@ } + // Shouldn't happen + if ( ! is_dir( "$work" ) ) + { + write_log( "$me: $work directory does not exist" ); + mail_to_user( "fail", "$work directory does not exist" ); + return( -1 ); + } + + if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 ); + chdir( "$work/$gfacID" ); + + $f = fopen( "analysis.tar", "w" ); + fwrite( $f, $tarfile ); + fclose( $f ); + $tar_out = array(); - exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err ); + exec( "tar -xf analysis.tar 2>&1", $tar_out, $err ); + + if ( $err != 0 ) + { + chdir( $work ); + exec( "rm -r $gfacID" ); + $output = implode( "\n", $tar_out ); + + write_log( "$me: Bad output tarfile: $output" ); + mail_to_user( "fail", "Bad output file" ); + return( -1 ); + } // Insert the model files and noise files @@ -363,6 +319,4 @@ $noiseIDs = array(); $modelGUIDs = array(); - $mrecsIDs = array(); - $fns_used = array(); foreach ( $files as $file ) @@ -381,15 +335,7 @@ $fn = $file; - if ( preg_match( "/mdl.tmp$/", $fn ) ) - continue; - - if ( in_array( $fn, $fns_used ) ) - continue; - - $fns_used[] = $fn; - if ( filesize( $fn ) < 100 ) { - write_log( "$me:fn is invalid $fn size filesize($fn)" ); + write_log( "$me:fn is invalid $fn" ); mail_to_user( "fail", "Internal error\n$fn is invalid" ); return( -1 ); @@ -400,16 +346,4 @@ $xml = file_get_contents( $fn ); $statistics = parse_xml( $xml, 'statistics' ); -// $ntries = 0; -// -// while ( $statistics['cpucount'] < 1 && $ntries < 3 ) -// { // job_statistics file not totally copied, so retry -// sleep( 10 ); -// $xml = file_get_contents( $fn ); -// $statistics = parse_xml( $xml, 'statistics' ); -// $ntries++; -//write_log( "$me:jobstats retry $ntries" ); -// } -//write_log( "$me:cputime=$statistics['cputime']" ); - $otherdata = parse_xml( $xml, 'id' ); @@ -423,15 +357,13 @@ "mgroupcount = {$otherdata['groupcount']} " . "WHERE HPCAnalysisResultID=$HPCAnalysisResultID"; - $result = mysqli_query( $us3_link, $query ); + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); } file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir - $file_type = "job_stats"; - $id = 1; } @@ -444,30 +376,26 @@ $modelGUID = $noise_data[ 'modelGUID' ]; $noiseGUID = $noise_data[ 'noiseGUID' ]; - $editGUID = '00000000-0000-0000-0000-000000000000'; - if ( isset( $model_data[ 'editGUID' ] ) ) - $editGUID = $model_data[ 'editGUID' ]; $query = "INSERT INTO noise SET " . "noiseGUID='$noiseGUID'," . "modelGUID='$modelGUID'," . - "editedDataID=" . - "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . + "editedDataID=1, " . "modelID=1, " . "noiseType='$type'," . "description='$desc'," . - "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'"; + "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'"; // Add later after all files are processed: editDataID, modelID - $result = mysqli_query( $us3_link, $query ); + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); + mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } - $id = mysqli_insert_id( $us3_link ); + $id = mysql_insert_id( $us3_link ); $file_type = "noise"; $noiseIDs[] = $id; @@ -477,46 +405,5 @@ } - - else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file - { - $xml = file_get_contents( $fn ); - $mrecs_data = parse_xml( $xml, "modelrecords" ); - $desc = $mrecs_data[ 'description' ]; - $editGUID = $mrecs_data[ 'editGUID' ]; -write_log( "$me: mrecs file editGUID=$editGUID" ); - if ( strlen( $editGUID ) < 36 ) - $editGUID = "12345678-0123-5678-0123-567890123456"; - $mrecGUID = $mrecs_data[ 'mrecGUID' ]; - $modelGUID = $mrecs_data[ 'modelGUID' ]; - - $query = "INSERT INTO pcsa_modelrecs SET " . - "editedDataID=" . - "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . - "modelID=0, " . - "mrecsGUID='$mrecGUID'," . - "description='$desc'," . - "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'"; - - // Add later after all files are processed: editDataID, modelID - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - $id = mysqli_insert_id( $us3_link ); - $file_type = "mrecs"; - $mrecsIDs[] = $id; - - // Keep track of modelGUIDs for later, when we replace them - $rmodlGUIDs[ $id ] = $modelGUID; -//write_log( "$me: mrecs file inserted into DB : id=$id" ); - } - - else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file + else // It's a model file { $xml = file_get_contents( $fn ); @@ -525,11 +412,4 @@ $modelGUID = $model_data[ 'modelGUID' ]; $editGUID = $model_data[ 'editGUID' ]; - - if ( $mc_iteration > 1 ) - { - $miter = sprintf( "_mcN%03d", $mc_iteration ); - $description = preg_replace( "/_mc[0-9]+/", $miter, $description ); -write_log( "$me: MODELUpd: O:description=$description" ); - } $query = "INSERT INTO model SET " . @@ -541,16 +421,16 @@ "meniscus='$meniscus'," . "variance='$variance'," . - "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'"; - - $result = mysqli_query( $us3_link, $query ); + "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'"; + + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { - write_log( "$me: Bad query:\n$query " . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); + write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) ); + mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } - $modelID = mysqli_insert_id( $us3_link ); + $modelID = mysql_insert_id( $us3_link ); $id = $modelID; $file_type = "model"; @@ -558,9 +438,6 @@ $query = "INSERT INTO modelPerson SET " . "modelID=$modelID, personID=$personID"; - $result = mysqli_query( $us3_link, $query ); - } - - else // Undetermined type: skip result data update - continue; + $result = mysql_query( $query, $us3_link ); + } $query = "INSERT INTO HPCAnalysisResultData SET " . @@ -569,10 +446,10 @@ "resultID=$id"; - $result = mysqli_query( $us3_link, $query ); + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); + mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } @@ -588,40 +465,18 @@ $query = "UPDATE noise SET " . "editedDataID=" . - "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," . + "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . "modelID=" . "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" . "WHERE noiseID=$noiseID"; - $result = mysqli_query( $us3_link, $query ); + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) ); + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); + mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) ); return( -1 ); } } - - // Now possibly fix up mrecs entries - - foreach ( $mrecsIDs as $mrecsID ) - { - $modelGUID = $rmodlGUIDs[ $mrecsID ]; - $query = "UPDATE pcsa_modelrecs SET " . - "modelID=" . - "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" . - "WHERE mrecsID=$mrecsID"; - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } -write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" ); - } -//write_log( "$me: mrecs entries updated" ); // Copy results to LIMS submit directory (files there are deleted after 7 days) @@ -631,23 +486,23 @@ $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " . "WHERE HPCAnalysisRequestID = $requestID "; - $result = mysqli_query( $us3_link, $query ); + $result = mysql_query( $query, $us3_link ); if ( ! $result ) { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); + write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) ); } -// list( $requestGUID ) = mysqli_fetch_array( $result ); -// -// chdir( "$submit_dir/$requestGUID" ); -// $f = fopen( "analysis-results.tar", "w" ); -// fwrite( $f, $tarfile ); -// fclose( $f ); + list( $requestGUID ) = mysql_fetch_array( $result ); + + chdir( "$submit_dir/$requestGUID" ); + $f = fopen( "analysis.tar", "w" ); + fwrite( $f, $tarfile ); + fclose( $f ); // Clean up -// chdir ( $work ); + chdir ( $work ); // exec( "rm -rf $gfacID" ); - mysqli_close( $us3_link ); + mysql_close( $us3_link ); ///////// @@ -665,5 +520,4 @@ global $email_address; global $submittime; - global $endtime; global $queuestatus; global $status; @@ -672,11 +526,8 @@ global $org_name; global $admin_email; - global $db; global $dbhost; global $requestID; global $gfacID; global $editXMLFilename; - global $stdout; - global $org_domain; global $me; @@ -684,7 +535,7 @@ // Get GFAC status and message - // function get_gfac_message() also sets global $status - $gfac_message = get_gfac_message( $gfacID ); - if ( $gfac_message === false ) $gfac_message = "Job Finished"; + // For right now, since we don't get clear and concise message + // $gfac_message = get_gfac_message( $gfacID ); + // if ( $gfac_message === false ) $gfac_message = ""; // Create a status to put in the subject line @@ -702,9 +553,4 @@ case "FAILED": $subj_status = 'failed'; - if ( preg_match( "/^US3-A/i", $gfacID ) ) - { // For A/Thrift FAIL, get error message - $gfac_message = getExperimentErrors( $gfacID ); -//$gfac_message .= "Test ERROR MESSAGE"; - } break; @@ -719,17 +565,4 @@ } - $queuestatus = $subj_status; - $limshost = $dbhost; - if ( $limshost == 'localhost' ) - { - $limshost = gethostname(); - if ( preg_match( "/scyld/", $limshost ) ) - $limshost = 'alamo.uthscsa.edu'; - else if ( preg_match( "/novalo/", $limshost ) ) - $limshost = 'uslims3.aucsolutions.com'; - else if ( ! preg_match( "/\./", $limshost ) ) - $limshost = $limshost . $org_domain; - } - // Parse the editXMLFilename list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) = @@ -738,6 +571,4 @@ $headers = "From: $org_name Admin<$admin_email>" . "\n"; $headers .= "Cc: $org_name Admin<$admin_email>" . "\n"; - $headers .= "CC: $org_name Admin" . "\n"; - $headers .= "CC: $org_name Admin" . "\n"; // Set the reply address @@ -746,6 +577,5 @@ // Try to avoid spam filters - $now = time(); - $tnow = date( 'Y-m-d H:i:s' ); + $now = time(); $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n"; $headers .= "X-Mailer: PHP v" . phpversion() . "\n"; @@ -758,9 +588,5 @@ Submission Time : $submittime - Job End Time : $endtime - Mail Time : $tnow - LIMS Host : $limshost Analysis ID : $gfacID - Request ID : $requestID ( $db ) RunID : $runID EditID : $editID @@ -771,9 +597,8 @@ Job Type : $jobtype GFAC Status : $status - GFAC Message : $gfac_message - Stdout : $stdout "; - if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n"; + // This would change to ( $status != 'COMPLETE' ) or something like that. + if ( $type != "success" ) $message .= "Error Message : $msg\n"; // Handle the error case where an error occurs before fetching the @@ -843,5 +668,4 @@ function parse_message( $xml ) { - global $status; $status = ""; $gfac_message = ""; @@ -878,181 +702,72 @@ global $me; global $db; - global $dbhost; global $status; - - $is_us3iab = preg_match( "/us3iab/", $cluster ); - $is_jetstr = preg_match( "/jetstream/", $cluster ); - $is_demeler3 = preg_match( "/demeler3/", $cluster ); - - $limshost = $dbhost; - $stderr = ''; - $stdout = ''; - $tarfile = ''; - - if ( $limshost == 'localhost' ) - { // If DB host is local host, get full LIMS host name - $limshost = gethostname(); - if ( preg_match( "/scyld/", $limshost ) ) - $limshost = 'alamo.uthscsa.edu'; - else if ( preg_match( "/novalo/", $limshost ) ) - $limshost = 'uslims3.aucsolutions.com'; - else if ( ! preg_match( "/\./", $limshost ) ) - $limshost = $limshost . $org_domain; - } - - if ( preg_match( "/alamo/", $limshost ) && - preg_match( "/alamo/", $cluster ) ) - { // If both LIMS and cluster are alamo, set up local transfers - $is_us3iab = 1; - if ( ! preg_match( "/\/local/", $work_remote ) ) - $work_remote = $work_remote . "/local"; - } - - // Figure out job's remote (or local) work directory + + // Figure out local working directory + if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 ); + $pwd = chdir( "$work/$gfacID" ); + + // Figure out remote directory $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID ); -//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" ); // Get stdout, stderr, output/analysis-results.tar $output = array(); - - if ( $is_us3iab == 0 ) - { - // For "-local", recompute remote work directory - $clushost = "$cluster.uthscsa.edu"; - $lworkdir = "~us3/lims/work/local"; - if ( $is_jetstr ) - { - $clushost = "js-169-137.jetstream-cloud.org"; - $lworkdir = "/N/us3_cluster/work/local"; - } - - if ( $is_demeler3 ) - { - $clushost = "demeler3.uleth.ca"; - $lworkdir = "/home/us3/work"; - } - - $cmd = "ssh us3@$clushost 'ls -d $lworkdir' 2>/dev/null"; +// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1"; +// +// exec( $cmd, $output, $stat ); +// if ( $stat != 0 ) +// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); +// +// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1"; +// exec( $cmd, $output, $stat ); +// if ( $stat != 0 ) +// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); + + $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1"; + exec( $cmd, $output, $stat ); + if ( $stat != 0 ) + write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); + + $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1"; + exec( $cmd, $output, $stat ); + if ( $stat != 0 ) + { + write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); + sleep( 10 ); + write_log( "$me: RETRY" ); exec( $cmd, $output, $stat ); - $work_remote = $output[ 0 ]; - $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID ); -write_log( "$me: -LOCAL: remoteDir=$remoteDir" ); - - // Figure out local working directory - if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 ); - $pwd = chdir( "$work/$gfacID" ); - - $cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1"; - + if ( $stat != 0 ) + write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); + } + + $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1"; + exec( $cmd, $output, $stat ); + if ( $stat != 0 ) + { + write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); + sleep( 10 ); + write_log( "$me: RETRY" ); exec( $cmd, $output, $stat ); - if ( $stat != 0 ) + if ( $stat != 0 ) write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); - - $cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1"; - - exec( $cmd, $output, $stat ); - if ( $stat != 0 ) - { - write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); - sleep( 10 ); - write_log( "$me: RETRY" ); - exec( $cmd, $output, $stat ); - if ( $stat != 0 ) - write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); - } - - $cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1"; - - exec( $cmd, $output, $stat ); - if ( $stat != 0 ) - { - write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); - sleep( 10 ); - write_log( "$me: RETRY" ); - exec( $cmd, $output, $stat ); - if ( $stat != 0 ) - write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) ); - } - } - else - { // Is US3IAB or alamo-to-alamo, so just change to local work directory - $pwd = chdir( "$remoteDir" ); -write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir"); - } - + } // Write the files to gfacDB - $secwait = 10; - $num_try = 0; - while ( ! file_exists( "stderr" ) && $num_try < 3 ) - { // Do waits and retries to let stderr appear - sleep( $secwait ); - $num_try++; - $secwait *= 2; -write_log( "$me: not-exist-stderr: num_try=$num_try" ); - } - - $lense = 0; - if ( file_exists( "stderr" ) ) - { - $lense = filesize( "stderr" ); - if ( $lense > 1000000 ) - { // Replace exceptionally large stderr with smaller version - exec( "mv stderr stderr-orig", $output, $stat ); - exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat ); - exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat ); - exec( "cat stderr-h stderr-t >stderr", $output, $stat ); - } - $stderr = file_get_contents( "stderr" ); - } - else - { - $stderr = ""; - } - + if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" ); if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" ); - - $fn1_tarfile = "analysis-results.tar"; - $fn2_tarfile = "output/" . $fn1_tarfile; - if ( file_exists( $fn1_tarfile ) ) - $tarfile = file_get_contents( $fn1_tarfile ); - else if ( file_exists( $fn2_tarfile ) ) - $tarfile = file_get_contents( $fn2_tarfile ); - -// $lense = strlen( $stderr ); -// if ( $lense > 1000000 ) -// { // Replace exceptionally large stderr with smaller version -// exec( "mv stderr stderr-orig", $output, $stat ); -// exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat ); -// exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat ); -// exec( "cat stderr-h stderr-t >stderr", $output, $stat ); -// $stderr = file_get_contents( "stderr" ); -// } -$lene = strlen( $stderr ); -write_log( "$me: stderr size: $lene (was $lense)"); -$leno = strlen( $stdout ); -write_log( "$me: stdout size: $leno"); -$lent = strlen( $tarfile ); -write_log( "$me: tarfile size: $lent"); - $esstde = mysqli_real_escape_string( $gfac_link, $stderr ); - $esstdo = mysqli_real_escape_string( $gfac_link, $stdout ); - $estarf = mysqli_real_escape_string( $gfac_link, $tarfile ); -$lene = strlen($esstde); -write_log( "$me: es-stderr size: $lene"); -$leno = strlen($esstdo); -write_log( "$me: es-stdout size: $leno"); -$lenf = strlen($estarf); -write_log( "$me: es-tarfile size: $lenf"); + if ( file_exists( "analysis-results.tar" ) ) + $tarfile = file_get_contents( "analysis-results.tar" ); + $query = "UPDATE analysis SET " . - "stderr='" . $esstde . "'," . - "stdout='" . $esstdo . "'," . - "tarfile='" . $estarf . "'"; - - $result = mysqli_query( $gfac_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); + "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," . + "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," . + "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'"; + + $result = mysql_query( $query, $gfac_link ); + + if ( ! $result ) + { + write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) ); echo "Bad query\n"; return( -1 ); Index: unk/cleanup_adev.php =================================================================== --- /trunk/cleanup_adev.php (revision 41) +++ (revision ) @@ -1,18 +1,0 @@ - Index: unk/cleanup_aira.php =================================================================== --- /trunk/cleanup_aira.php (revision 41) +++ (revision ) @@ -1,18 +1,0 @@ - Index: unk/cleanup_gfac.php =================================================================== --- /trunk/cleanup_gfac.php (revision 41) +++ (revision ) @@ -1,672 +1,0 @@ - 0 ) - { // Log success at fetch attempt - write_log( "$me: Successful data fetch: $requestID $gfacID" ); - } - else - { // Log failure at fetch attempt - write_log( "$me: Failed data fetch: $requestID $gfacID" ); - if ( $analysisID == '' ) - $analysisID = '0'; - } - - // Save queue messages for post-mortem analysis - $query = "SELECT message, time FROM queue_messages " . - "WHERE analysisID = $analysisID " . - "ORDER BY time "; - $result = mysqli_query( $gfac_link, $query ); - - if ( ! $result ) - { - // Just log it and continue - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); - } - - $now = date( 'Y-m-d H:i:s' ); - $message_log = "US3 DB: $db\n" . - "RequestID: $requestID\n" . - "GFAC ID: $gfacID\n" . - "Processed: $now\n\n" . - "Queue Messages\n\n" ; - - $need_finish = ( $status == 'COMPLETE' ); - - if ( mysqli_num_rows( $result ) > 0 ) - { - $time_msg = time(); - while ( list( $message, $time ) = mysqli_fetch_array( $result ) ) - { -//write_log( "$me: message=$message" ); - $message_log .= "$time $message\n"; - if ( preg_match( "/^Finished/i", $message ) ) - $need_finish = false; - $time_msg = strtotime( $time ); - } - - if ( $need_finish ) - { // No 'Finished' yet: forget if too much time has passed - $time_now = time(); - $tdelta = $time_now - $time_msg; -write_log( "$me: no-Finish time: tnow=$time_now, tmsg=$time_msg, tdelt=$tdelta" ); - if ( $tdelta > 600 ) - $need_finish = false; - } -//write_log( "$me: no-Finish time: tnow=$time_now, tmsg=$time_msg, tdelt=$tdelta" ); - } - else - { - write_log( "$me: No messages for analysisID=$analysisID ." ); - $need_finish = false; - } - - if ( $need_finish ) - { - write_log( "$me: Cleanup has not yet found 'Finished' for $gfacID" ); - return( 0 ); - } - - $query = "DELETE FROM queue_messages " . - "WHERE analysisID = $analysisID "; - - $result = mysqli_query( $gfac_link, $query ); - - if ( ! $result ) - { - // Just log it and continue - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); - } - - // Save stdout, stderr, etc. for message log - $query = "SELECT stdout, stderr, status, queue_msg FROM analysis " . - "WHERE gfacID='$gfacID' "; - $result = mysqli_query( $gfac_link, $query ); - try - { - // What if this is too large? - list( $stdout, $stderr, $status, $queue_msg ) = mysqli_fetch_array( $result ); - } - catch ( Exception $e ) - { - write_log( "$me: stdout + stderr larger than 128M - $gfacID\n" . mysqli_error( $gfac_link ) ); - // Just go ahead and clean up - } - - // But let's allow for investigation of other large stdout and/or stderr - if ( strlen( $stdout ) > 20480000 || - strlen( $stderr ) > 20480000 ) - write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" ); - - $message_log .= "\n\n\nStdout Contents\n\n" . - $stdout . - "\n\n\nStderr Contents\n\n" . - $stderr . - "\n\n\nGFAC Status: $status\n" . - "GFAC message field: $queue_msg\n"; - - // Delete data from GFAC DB - $query = "DELETE from analysis WHERE gfacID='$gfacID'"; - - $result = mysqli_query( $gfac_link, $query ); - - if ( ! $result ) - { - // Just log it and continue - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) ); - } -write_log( "$me: GFAC DB entry deleted" ); - - // Copy queue messages to LIMS submit directory (files there are deleted after 7 days) - global $submit_dir; - - // Get the request guid (LIMS submit dir name) - $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " . - "WHERE HPCAnalysisRequestID = $requestID "; - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - } - - list( $requestGUID ) = mysqli_fetch_array( $result ); - $output_dir = "$submit_dir/$requestGUID"; -write_log( "$me: Output dir determined: $output_dir" ); - - // Try to create it if necessary, and write the file - // Let's use FILE_APPEND, in case this is the second time around and the - // GFAC job status was INSERTed, rather than UPDATEd - if ( ! is_dir( $output_dir ) ) - mkdir( $output_dir, 0775, true ); - $message_filename = "$output_dir/$db-$requestID-messages.txt"; - file_put_contents( $message_filename, $message_log, FILE_APPEND ); - // mysqli_close( $gfac_link ); -write_log( "$me: *messages.txt written" ); - - ///////// - // Insert data into HPCAnalysis - - $query = "UPDATE HPCAnalysisResult SET " . - "stderr='" . mysqli_real_escape_string( $us3_link, $stderr ) . "', " . - "stdout='" . mysqli_real_escape_string( $us3_link, $stdout ) . "' " . - "WHERE HPCAnalysisResultID=$HPCAnalysisResultID"; - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - // Save the tarfile and expand it - - if ( strlen( $tarfile ) == 0 ) - { - write_log( "$me: No tarfile" ); - mail_to_user( "fail", "No results" ); - return( -1 ); - } - - // Shouldn't happen - if ( ! is_dir( "$work" ) ) - { - write_log( "$me: $work directory does not exist" ); - mail_to_user( "fail", "$work directory does not exist" ); - return( -1 ); - } - - if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 ); - chdir( "$work/$gfacID" ); - - $f = fopen( "analysis-results.tar", "w" ); - fwrite( $f, $tarfile ); - fclose( $f ); -//write_log( "$me: analysis-results.tar file written to work dir" ); - - $tar_out = array(); - exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err ); - - if ( $err != 0 ) - { - chdir( $work ); - exec( "rm -r $gfacID" ); - $output = implode( "\n", $tar_out ); - - write_log( "$me: Bad output tarfile: $output" ); - mail_to_user( "fail", "Bad output file" ); - return( -1 ); - } -//write_log( "$me: tar files extracted" ); - - // Insert the model files and noise files - $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES ); - $noiseIDs = array(); - $modelGUIDs = array(); - $mrecsIDs = array(); - $rmodlGUIDs = array(); - - foreach ( $files as $file ) - { - $split = explode( ";", $file ); - - if ( count( $split ) > 1 ) - { - list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file ); - - list( $other, $mc_iteration ) = explode( "=", $mc_iteration ); - list( $other, $variance ) = explode( "=", $variance ); - list( $other, $meniscus ) = explode( "=", $meniscus ); - } - else - $fn = $file; - - if ( preg_match( "/mdl.tmp$/", $fn ) ) - continue; - - if ( filesize( $fn ) < 100 ) - { - write_log( "$me:fn is invalid $fn" ); - mail_to_user( "fail", "Internal error\n$fn is invalid" ); - return( -1 ); - } -//write_log( "$me: handling file: $fn" ); - - if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file - { - $xml = file_get_contents( $fn ); - $statistics = parse_xml( $xml, 'statistics' ); - $otherdata = parse_xml( $xml, 'id' ); - - $query = "UPDATE HPCAnalysisResult SET " . - "wallTime = {$statistics['walltime']}, " . - "CPUTime = {$statistics['cputime']}, " . - "CPUCount = {$statistics['cpucount']}, " . - "max_rss = {$statistics['maxmemory']}, " . - "startTime = '{$otherdata['starttime']}', " . - "endTime = '{$otherdata['endtime']}', " . - "mgroupcount = {$otherdata['groupcount']} " . - "WHERE HPCAnalysisResultID=$HPCAnalysisResultID"; - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - } - - file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir - $file_type = "job_stats"; -//write_log( "$me: job_statistics file updated in Result and written" ); - - } - - else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file - { - $xml = file_get_contents( $fn ); - $noise_data = parse_xml( $xml, "noise" ); - $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise"; - $desc = $noise_data[ 'description' ]; - $modelGUID = $noise_data[ 'modelGUID' ]; - $noiseGUID = $noise_data[ 'noiseGUID' ]; - $editGUID = '00000000-0000-0000-0000-000000000000'; - if ( isset( $model_data[ 'editGUID' ] ) ) - $editGUID = $model_data[ 'editGUID' ]; - - $query = "INSERT INTO noise SET " . - "noiseGUID='$noiseGUID'," . - "modelGUID='$modelGUID'," . - "editedDataID=" . - "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . - "modelID=1, " . - "noiseType='$type'," . - "description='$desc'," . - "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'"; - - // Add later after all files are processed: editDataID, modelID - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - $id = mysqli_insert_id( $us3_link ); - $file_type = "noise"; - $noiseIDs[] = $id; - - // Keep track of modelGUIDs for later, when we replace them - $modelGUIDs[ $id ] = $modelGUID; -//write_log( "$me: noise file inserted into DB : id=$id modelGUID=$modelGUID" ); - - } - - else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file - { - $xml = file_get_contents( $fn ); - $mrecs_data = parse_xml( $xml, "modelrecords" ); - $desc = $mrecs_data[ 'description' ]; - $editGUID = $mrecs_data[ 'editGUID' ]; -write_log( "$me: mrecs file editGUID=$editGUID" ); - if ( strlen( $editGUID ) < 36 ) - $editGUID = "12345678-0123-5678-0123-567890123456"; - $mrecGUID = $mrecs_data[ 'mrecGUID' ]; - $modelGUID = $mrecs_data[ 'modelGUID' ]; - - $query = "INSERT INTO pcsa_modelrecs SET " . - "editedDataID=" . - "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . - "modelID=0, " . - "mrecsGUID='$mrecGUID'," . - "description='$desc'," . - "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'"; - - // Add later after all files are processed: editDataID, modelID - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - $id = mysqli_insert_id( $us3_link ); - $file_type = "mrecs"; - $mrecsIDs[] = $id; - - // Keep track of modelGUIDs for later, when we replace them - $rmodlGUIDs[ $id ] = $modelGUID; -//write_log( "$me: mrecs file inserted into DB : id=$id" ); - } - - else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file - { - $xml = file_get_contents( $fn ); - $model_data = parse_xml( $xml, "model" ); - $description = $model_data[ 'description' ]; - $modelGUID = $model_data[ 'modelGUID' ]; - $editGUID = $model_data[ 'editGUID' ]; - - if ( $mc_iteration > 1 ) - { -//write_log( "$me: MODELUpd: mc_iteration=$mc_iteration" ); - $miter = sprintf( "_mcN%03d", $mc_iteration ); -//write_log( "$me: MODELUpd: miter=$miter" ); -//write_log( "$me: MODELUpd: I:description=$description" ); - $description = preg_replace( "/_mc[0-9]+/", $miter, $description ); -write_log( "$me: MODELUpd: O:description=$description" ); - } - - $query = "INSERT INTO model SET " . - "modelGUID='$modelGUID'," . - "editedDataID=" . - "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," . - "description='$description'," . - "MCIteration='$mc_iteration'," . - "meniscus='$meniscus'," . - "variance='$variance'," . - "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'"; - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query " . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } - - $modelID = mysqli_insert_id( $us3_link ); - $id = $modelID; - $file_type = "model"; - - $query = "INSERT INTO modelPerson SET " . - "modelID=$modelID, personID=$personID"; - $result = mysqli_query( $us3_link, $query ); -//write_log( "$me: model file inserted into DB : id=$id" ); - } - - $query = "INSERT INTO HPCAnalysisResultData SET " . - "HPCAnalysisResultID='$HPCAnalysisResultID', " . - "HPCAnalysisResultType='$file_type', " . - "resultID=$id"; - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } -//write_log( "$me: ResultData updated : file_type=$file_type" ); - } - - // Now fix up noise entries - // For noise files, there is, at most two: ti_noise and ri_noise - // In this case there will only be one modelID - - foreach ( $noiseIDs as $noiseID ) - { - $modelGUID = $modelGUIDs[ $noiseID ]; - $query = "UPDATE noise SET " . - "editedDataID=" . - "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," . - "modelID=" . - "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" . - "WHERE noiseID=$noiseID"; - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } -//write_log( "$me: noise entry updated : noiseID=$noiseID" ); - } -//write_log( "$me: noise entries updated" ); - - // Now possibly fix up mrecs entries - - foreach ( $mrecsIDs as $mrecsID ) - { - $modelGUID = $rmodlGUIDs[ $mrecsID ]; - $query = "UPDATE pcsa_modelrecs SET " . - "modelID=" . - "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" . - "WHERE mrecsID=$mrecsID"; - - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) ); - return( -1 ); - } -//write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" ); - } -//write_log( "$me: mrecs entries updated" ); - - // Copy results to LIMS submit directory (files there are deleted after 7 days) - global $submit_dir; // LIMS submit files dir - - // Get the request guid (LIMS submit dir name) - $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " . - "WHERE HPCAnalysisRequestID = $requestID "; - $result = mysqli_query( $us3_link, $query ); - - if ( ! $result ) - { - write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) ); - } - - list( $requestGUID ) = mysqli_fetch_array( $result ); - - chdir( "$submit_dir/$requestGUID" ); - $f = fopen( "analysis.tar", "w" ); - fwrite( $f, $tarfile ); - fclose( $f ); - - // Clean up - chdir ( $work ); - // exec( "rm -rf $gfacID" ); - - mysqli_close( $us3_link ); - - ///////// - // Send email - - mail_to_user( "success", "" ); -} -?> Index: /trunk/cluster_status.php =================================================================== --- /trunk/cluster_status.php (revision 41) +++ /trunk/cluster_status.php (revision 1) @@ -1,27 +1,19 @@ &1|tail -1`; - - $sparts = preg_split( '/\s+/', $qstat ); - $que = $sparts[ 3 ]; - $run = $sparts[ 4 ]; - $sta = $sparts[ 10 ]; - if ( $sta == "Active" ) - $sta = "up"; - else - $sta = "down"; - break; - } - case 'alamo-local': - case 'alamo': - { - $host = "us3@alamo.uthscsa.edu"; - $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`; - $sparts = preg_split( '/\s+/', $qstat ); - $que = $sparts[ 3 ]; - $run = $sparts[ 4 ]; - $sta = $sparts[ 10 ]; - if ( $sta == "Active" ) - $sta = "up"; - else - $sta = "down"; - break; - } - - case 'demeler3-local': - { - $host = "us3@demeler3.uleth.ca"; - $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`; - $sparts = preg_split( '/\s+/', $qstat ); - $que = $sparts[ 3 ]; - $run = $sparts[ 4 ]; - $sta = $sparts[ 10 ]; - if ( $sta == "Active" ) - $sta = "up"; - else - $sta = "down"; - break; - } - - case 'jacinto': - { - $host = "us3@jacinto.uthscsa.edu"; - $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`; - $sparts = preg_split( '/\s+/', $qstat ); - $que = $sparts[ 3 ]; - $run = $sparts[ 4 ]; - $sta = $sparts[ 9 ]; - if ( $sta == "Active" ) - $sta = "up"; - else - $sta = "down"; - break; - } - case 'stampede2': - { - $host = "us3@stampede2.tacc.utexas.edu"; - $qstat = `ssh $host '~us3/scripts/clusstat skx-normal 2>/dev/null'`; - $sparts = preg_split( '/\s+/', $qstat ); - $tot = $sparts[ 2 ]; - $run = $sparts[ 5 ]; - $que = $sparts[ 8 ]; - $sta = "up"; - if ( $tot == '' || $tot == '0' ) - $sta = "down"; - break; - } - case 'lonestar5': - { - $host = "us3@ls5.tacc.utexas.edu"; - $qstat = `ssh $host '/opt/apps/tacc/bin/showq 2>/dev/null|grep "Total Jobs"'`; - $sparts = preg_split( '/\s+/', $qstat ); - $tot = $sparts[ 2 ]; - $run = '0'; - $que = '0'; - $sta = "up"; - if ( $tot == '' || $tot == '0' ) - { - $sta = "down"; - } - else - { - $run = $sparts[ 5 ]; - $que = $sparts[ 8 ]; -// $que = $sparts[ 11 ]; - } - break; - } - case 'comet': - { - $host = "us3@comet.sdsc.edu"; - //$qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" |tail -1'`; - $qstat = `ssh $host '/home/us3/scripts/cstat 2>&1'`; - $sparts = preg_split( '/\s+/', $qstat ); - $tot = $sparts[ 1 ]; - $run = '0'; - $que = '0'; - $sta = "up"; - if ( $tot == '' || $tot == '0' ) - { - $sta = "down"; - } - else - { - $run = $sparts[ 3 ]; - $que = $sparts[ 5 ]; - if ( $run == '0' && $que == '0' ) - { - $sta = "down"; - } - } - break; - } - case 'jureca': - { - $host = "swus1@jureca.fz-juelich.de"; - $qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`; - $sparts = preg_split( '/\s+/', $qstat ); - $sta = $sparts[ 0 ]; - $run = $sparts[ 1 ]; - $que = $sparts[ 2 ]; - break; - } - case 'jetstream-local': - case 'jetstream': - { - $host = "us3@js-169-137.jetstream-cloud.org"; -// $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" |tail -1'`; - $qstat = `ssh $host '/home/us3/bin/clusstat |tail -n 1'`; - $sparts = preg_split( '/\s+/', $qstat ); - $sta = $sparts[ 0 ]; - $knts = $sparts[ 1 ]; - $sparts = preg_split( '/\//', $knts ); - $run = $sparts[ 0 ]; - $que = $sparts[ 2 ]; - $tot = $sparts[ 3 ]; - if ( $sta == "" ) - $sta = "down"; - break; - } - case 'taito-local': - { - $host = "rb_2001068_taito01@taito.csc.fi"; - $qstat = `ssh -i /home/us3/.ssh/id_rsa_taito_robot $host '/homeappl/home/rb_2001068_taito01/scripts/cstat 2>&1'`; - $sparts = preg_split( '/\s+/', $qstat ); - $tot = $sparts[ 1 ]; - $run = '0'; - $que = '0'; - $sta = "up"; - if ( $tot == '' || $tot == '0' ) - { - $sta = "down"; - } - else - { - $run = $sparts[ 3 ]; - $que = $sparts[ 5 ]; - } - break; - } - - } - - if ( $sta == "" ) - $sta = "down"; - - if ( $sta == "down" ) - { - $que = "0"; - $run = "0"; - } - - // Insure queued,running counts are numeric - $que_s = $que; - $run_s = $run; - $que_e = preg_replace( '/=/', "", $que_s ); - $run_e = preg_replace( '/=/', "", $run_s ); - $que = intval( $que_e ); - $run = intval( $run_e ); -if($que!=$que_s || $run!=$run_s) - echo "$self: *** que s,e,i $que_s $que_e $que run s,e,i $run_s $run_e $run\n"; - - // Save cluster status values - $a[ 'cluster' ] = $clname; - $a[ 'queued' ] = $que; - $a[ 'running' ] = $run; - $a[ 'status' ] = $sta; -echo "$self: $clname $que $run $sta\n"; - - $data[] = $a; - - if ( $clname == 'alamo' || - $clname == 'jacinto' || - $clname == 'jetstream' ) - { - $a[ 'cluster' ] = $clname . "-local"; - $data[] = $a; - } + write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) ); + echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n"; } } @@ -437,5 +191,5 @@ if ( $name == "resourceHealth" ) { -## $name .= $this->_index; + $name .= $this->_index; $this->_index++; } Index: unk/cronic.txt =================================================================== --- /trunk/cronic.txt (revision 41) +++ (revision ) @@ -1,49 +1,0 @@ -/bin/cronic -#!/bin/bash - -# Cronic v2 - cron job report wrapper -# Copyright 2007 Chuck Houpt. No rights reserved, whatsoever. -# Public Domain CC0: http://creativecommons.org/publicdomain/zero/1.0/ - -set -eu - -OUT=/tmp/cronic.out.$$ -ERR=/tmp/cronic.err.$$ -TRACE=/tmp/cronic.trace.$$ - -set +e -"$@" >$OUT 2>$TRACE -RESULT=$? -set -e - -PATTERN="^${PS4:0:1}\\+${PS4:1}" -if grep -aq "$PATTERN" $TRACE -then - ! grep -av "$PATTERN" $TRACE > $ERR -else - ERR=$TRACE -fi - -if [ $RESULT -ne 0 -o -s "$ERR" ] - then - echo "Cronic detected failure or error output for the command:" - echo "$@" - echo - echo "RESULT CODE: $RESULT" - echo - echo "ERROR OUTPUT:" - cat "$ERR" - echo - echo "STANDARD OUTPUT:" - cat "$OUT" - if [ $TRACE != $ERR ] - then - echo - echo "TRACE-ERROR OUTPUT:" - cat "$TRACE" - fi -fi - -rm -f "$OUT" -rm -f "$ERR" -rm -f "$TRACE" Index: unk/crontab_root.txt =================================================================== --- /trunk/crontab_root.txt (revision 41) +++ (revision ) @@ -1,12 +1,0 @@ -#m h d m wd cmd line -5 3 * * * /usr/sbin/logrotate /etc/logrotate.conf - -# run nightly system backup script -# also cleans out old files in the common data directory -#!runas(root) -#!mailto(gegorbet) -0 22 * * * /root/backup-cauma-db.sh - -#ssl certificate renewal: -30 0,12 * * * /usr/bin/certbot renew --quiet - Index: unk/crontab_us3.txt =================================================================== --- /trunk/crontab_us3.txt (revision 41) +++ (revision ) @@ -1,11 +1,0 @@ -##MAILTO=gegorbet@gmail.com -#nolog(true) -*/1 * * * * cd /home/us3/lims/bin; cronic /usr/bin/php /home/us3/lims/bin/gridctl_pro.php -*/1 * * * * cd /home/us3/lims/bin; cronic /usr/bin/php /home/us3/lims/bin/gridctl_dev.php -*/12 * * * * cd /home/us3/lims/bin; cronic /usr/bin/php /home/us3/lims/bin/cluster_status.php -*/20 * * * * cd /home/us3/lims/bin; cronic /usr/bin/php /home/us3/lims/bin/update_notice.php - -# each morning: save data directory */job_statistics.xml files -MAILTO=gegorbet@gmail.com -15 2 * * * /home/us3/lims/bin/save-jobstats.sh - Index: /trunk/gridctl.php =================================================================== --- /trunk/gridctl.php (revision 41) +++ /trunk/gridctl.php (revision 1) @@ -1,10 +1,6 @@ &1|tail -n 1"; - else - $cmd = "/usr/bin/qstat -a $gfacID 2>&1|tail -n 1"; -//write_log( "$self cmd: $cmd" ); -//write_log( "$self cluster: $cluster" ); -//write_log( "$self gfacID: $gfacID" ); - - if ( ! preg_match( "/us3iab/", $cluster ) ) - { - $system = "$cluster.uthscsa.edu"; - if ( $is_jetstr ) - $system = "$cluster"; - $system = preg_replace( "/\-local/", "", $system ); - - if ( $is_demeler3 ) - { - $system = "demeler3.uleth.ca"; - } - - $cmd = "/usr/bin/ssh -x us3@$system " . $cmd; -//write_log( "$self cmd: $cmd" ); - } + + $system = "$cluster.uthscsa.edu"; + $system = preg_replace( "/\-local/", "", $system ); + $cmd = "/usr/bin/ssh -x us3@$system qstat -a $gfacID 2>&1"; $result = exec( $cmd ); -//write_log( "$self result: $result" ); - - $secwait = 2; - $num_try = 0; - // Sleep and retry up to 3 times if ssh has "ssh_exchange_identification" error - while ( preg_match( "/ssh_exchange_id/", $result ) && $num_try < 3 ) - { - sleep( $secwait ); - $num_try++; - $secwait *= 2; -write_log( "$me: num_try=$num_try secwait=$secwait" ); - } - - if ( preg_match( "/^qstat: Unknown/", $result ) || - preg_match( "/ssh_exchange_id/", $result ) ) - { - write_log( "$self get_local_status: Local job $gfacID unknown result=$result" ); + + if ( $result == "" || preg_match( "/^qstat: Unknown/", $result ) ) + { + write_log( "$self get_local_status: Local job $gfacID unknown" ); return 'UNKNOWN'; } $values = preg_split( "/\s+/", $result ); - $jstat = ( $is_jetstr == 0 ) ? $values[ 9 ] : $values[ 5 ]; -//write_log( "$self: get_local_status: job status = /$jstat/"); - switch ( $jstat ) +// write_log( "$self: get_local_status: job status = /{$values[9]}/"); + switch ( $values[ 9 ] ) { case "W" : // Waiting for execution time to be reached case "E" : // Job is exiting after having run case "R" : // Still running - case "CG" : // Job is completing $status = 'ACTIVE'; break; case "C" : // Job has completed - case "ST" : // Job has disappeared - case "CD" : // Job has completed $status = 'COMPLETED'; break; @@ -911,19 +643,5 @@ case "H" : // Held case "Q" : // Queued - case "PD" : // Queued - case "CF" : // Queued $status = 'SUBMITTED'; - break; - - case "CA" : // Job has been canceled - $status = 'CANCELED'; - break; - - case "F" : // Job has failed - case "BF" : // Job has failed - case "NF" : // Job has failed - case "TO" : // Job has timed out - case "" : // Job has disappeared - $status = 'FAILED'; break; @@ -945,20 +663,20 @@ $query = "SELECT id FROM analysis " . "WHERE gfacID = '$gfacID' "; - $result = mysqli_query( $gLink, $query ); - if ( ! $result ) - { - write_log( "$self: Query failed $query - " . mysqli_error( $gLink ) ); - return; - } - list( $analysisID ) = mysqli_fetch_array( $result ); + $result = mysql_query( $query, $gLink ); + if ( ! $result ) + { + write_log( "$self: Query failed $query - " . mysql_error( $gLink ) ); + return; + } + list( $analysisID ) = mysql_fetch_array( $result ); // Insert message into queue_message table $query = "INSERT INTO queue_messages SET " . - "message = '" . mysqli_real_escape_string( $gLink, $message ) . "', " . - "analysisID = '$analysisID' "; - $result = mysqli_query( $gLink, $query ); - if ( ! $result ) - { - write_log( "$self: Query failed $query - " . mysqli_error( $gLink ) ); + "message = '" . mysql_real_escape_string( $message, $gLink ) . "'," . + "analysisID = $analysisID "; + $result = mysql_query( $query, $gLink ); + if ( ! $result ) + { + write_log( "$self: Query failed $query - " . mysql_error( $gLink ) ); return; } @@ -974,19 +692,29 @@ global $us3_db; - $us3_link = mysqli_connect( $dbhost, $user, $passwd, $us3_db ); + $us3_link = mysql_connect( $dbhost, $user, $passwd ); if ( ! $us3_link ) { write_log( "$self: could not connect: $dbhost, $user, $passwd" ); - mail_to_admin( "fail", "Could not connect to $dbhost : $us3_db" ); + mail_to_admin( "fail", "Could not connect to $dbhost" ); return 0; } + + $result = mysql_select_db( $us3_db, $us3_link ); + + if ( ! $result ) + { + write_log( "$self: could not select DB $us3_db" ); + mail_to_admin( "fail", "Could not select DB $us3_db, $dbhost, $user, $passwd" ); + return 0; + } + $query = "UPDATE HPCAnalysisResult SET " . - "lastMessage='" . mysqli_real_escape_string( $us3_link, $message ) . "'" . + "lastMessage='" . mysql_real_escape_string( $message, $us3_link ) . "'" . "WHERE gfacID = '$gfacID' "; - mysqli_query( $us3_link, $query ); - mysqli_close( $us3_link ); + mysql_query( $query, $us3_link ); + mysql_close( $us3_link ); } @@ -1003,7 +731,4 @@ $headers = "From: $org_name Admin<$admin_email>" . "\n"; $headers .= "Cc: $org_name Admin<$admin_email>" . "\n"; - $headers .= "Cc: $org_name Admin" . "\n"; - $headers .= "Bcc: Gary Gorbet" . "\n"; // make sure - // Set the reply address @@ -1013,5 +738,4 @@ // Try to avoid spam filters $now = time(); - $tnow = date( 'Y-m-d H:i:s' ); $headers .= "Message-ID: <" . $now . "gridctl@$dbhost>$requestID\n"; $headers .= "X-Mailer: PHP v" . phpversion() . "\n"; @@ -1023,5 +747,5 @@ UltraScan job error notification from gridctl.php: - Update Time : $updateTime [ now=$tnow ] + Update Time : $updateTime GFAC Status : $status Cluster : $cluster @@ -1032,156 +756,3 @@ mail( $admin_email, $subject, $message, $headers ); } - -// Convert a status string to one of the standard DB status strings -function standard_status( $status_in ) -{ - switch ( $status_in ) - { // Map variations to standard gateway status values - case 'QUEUED' : - case 'LAUNCHED' : - case 'CREATED' : - case 'VALIDATED' : - case 'SCHEDULED' : - case 'submitted' : - case 'SUBMITTED' : - case '' : - $status = 'SUBMITTED'; - break; - - case 'EXECUTING' : - case 'ACTIVE' : - case 'running' : - case 'executing' : - $status = 'RUNNING'; - break; - - case 'PENDING' : - case 'CANCELING' : - $status = 'UPDATING'; - break; - - case 'CANCELLED' : - case 'canceled' : - $status = 'CANCELED'; - break; - -// $status = 'DATA'; -// break; - - case 'COMPLETED' : - case 'completed' : - $status = 'COMPLETE'; - break; - - case 'FAILED_DATA' : - case 'SUBMIT_TIMEOUT' : - case 'RUN_TIMEOUT' : - case 'DATA_TIMEOUT' : - $status = 'FAILED'; - break; - - case 'COMPLETE' : - $status = 'DONE'; - break; - - case 'UNKNOWN' : - $status = 'ERROR'; - break; - - // Where already standard value, retain value - case 'ERROR' : - case 'RUNNING' : - case 'SUBMITTED' : - case 'UPDATING' : - case 'CANCELED' : - case 'DATA' : - case 'FAILED' : - case 'DONE' : - case 'FINISHED' : - default : - $status = $status_in; - break; - } - - return $status; -} - -function aira_status( $gfacID, $status_in ) -{ - global $self; - global $loghdr; - global $class_dir; -//echo "a_st: st_in$status_in : $gfacID\n"; - //$status_gw = standard_status( $status_in ); - $status_gw = $status_in; -//echo "a_st: st_db=$status_gw\n"; - $status = $status_gw; - $me_devel = preg_match( "/class_devel/", $class_dir ); - $job_devel = preg_match( "/US3-ADEV/i", $gfacID ); - $devmatch = ( ( !$me_devel && !$job_devel ) || - ( $me_devel && $job_devel ) ); - - if ( preg_match( "/US3-A/i", $gfacID ) && $devmatch ) - { -//write_log( "$loghdr status_in=$status_in status=$status gfacID=$gfacID" ); - $status_ex = getExperimentStatus( $gfacID ); -//write_log( "$loghdr status_ex=$status_ex" ); - - if ( $status_ex == 'COMPLETED' ) - { // Experiment is COMPLETED: check for 'FINISHED' or 'DONE' - if ( $status_gw == 'FINISHED' || $status_gw == 'DONE' ) - { // COMPLETED + FINISHED/DONE : gateway status is now COMPLETE - $status = 'COMPLETE'; - } - - else - { // COMPLETED + NOT-FINISHED/DONE: gw status now DONE - $status = 'DONE'; - } - } - - else if ( $status_gw == 'FINISHED' || $status_gw == 'DONE' ) - { // Gfac status == FINISHED/DONE: leave as is (unless FAILED) - $status = $status_gw; - if ( $status_ex == 'FAILED' ) - { - sleep( 10 ); - $status_ex = getExperimentStatus( $gfacID ); - if ( $status_ex == 'FAILED' ) - { - write_log( "$loghdr status still 'FAILED' after 10-second delay" ); - sleep( 10 ); - $status_ex = getExperimentStatus( $gfacID ); - if ( $status_ex == 'FAILED' ) - write_log( "$loghdr status still 'FAILED' after 20-second delay" ); - else - write_log( "$loghdr status is $status_ex after 20-second delayed retry" ); - } - write_log( "$loghdr status reset to 'COMPLETE'" ); - $status = 'COMPLETE'; - } - } - - else if ( $status_ex == 'EXECUTING' ) - { - $status = standard_status( $status_gw ); -write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" ); - } - - else - { // Experiment not COMPLETED/FINISHED/DONE: use experiment status - $status = standard_status( $status_ex ); - } - -//if ( $status != 'SUBMITTED' ) -//write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" ); - if ( $status != $status_gw ) - { - update_job_status( $status, $gfacID ); - } - } - - return $status; -} - ?> Index: unk/gridctl_dev.php =================================================================== --- /trunk/gridctl_dev.php (revision 41) +++ (revision ) @@ -1,12 +1,0 @@ - - Index: unk/gridctl_pro.php =================================================================== --- /trunk/gridctl_pro.php (revision 41) +++ (revision ) @@ -1,12 +1,0 @@ - - Index: /trunk/listen-config.php.template =================================================================== --- /trunk/listen-config.php.template (revision 41) +++ /trunk/listen-config.php.template (revision 1) @@ -1,5 +1,5 @@ >$us3etc/manage.log 2>&1 Index: unk/runID_info.php =================================================================== --- /trunk/runID_info.php (revision 41) +++ (revision ) @@ -1,803 +1,0 @@ - - -
- -

Info by Run ID

- - - -
- - 2 ) ? "" : "AND people.personID = $myID "; - - $query = "SELECT experimentID, runID, lname " . - "FROM experiment, projectPerson, people " . - "WHERE experiment.projectID = projectPerson.projectID " . - "AND projectPerson.personID = people.personID " . - $users_clause . - "ORDER BY lname, runID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) == 0 ) return ""; - - $text = "
\n" . - " \n" . - "
\n"; - - return $text; -} - -// A function to retrieve information about that runID -function runID_info( $experimentID ) -{ - global $link; - $query = "SELECT people.personID, personGUID, lname, fname, email " . - "FROM experiment, projectPerson, people " . - "WHERE experiment.experimentID = $experimentID " . - "AND experiment.projectID = projectPerson.projectID " . - "AND projectPerson.personID = people.personID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - list( $ID, $GUID, $lname, $fname, $email ) = mysqli_fetch_array( $result ); - - $text = << - Investigator Information - ID: - $ID - - GUID: - $GUID - - Name: - $fname $lname - - Email: - $email - - -HTML; - - $query = "SELECT experimentGUID, coeff1, coeff2, type, runType " . - "FROM experiment, rotorCalibration " . - "WHERE experimentID = $experimentID " . - "AND experiment.rotorCalibrationID = rotorCalibration.rotorCalibrationID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - list( $GUID, $coeff1, $coeff2, $type, $runType ) = mysqli_fetch_array( $result ); - $text .= << - Run Information - GUID: - $GUID - - Rotor stretch coeff 1: - $coeff1 - - Rotor stretch coeff 2: - $coeff2 - - Experiment type: - $type - - Run Type: - $runType - - -HTML; - - $query = "SELECT rawDataID, rawDataGUID, filename, solutionID " . - "FROM rawData " . - "WHERE experimentID = $experimentID " . - "ORDER BY filename "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) == 0 ) - return $text; - - $rawIDs = array(); - $solutionIDs = array(); - $text .= << - Raw Data - - ID - GUID - Filename - Solution - - - - -HTML; - - while ( list( $ID, $GUID, $filename, $solutionID ) = mysqli_fetch_array( $result ) ) - { - $rawIDs[] = $ID; - $solutionIDs[] = $solutionID; - - $text .= <<$ID - $GUID - $filename - $solutionID - - -HTML; - - } - - $text .= "\n\n" . - "\n"; - - $rawIDs_csv = implode( ", ", $rawIDs ); - $query = "SELECT editedDataID, rawDataID, editGUID, filename " . - "FROM editedData " . - "WHERE rawDataID IN ( $rawIDs_csv ) " . - "ORDER BY editedDataID, filename "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) == 0 ) - return $text; - - $text .= << - Edit Profiles - - ID - GUID - Filename - Raw ID - - - - - -HTML; - - $editIDs = array(); - while ( list ( $editID, $rawID, $GUID, $filename ) = mysqli_fetch_array( $result ) ) - { - $editIDs[] = $editID; - - $text .= <<$editID - $GUID - $filename - $rawID - - -HTML; - } - - $text .= "\n\n" . - "\n"; - - $editIDs_csv = implode( ", ", $editIDs ); - $query = "SELECT model.modelID, editedDataID, modelGUID, variance, meniscus, personID " . - "FROM model LEFT JOIN modelPerson " . - "ON ( model.modelID = modelPerson.modelID ) " . - "WHERE editedDataID IN ( $editIDs_csv ) " . - "ORDER BY modelID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) != 0 ) - { - $text .= << - Models - - ID - GUID - Edit ID - Variance - Meniscus - Owner ID - - - - - -HTML; - - $modelIDs = array(); - while ( list ( $modelID, $editID, $GUID, $variance, $meniscus, $personID ) = mysqli_fetch_array( $result ) ) - { - $modelIDs[] = $modelID; - - $text .= <<$modelID - $GUID - $editID - $variance - $meniscus - $personID - - -HTML; - } - - $text .= "\n\n" . - "\n"; - } - - if ( count( $modelIDs ) > 0 ) - { - $modelIDs_csv = implode( ", ", $modelIDs ); - $query = "SELECT noiseID, noiseGUID, editedDataID, modelID, modelGUID, noiseType " . - "FROM noise " . - "WHERE modelID IN ( $modelIDs_csv ) " . - "ORDER BY noiseID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) != 0 ) - { - $text .= << - Noise Linked to Models - - ID - GUID - Edit ID - Model ID - Model GUID - Type - - - - - -HTML; - - while ( list ( $noiseID, $GUID, $editID, $modelID, $modelGUID, $type ) = mysqli_fetch_array( $result ) ) - { - $text .= <<$noiseID - $GUID - $editID - $modelID - $modelGUID - $type - - -HTML; - } - - $text .= "\n\n" . - "\n"; - } - } - - $query = "SELECT noiseID, noiseGUID, editedDataID, modelID, modelGUID, noiseType " . - "FROM noise " . - "WHERE editedDataID IN ( $editIDs_csv ) " . - "ORDER BY noiseID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) != 0 ) - { - $text .= << - Noise Linked to Edit Profiles - - ID - GUID - Edit ID - Model ID - Model GUID - Type - - - - - -HTML; - - while ( list ( $noiseID, $GUID, $editID, $modelID, $modelGUID, $type ) = mysqli_fetch_array( $result ) ) - { - $text .= <<$noiseID - $GUID - $editID - $modelID - $modelGUID - $type - - -HTML; - } - - $text .= "\n\n" . - "\n"; - } - - $reportIDs = array(); - $query = "SELECT reportID, reportGUID, title " . - "FROM report " . - "WHERE experimentID = $experimentID " . - "ORDER BY reportID "; - - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) != 0 ) - { - $text .= << - Reports Related to This Experiment - - ID - GUID - Title - - - - - -HTML; - - while ( list ( $reportID, $GUID, $title ) = mysqli_fetch_array( $result ) ) - { - $reportIDs[] = $reportID; - $text .= <<$reportID - $GUID - $title - - -HTML; - } - - $text .= "\n\n" . - "\n"; - } - - $reportTripleIDs = array(); - if ( ! empty( $reportIDs ) ) - { - $reportIDs_csv = implode( ",", $reportIDs ); - $query = "SELECT reportTripleID, reportTripleGUID, resultID, triple, dataDescription, reportID " . - "FROM reportTriple " . - "WHERE reportID IN ( $reportIDs_csv ) " . - "ORDER BY reportID, reportTripleID "; - - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) != 0 ) - { - $text .= << - Report Triples Related to Reports - - ID - GUID - Result ID - Triple - Description - Report ID - - - - - -HTML; - - while ( list ( $reportTripleID, $GUID, $resultID, $triple, $dataDesc, $rptID ) - = mysqli_fetch_array( $result ) ) - { - $reportTripleIDs[] = $reportTripleID; - $text .= <<$reportTripleID - $GUID - $resultID - $triple - $dataDesc - $rptID - - -HTML; - } - - $text .= "\n\n" . - "\n"; - } - } - - if ( ! empty( $reportTripleIDs ) ) - { - $reportTripleIDs_csv = implode( ",", $reportTripleIDs ); - $query = "SELECT d.reportDocumentID, reportDocumentGUID, editedDataID, label, " . - "filename, analysis, subAnalysis, documentType, l.reportTripleID " . - "FROM documentLink l, reportDocument d " . - "WHERE reportTripleID IN ( $reportTripleIDs_csv ) " . - "AND l.reportDocumentID = d.reportDocumentID " . - "ORDER BY reportTripleID, reportDocumentID "; - - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) != 0 ) - { - $text .= << - Report Documents Related to Triples - - ID - GUID - Edit ID - Label/Filename - Anal/Sub/DocType - Trip ID - - - - - -HTML; - - while ( list ( $reportDocumentID, $GUID, $editID, $label, $filename, - $analysis, $subAnal, $docType, $tripID ) - = mysqli_fetch_array( $result ) ) - { - $text .= <<$reportDocumentID - $GUID - $editID - $label/$filename - $analysis/$subAnal/$docType - $tripID - - -HTML; - } - - $text .= "\n\n" . - "\n"; - } - } - - $query = "SELECT HPCAnalysisRequestID, HPCAnalysisRequestGUID, editXMLFilename, " . - "submitTime, clusterName, method " . - "FROM HPCAnalysisRequest " . - "WHERE experimentID = $experimentID " . - "ORDER BY HPCAnalysisRequestID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) == 0 ) - return $text; - - $requestIDs = array(); - $text .= << - HPC Requests - - ID - GUID - XML Filename - Submit - Cluster - Method - - - - -HTML; - - while ( list( $ID, $GUID, $filename, $submit, $cluster, $method ) = mysqli_fetch_array( $result ) ) - { - $requestIDs[] = $ID; - - $text .= <<$ID - $GUID - $filename - $submit - $cluster - $method - - -HTML; - - } - - $text .= "\n\n" . - "\n"; - - $requestIDs_csv = implode( ", ", $requestIDs ); - $query = "SELECT HPCAnalysisResultID, HPCAnalysisRequestID, gfacID, queueStatus, updateTime " . - "FROM HPCAnalysisResult " . - "WHERE HPCAnalysisRequestID IN ( $requestIDs_csv ) " . - "ORDER BY HPCAnalysisResultID "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) != 0 ) - { - $text .= << - HPC Results - - ID - Request ID - gfac ID - Status - Updated - - - - -HTML; - - $incomplete = array(); - while ( list( $ID, $requestID, $gfacID, $status, $updated ) = mysqli_fetch_array( $result ) ) - { - if ( $status != 'completed' ) - $incomplete[] = $gfacID; - - $text .= <<$ID - $requestID - $gfacID - $status - $updated - - -HTML; - - } - - $text .= "\n\n" . - "\n"; - - } - - if ( empty( $incomplete ) ) - return $text; - - // Now switch over to the global db - global $globaldbhost, $globaldbuser, $globaldbpasswd, $globaldbname; - - $globaldb = mysqli_connect( $globaldbhost, $globaldbuser, $globaldbpasswd, $globaldbname ); - - if ( ! $globaldb ) - { - $text .= "

Cannot open global database on $globaldbhost : $globaldbname

\n"; - return $text; - } - - $text .= << - GFAC Status - - gfacID - Cluster - DB - Status - Message - Updated - - - - -HTML; - - $in_queue = 0; - foreach ( $incomplete as $gfacID ) - { - - $query = "SELECT cluster, us3_db, status, queue_msg, time " . - "FROM analysis " . - "WHERE gfacID = '$gfacID' "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link) ); - - if ( mysqli_num_rows( $result ) == 1 ) - { - $in_queue++; - list( $cluster, $db, $status, $msg, $time ) = mysqli_fetch_array( $result ); - $text .= <<$gfacID - $cluster - $db - $status - $msg - $time - - -HTML; - } - } - - if ( $in_queue == 0 ) - $text .= "No local jobs currently in the queue\n"; - - $text .= "\n\n" . - "\n"; - - mysqli_close( $globaldb ); - - return $text; -} - -function HPCDetail( $requestID ) -{ - global $link; - $query = "SELECT * FROM HPCAnalysisRequest WHERE HPCAnalysisRequestID=$requestID"; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link)); - $row = mysqli_fetch_assoc( $result ); - $row['requestXMLFile'] = '
' . htmlentities( $row['requestXMLFile'] ) . '
'; - - // Save for later - $requestGUID = $row['HPCAnalysisRequestGUID']; - $cluster = $row['clusterName']; - #var_dump($cluster); - $text = << - HPC Request Detail - -HTML; - - foreach ($row as $key => $value) - { - $text .= " $key$value\n"; - } - - $text .= "\n"; - - $query = "SELECT * FROM HPCAnalysisResult WHERE HPCAnalysisRequestID=$requestID"; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link)); - $row = mysqli_fetch_assoc( $result ); - $row['jobfile'] = '
' . htmlentities( $row['jobfile'] ) . '
'; - - // Get GFAC job status - global $uses_airavata; - - if ( $uses_airavata === true ) - { - $row['gfacStatus'] = nl2br( getExperimentStatus( $row['gfacID'] ) ); - } - else - { - $row['gfacStatus'] = nl2br( getJobstatus( $row['gfacID'] ) ); - } - - // Get queue messages from disk directory, if it still exists - global $submit_dir; - global $dbname; - - $msg_filename = "$submit_dir$requestGUID/$dbname-$requestID-messages.txt"; - $queue_msgs = false; - if ( file_exists( $msg_filename ) ) - { - $queue_msgs = file_get_contents( $msg_filename ); - $len_msgs = strlen( $queue_msgs ); - $queue_msgs = '
' . $queue_msgs . '
'; - } - - // Get resulting model and noise information - if ( ! empty( $resultID ) ) - { - $resultID = $row['HPCAnalysisResultID']; - $models = array(); - $noise = array(); - $query = "SELECT resultID FROM HPCAnalysisResultData " . - "WHERE HPCAnalysisResultID = $resultID " . - "AND HPCAnalysisResultType = 'model' "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link)); - $models = mysqli_fetch_row( $result ); // An array with all of them - if ( $models !== false ) - $row['modelIDs'] = implode( ", ", $models ); - - $query = "SELECT resultID FROM HPCAnalysisResultData " . - "WHERE HPCAnalysisResultID = $resultID " . - "AND HPCAnalysisResultType = 'noise' "; - $result = mysqli_query( $link, $query ) - or die( "Query failed : $query
\n" . mysqli_error($link)); - $noise = mysqli_fetch_row( $result ); // An array with all of them - if ( $noise !== false ) - $row['noiseIDs'] = implode( ", ", $noise ); - } - - $text .= << - - - -HTML; - - foreach ($row as $key => $value) - { - $text .= " \n"; - } - - if ( $queue_msgs !== false ) - { - $linkmsg = "Length Messages"; - - $text .= " \n"; - if ( isset( $_GET[ 'msgs' ] ) ) - $text .= " \n"; - } - - $text .= "
HPC Result Detail
$key$value
$linkmsg$len_msgs
Queue Messages$queue_msgs
\n"; - - return $text; -} - -?> Index: unk/save-jobstats.sh =================================================================== --- /trunk/save-jobstats.sh (revision 41) +++ (revision ) @@ -1,70 +1,0 @@ -#!/bin/sh -# save-jobstats.sh -- save uslims3_data job_statistics.xml files - -BASEDIR=/srv/www/htdocs/uslims3/uslims3_data -SAVEDIR=/srv/www/htdocs/uslims3/job_stats -STATFILE=job_statistics.xml -if [ ! -d ${SAVEDIR} ]; then - mkdir ${SAVEDIR} -fi -HOSTNAME=`uname -n` -if [ `echo ${HOSTNAME}|grep -c "novalo"` -ne 0 ]; then - HOSTNAME="uslims3.aucsolutions.com" -fi -echo "HOSTNAME=${HOSTNAME}" - -##JSTIME=`stat ${SAVEDIR} -c %Y` -JSTIME=`date +%s` -##echo "JSTIME=${JSTIME}" -NCFILE=0 -NMFILE=0 -ONEDAY=86400 # 24-hours worth of seconds - -# Get uslims3_data subdirectories -cd ${BASEDIR} -DATADIRS=`ls -rtF|tail -500|grep '/'` -##DATADIRS=`ls -rtF|tail -200|grep '/'` - -for D in $DATADIRS ; do - # Change to job subdirectory - JDTIME=`stat ${D} -c %Y` - AGE=`expr ${JSTIME} - ${JDTIME}` -##echo "D=${D}" -##echo " JDTIME=${JDTIME}" -##echo " AGE=${AGE}" - - # Only process directories less than a day old - if [ ${AGE} -lt ${ONEDAY} ]; then - JDDATE=`stat ${D} -c %y | cut -d: -f1-2 | sed -e 's/ /-/' -e 's/://g'` - cd ${BASEDIR}/${D} -##echo " +++Less than 1 day old+++" - - if [ -f ${STATFILE} ]; then - # Build job stats if job_statistics.xml exists - PART1A=`grep cluster= ${STATFILE}|sed -e 's/^.*cluster=\"//'|cut -d'"' -f1` - PART1A=`echo ${PART1A} | sed -e 's@-@_@'` - PART1B=`echo ${D} | cut -d'-' -f1` - PART1=${PART1A}-${PART1B} - PART2=`grep submittime= ${STATFILE}|sed -e 's/^.*submittime=\"//'|cut -d' ' -f1` - PART2=`echo ${PART2}|sed -e 's/\-//g'` - SOUTFILE=${SAVEDIR}/${PART1}-${PART2}-${STATFILE} -##echo "we can build: ${SOUTFILE}" - if [ ! -f ${SOUTFILE} ]; then - echo "cp -p ${STATFILE} ${SOUTFILE}" - cp -p ${STATFILE} ${SOUTFILE} - NCFILE=`expr ${NCFILE} + 1` - fi - else - CLUST=`grep shortname ${BASEDIR}/${D}hpcr*xml|cut -d'"' -f4` - echo "job_statistics.xml in ${D} [ ${JDDATE} ${CLUST} ] does not exist!" - NMFILE=`expr ${NMFILE} + 1` - fi - fi - - # Change back to ./uslims3_data - cd ${BASEDIR} -done - -echo "job_stats+++ Files Created: ${NCFILE}" -echo "jobxml/stats Files Missing: ${NMFILE} (aborted,running jobs?)" - Index: unk/update_notice.php =================================================================== --- /trunk/update_notice.php (revision 41) +++ (revision ) @@ -1,192 +1,0 @@ -xml( $xml ); - -$notices = array(); -$keys = array(); - -echo "=====START of PARSE LOOP===== \n"; -while( $parser->read() ) -{ - $n_type = $parser->nodeType; -//echo "n_type=$n_type \n"; - - if ( $n_type == XMLReader::ELEMENT ) - { - $name = $parser->name; -//echo "name=$name \n"; - - if ( $name == "notice" ) - { - $type = $parser->getAttribute( "type" ); -//echo " type=$type \n"; - $parser->moveToAttribute( "revision" ); - $rev = $parser->value; -//echo " rev=$rev \n"; - if ( $rev == "latest" ) - $rev = $sysrev; -//echo " rev=$rev \n"; - - $key = $type . $rev; - $notices[ $key ] = array(); -//echo " key=$key \n"; - } - } - - else if ( $n_type == XMLReader::TEXT ) - { - $msg = $parser->readString(); -//echo " msg=+++$msg+++ \n"; - $len = strlen( $msg ); - - if ( $len > 0 ) - { // Only add if not an empty message - $msg = preg_replace( "/\@revision/", $rev, $msg ); - $msg = preg_replace( "/'/", "\\'", $msg ); - $notices[ $key ][ 'type' ] = $type; - $notices[ $key ][ 'rev' ] = $rev; - $notices[ $key ][ 'msg' ] = $msg; - $notices[ $key ][ 'act' ] = 'add'; - $notices[ $key ][ 'id' ] = '0'; - - $keys[] = $key; - } - } -} - -$parser->close(); -echo "=====END of PARSE LOOP===== \n"; - -// Get data from notice DB. Update the action field to -// reflect which action is required on each entry: -// "add" - is only present in file (add to DB); -// "del" - is only present in DB (delete from DB); -// "upd" - present in both, but messages differ (update in DB); -// "none" - present in both and messages identical (no DB update). - -$noteLink = mysqli_connect( $dbhost, $dbuser, $dbpassw, $notice_db ); - -if ( ! $noteLink ) -{ - echo "Could not connect to DB $notice_db - " . mysqli_error($noteLink) . "\n"; - exit(); -} - -$query = "SELECT id, type, revision, message FROM notice"; - -$result = mysqli_query( $noteLink, $query ) - or die( "Query failed : $query
" . mysqli_error($noteLink) ); - -echo "=====START of DB QUERY LOOP===== \n"; -$num_rows = mysqli_num_rows( $result ); - -echo " numrows = $num_rows \n"; -while ( list( $id, $type, $rev, $msg ) = mysqli_fetch_array( $result ) ) -{ - $key = $type . $rev; - - if ( in_array( $key, $keys ) ) - { // Entry is in both file and DB - $msgf = $notices[ $key ][ 'msg' ]; - $notices[ $key ][ 'id' ] = $id; - $notices[ $key ][ 'type' ] = $type; - $notices[ $key ][ 'rev' ] = $rev; - - if ( strcmp( $msg, $msgf ) == 0 ) - { // Messages match, so no update is needed - $notices[ $key ][ 'act' ] = 'none'; - } - else - { // Messages differ, so DB entry must be updated - $notices[ $key ][ 'act' ] = 'upd'; - $notices[ $key ][ 'msg' ] = $msgf; -echo " msg = ++$msg++\n"; -echo " msgf = ++$msgf++\n"; - } - } - - else - { // Entry is only in DB, so a delete is needed - $notices[ $key ] = array(); - $notices[ $key ][ 'id' ] = $id; - $notices[ $key ][ 'type' ] = $type; - $notices[ $key ][ 'rev' ] = $rev; - $notices[ $key ][ 'act' ] = 'del'; - $notices[ $key ][ 'msg' ] = $msg; - $keys[] = $key; - } -} -echo "=====END of DB QUERY LOOP===== \n"; - -echo "=====START of DB Update LOOP===== \n"; -foreach ( $keys as $key ) -{ - $type = $notices[ $key ][ 'type' ]; - $rev = $notices[ $key ][ 'rev' ]; - $act = $notices[ $key ][ 'act' ]; - $id = $notices[ $key ][ 'id' ]; - $msg = $notices[ $key ][ 'msg' ]; - $len = strlen( $msg ); - echo "-- key=$key --\n"; - echo " type: $type \n"; - echo " rev: $rev \n"; - echo " act: $act \n"; - echo " id: $id \n"; -// echo " msg: $msg \n"; - echo " msg: ( $len characters ) \n"; - - if ( $act == "add" ) - { // Must add the entry to the database - $query = "INSERT INTO notice " . - "SET type='$type', revision='$rev', message='$msg'"; - } - else if ( $act == "del" ) - { // Must delete the entry from the database - $query = "DELETE FROM notice WHERE id='$id'"; - } - else if ( $act == "upd" ) - { // Must update an existing entry in the database - $query = "UPDATE notice " . - "SET message='$msg' " . - "WHERE id='$id'"; - } - echo " query: [ $query ] \n"; - - $result = mysqli_query( $noteLink, $query ) - or die( "Query failed : $query
" . mysqli_error($noteLink) ); - -} -echo "=====END of DB Update LOOP===== \n"; - -?> Index: unk/us3-listen =================================================================== --- /trunk/us3-listen (revision 41) +++ (revision ) @@ -1,47 +1,0 @@ -#!/bin/sh -######################################################################## -# Begin $rc_base/init.d/ -# -# Description : gridctl us3-listen daemons (listen, manage-us3-pipe) -# -# Authors : Gary Gorbet -# -# Version : 01.01 -# -# Notes : Goes in /etc/rc.d/init.d -# -######################################################################## - -. /etc/init.d/functions - -case "${1}" in - start) - echo -n "Starting us3-listen daemons ..." - /usr/bin/php ~us3/lims/bin/listen.php & - echo "" - sleep 1 - ${0} status - ;; - - stop) - echo -n "Stopping us3-listen daemons ..." - killproc /usr/bin/php - ;; - - restart) - ${0} stop - sleep 1 - ${0} start - ;; - - status) - ps -aef|grep -v grep|egrep 'listen.php|us3-pipe' - ;; - - *) - echo "Usage: ${0} {start|stop|restart|status}" - exit 1 - ;; -esac - -# End $rc_base/init.d/ Index: unk/us3-notice.php =================================================================== --- /trunk/us3-notice.php (revision 41) +++ (revision ) @@ -1,4 +1,0 @@ -$us3bin = exec( "ls -d ~us3/lims/bin" ); -include_once "$us3bin/listen-config.php"; -include "$us3bin/cleanup_aira.php"; -include "$us3bin/cleanup_gfac.php";