source: trunk/cleanup.php@ 11

Last change on this file since 11 was 10, checked in by gegorbet, 9 years ago

fixes to allow simultaneous production/development jobs
File size: 27.3 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $dbhost;
20 global $user;
21 global $passwd;
22 global $db;
23 global $guser;
24 global $gpasswd;
25 global $gDB;
26 global $me;
27 global $work;
28 global $email_address;
29 global $queuestatus;
30 global $jobtype;
31 global $editXMLFilename;
32 global $submittime;
33 global $status;
34 global $stderr;
35 global $stdout;
36 global $tarfile;
37 global $requestID;
38 global $submit_dir;
39 $me = 'cleanup_aira.php';
40
41 $requestID = $reqID;
42 $db = $us3_db;
43 write_log( "$me: debug db=$db; requestID=$requestID" );
44
45 $us3_link = mysql_connect( $dbhost, $user, $passwd );
46
47 if ( ! $us3_link )
48 {
49 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
50 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
51 return( -1 );
52 }
53
54 $result = mysql_select_db( $db, $us3_link );
55
56 if ( ! $result )
57 {
58 write_log( "$me: could not select DB $db" );
59 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
60 return( -1 );
61 }
62
63 // First get basic info for email messages
64 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
65 "WHERE HPCAnalysisRequestID=$requestID";
66 $result = mysql_query( $query, $us3_link );
67
68 if ( ! $result )
69 {
70 write_log( "$me: Bad query: $query" );
71 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
72 return( -1 );
73 }
74
75 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
76
77 $query = "SELECT personID FROM people " .
78 "WHERE personGUID='$investigatorGUID'";
79 $result = mysql_query( $query, $us3_link );
80
81 list( $personID ) = mysql_fetch_array( $result );
82
83 $query = "SELECT clusterName, submitTime, queueStatus, method " .
84 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
85 "WHERE h.HPCAnalysisRequestID=$requestID " .
86 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
87
88 $result = mysql_query( $query, $us3_link );
89
90 if ( ! $result )
91 {
92 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
93 return( -1 );
94 }
95
96 if ( mysql_num_rows( $result ) == 0 )
97 {
98 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
99 return( -1 );
100 }
101
102 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
103
104 // Get the GFAC ID
105 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
106 "WHERE HPCAnalysisRequestID=$requestID";
107
108 $result = mysql_query( $query, $us3_link );
109
110 if ( ! $result )
111 {
112 write_log( "$me: Bad query: $query" );
113 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
114 return( -1 );
115 }
116
117 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
118
119 // Get data from global GFAC DB then insert it into US3 DB
120
121 $result = mysql_select_db( $gDB, $gfac_link );
122
123 if ( ! $result )
124 {
125 write_log( "$me: Could not connect to DB $gDB" );
126 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
127 return( -1 );
128 }
129
130 $query = "SELECT status, cluster, id FROM analysis " .
131 "WHERE gfacID='$gfacID'";
132
133 $result = mysql_query( $query, $gfac_link );
134 if ( ! $result )
135 {
136 write_log( "$me: Could not select GFAC status for $gfacID" );
137 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
138 return( -1 );
139 }
140
141 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
142
143 if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' )
144 {
145 $clushost = $cluster;
146 $clushost = preg_replace( "/\-local/", "", $clushost );
147 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
148 }
149
150
151 $query = "SELECT id FROM analysis " .
152 "WHERE gfacID='$gfacID'";
153
154 $result = mysql_query( $query, $gfac_link );
155
156 if ( ! $result )
157 {
158 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
159 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
160 return( -1 );
161 }
162
163 list( $analysisID ) = mysql_fetch_array( $result );
164
165 // Get the request guid (LIMS submit dir name)
166 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
167 "WHERE HPCAnalysisRequestID = $requestID ";
168 $result = mysql_query( $query, $us3_link );
169
170 if ( ! $result )
171 {
172 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
173 }
174
175 list( $requestGUID ) = mysql_fetch_array( $result );
176 $output_dir = "$submit_dir/$requestGUID";
177
178 // Get stderr,stdout,tarfile from work directory
179 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
180 chdir( "$output_dir" );
181//write_log( "$me: gfacID=$gfacID" );
182//write_log( "$me: submit_dir=$submit_dir" );
183//write_log( "$me: requestGUID=$requestGUID" );
184write_log( "$me: output_dir=$output_dir" );
185
186 $stderr = "";
187 $stdout = "";
188 $tarfile = "";
189 $fn_stderr = "Ultrascan.stderr";
190 $fn_stdout = "Ultrascan.stdout";
191 $fn_tarfile = "analysis-results.tar";
192 $num_try = 0;
193 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
194 {
195 sleep( 10 );
196 $num_try++;
197 }
198
199 $ofiles = scandir( $output_dir );
200 foreach ( $ofiles as $ofile )
201 {
202 if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
203 $fn_stderr = $ofile;
204 if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
205 $fn_stdout = $ofile;
206//write_log( "$me: ofile=$ofile" );
207 }
208write_log( "$me: fn_stderr=$fn_stderr" );
209write_log( "$me: fn_stdout=$fn_stdout" );
210if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
211else write_log( "$me: NOT FOUND: $fn_tarfile" );
212
213 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
214 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
215 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
216
217 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
218 { // Filter "ipath_userinit" lines out of alamo stdout lines
219 $prefln = strlen( $stdout );
220 $output = array();
221 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
222 $stdout = implode( "\n", $output );
223 $posfln = strlen( $stdout );
224write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
225 }
226
227 // Save queue messages for post-mortem analysis
228 $query = "SELECT message, time FROM queue_messages " .
229 "WHERE analysisID = $analysisID " .
230 "ORDER BY time ";
231 $result = mysql_query( $query, $gfac_link );
232
233 if ( ! $result )
234 {
235 // Just log it and continue
236 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
237 }
238
239 $now = date( 'Y-m-d H:i:s' );
240 $message_log = "US3 DB: $db\n" .
241 "RequestID: $requestID\n" .
242 "GFAC ID: $gfacID\n" .
243 "Processed: $now\n\n" .
244 "Queue Messages\n\n" ;
245 if ( mysql_num_rows( $result ) > 0 )
246 {
247 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
248 $message_log .= "$time $message\n";
249 }
250
251 $query = "DELETE FROM queue_messages " .
252 "WHERE analysisID = $analysisID ";
253
254 $result = mysql_query( $query, $gfac_link );
255
256 if ( ! $result )
257 {
258 // Just log it and continue
259 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
260 }
261
262 $query = "SELECT queue_msg FROM analysis " .
263 "WHERE gfacID='$gfacID' ";
264
265 $result = mysql_query( $query, $gfac_link );
266 list( $queue_msg ) = mysql_fetch_array( $result );
267
268 // But let's allow for investigation of other large stdout and/or stderr
269 if ( strlen( $stdout ) > 20480000 ||
270 strlen( $stderr ) > 20480000 )
271 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
272
273 $message_log .= "\n\n\nStdout Contents\n\n" .
274 $stdout .
275 "\n\n\nStderr Contents\n\n" .
276 $stderr .
277 "\n\n\nGFAC Status: $status\n" .
278 "GFAC message field: $queue_msg\n";
279
280 // Delete data from GFAC DB
281 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
282
283 $result = mysql_query( $query, $gfac_link );
284
285 if ( ! $result )
286 {
287 // Just log it and continue
288 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
289 }
290
291
292 // Try to create it if necessary, and write the file
293 // Let's use FILE_APPEND, in case this is the second time around and the
294 // GFAC job status was INSERTed, rather than UPDATEd
295 if ( ! is_dir( $output_dir ) )
296 mkdir( $output_dir, 0775, true );
297 $message_filename = "$output_dir/$db-$requestID-messages.txt";
298 file_put_contents( $message_filename, $message_log, FILE_APPEND );
299 // mysql_close( $gfac_link );
300
301 /////////
302 // Insert data into HPCAnalysis
303
304 $query = "UPDATE HPCAnalysisResult SET " .
305 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
306 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
307 "queueStatus='completed' " .
308 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
309
310 $result = mysql_query( $query, $us3_link );
311
312 if ( ! $result )
313 {
314 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
315 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
316 return( -1 );
317 }
318
319 // Delete data from GFAC DB
320 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
321
322 $result = mysql_query( $query, $gfac_link );
323
324 if ( ! $result )
325 {
326 // Just log it and continue
327 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
328 }
329
330 // Expand the tar file
331
332 if ( strlen( $tarfile ) == 0 )
333 {
334 write_log( "$me: No tarfile" );
335 mail_to_user( "fail", "No results" );
336 return( -1 );
337 }
338
339 $tar_out = array();
340 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
341
342 // Insert the model files and noise files
343 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
344 $noiseIDs = array();
345 $modelGUIDs = array();
346
347 foreach ( $files as $file )
348 {
349 $split = explode( ";", $file );
350
351 if ( count( $split ) > 1 )
352 {
353 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
354
355 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
356 list( $other, $variance ) = explode( "=", $variance );
357 list( $other, $meniscus ) = explode( "=", $meniscus );
358 }
359 else
360 $fn = $file;
361
362 if ( preg_match( "/mdl.tmp$/", $fn ) )
363 continue;
364
365 if ( filesize( $fn ) < 100 )
366 {
367 write_log( "$me:fn is invalid $fn size filesize($fn)" );
368 mail_to_user( "fail", "Internal error\n$fn is invalid" );
369 return( -1 );
370 }
371
372 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
373 {
374 $xml = file_get_contents( $fn );
375 $statistics = parse_xml( $xml, 'statistics' );
376// $ntries = 0;
377//
378// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
379// { // job_statistics file not totally copied, so retry
380// sleep( 10 );
381// $xml = file_get_contents( $fn );
382// $statistics = parse_xml( $xml, 'statistics' );
383// $ntries++;
384//write_log( "$me:jobstats retry $ntries" );
385// }
386//write_log( "$me:cputime=$statistics['cputime']" );
387
388 $otherdata = parse_xml( $xml, 'id' );
389
390 $query = "UPDATE HPCAnalysisResult SET " .
391 "wallTime = {$statistics['walltime']}, " .
392 "CPUTime = {$statistics['cputime']}, " .
393 "CPUCount = {$statistics['cpucount']}, " .
394 "max_rss = {$statistics['maxmemory']}, " .
395 "startTime = '{$otherdata['starttime']}', " .
396 "endTime = '{$otherdata['endtime']}', " .
397 "mgroupcount = {$otherdata['groupcount']} " .
398 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
399 $result = mysql_query( $query, $us3_link );
400
401 if ( ! $result )
402 {
403 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
404 }
405
406 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
407
408 }
409
410 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
411 {
412 $xml = file_get_contents( $fn );
413 $noise_data = parse_xml( $xml, "noise" );
414 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
415 $desc = $noise_data[ 'description' ];
416 $modelGUID = $noise_data[ 'modelGUID' ];
417 $noiseGUID = $noise_data[ 'noiseGUID' ];
418
419 $query = "INSERT INTO noise SET " .
420 "noiseGUID='$noiseGUID'," .
421 "modelGUID='$modelGUID'," .
422 "editedDataID=1, " .
423 "modelID=1, " .
424 "noiseType='$type'," .
425 "description='$desc'," .
426 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
427
428 // Add later after all files are processed: editDataID, modelID
429
430 $result = mysql_query( $query, $us3_link );
431
432 if ( ! $result )
433 {
434 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
435 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
436 return( -1 );
437 }
438
439 $id = mysql_insert_id( $us3_link );
440 $file_type = "noise";
441 $noiseIDs[] = $id;
442
443 // Keep track of modelGUIDs for later, when we replace them
444 $modelGUIDs[ $id ] = $modelGUID;
445
446 }
447
448 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
449 {
450 $xml = file_get_contents( $fn );
451 $mrecs_data = parse_xml( $xml, "modelrecords" );
452 $desc = $mrecs_data[ 'description' ];
453 $editGUID = $mrecs_data[ 'editGUID' ];
454write_log( "$me: mrecs file editGUID=$editGUID" );
455 if ( strlen( $editGUID ) < 36 )
456 $editGUID = "12345678-0123-5678-0123-567890123456";
457 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
458 $modelGUID = $mrecs_data[ 'modelGUID' ];
459
460 $query = "INSERT INTO pcsa_modelrecs SET " .
461 "editedDataID=" .
462 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
463 "modelID=0, " .
464 "mrecsGUID='$mrecGUID'," .
465 "description='$desc'," .
466 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
467
468 // Add later after all files are processed: editDataID, modelID
469
470 $result = mysql_query( $query, $us3_link );
471
472 if ( ! $result )
473 {
474 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
475 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
476 return( -1 );
477 }
478
479 $id = mysql_insert_id( $us3_link );
480 $file_type = "mrecs";
481 $mrecsIDs[] = $id;
482
483 // Keep track of modelGUIDs for later, when we replace them
484 $rmodlGUIDs[ $id ] = $modelGUID;
485//write_log( "$me: mrecs file inserted into DB : id=$id" );
486 }
487
488 else // It's a model file
489 {
490 $xml = file_get_contents( $fn );
491 $model_data = parse_xml( $xml, "model" );
492 $description = $model_data[ 'description' ];
493 $modelGUID = $model_data[ 'modelGUID' ];
494 $editGUID = $model_data[ 'editGUID' ];
495
496 if ( $mc_iteration > 1 )
497 {
498 $miter = sprintf( "_mcN%03d", $mc_iteration );
499 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
500write_log( "$me: MODELUpd: O:description=$description" );
501 }
502
503 $query = "INSERT INTO model SET " .
504 "modelGUID='$modelGUID'," .
505 "editedDataID=" .
506 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
507 "description='$description'," .
508 "MCIteration='$mc_iteration'," .
509 "meniscus='$meniscus'," .
510 "variance='$variance'," .
511 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
512
513 $result = mysql_query( $query, $us3_link );
514
515 if ( ! $result )
516 {
517 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
518 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
519 return( -1 );
520 }
521
522 $modelID = mysql_insert_id( $us3_link );
523 $id = $modelID;
524 $file_type = "model";
525
526 $query = "INSERT INTO modelPerson SET " .
527 "modelID=$modelID, personID=$personID";
528 $result = mysql_query( $query, $us3_link );
529 }
530
531 $query = "INSERT INTO HPCAnalysisResultData SET " .
532 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
533 "HPCAnalysisResultType='$file_type', " .
534 "resultID=$id";
535
536 $result = mysql_query( $query, $us3_link );
537
538 if ( ! $result )
539 {
540 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
541 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
542 return( -1 );
543 }
544 }
545
546 // Now fix up noise entries
547 // For noise files, there is, at most two: ti_noise and ri_noise
548 // In this case there will only be one modelID
549
550 foreach ( $noiseIDs as $noiseID )
551 {
552 $modelGUID = $modelGUIDs[ $noiseID ];
553 $query = "UPDATE noise SET " .
554 "editedDataID=" .
555 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
556 "modelID=" .
557 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
558 "WHERE noiseID=$noiseID";
559
560 $result = mysql_query( $query, $us3_link );
561
562 if ( ! $result )
563 {
564 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
565 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
566 return( -1 );
567 }
568 }
569
570 // Copy results to LIMS submit directory (files there are deleted after 7 days)
571 global $submit_dir; // LIMS submit files dir
572
573 // Get the request guid (LIMS submit dir name)
574 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
575 "WHERE HPCAnalysisRequestID = $requestID ";
576 $result = mysql_query( $query, $us3_link );
577
578 if ( ! $result )
579 {
580 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
581 }
582
583// list( $requestGUID ) = mysql_fetch_array( $result );
584//
585// chdir( "$submit_dir/$requestGUID" );
586// $f = fopen( "analysis-results.tar", "w" );
587// fwrite( $f, $tarfile );
588// fclose( $f );
589
590 // Clean up
591// chdir ( $work );
592 // exec( "rm -rf $gfacID" );
593
594 mysql_close( $us3_link );
595
596 /////////
597 // Send email
598
599 mail_to_user( "success", "" );
600}
601
602function mail_to_user( $type, $msg )
603{
604 // Note to me. Just changed subject line to include a modified $status instead
605 // of the $type variable passed. More informative than just "fail" or "success."
606 // See how it works for awhile and then consider removing $type parameter from
607 // function.
608 global $email_address;
609 global $submittime;
610 global $queuestatus;
611 global $status;
612 global $cluster;
613 global $jobtype;
614 global $org_name;
615 global $admin_email;
616 global $db;
617 global $dbhost;
618 global $requestID;
619 global $gfacID;
620 global $editXMLFilename;
621 global $stdout;
622
623global $me;
624write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
625
626 // Get GFAC status and message
627 // function get_gfac_message() also sets global $status
628 $gfac_message = get_gfac_message( $gfacID );
629 if ( $gfac_message === false ) $gfac_message = "Job Finished";
630
631 // Create a status to put in the subject line
632 switch ( $status )
633 {
634 case "COMPLETE":
635 $subj_status = 'completed';
636 break;
637
638 case "CANCELLED":
639 case "CANCELED":
640 $subj_status = 'canceled';
641 break;
642
643 case "FAILED":
644 $subj_status = 'failed';
645 if ( preg_match( "/^US3-A/i", $gfacID ) )
646 { // For A/Thrift FAIL, get error message
647 $gfac_message = getExperimentErrors( $gfacID );
648//$gfac_message .= "Test ERROR MESSAGE";
649 }
650 break;
651
652 case "ERROR":
653 $subj_status = 'unknown error';
654 break;
655
656 default:
657 $subj_status = $status; // For now
658 break;
659
660 }
661
662 $queuestatus = $subj_status;
663 $limshost = $dbhost;
664 if ( $limshost == 'localhost' )
665 $limshost = gethostname();
666
667 // Parse the editXMLFilename
668 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
669 explode( ".", $editXMLFilename );
670
671 $headers = "From: $org_name Admin<$admin_email>" . "\n";
672 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
673
674 // Set the reply address
675 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
676 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
677
678 // Try to avoid spam filters
679 $now = time();
680 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
681 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
682 $headers .= "MIME-Version: 1.0" . "\n";
683 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
684
685 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
686 $message = "
687 Your UltraScan job is complete:
688
689 Submission Time : $submittime
690 LIMS Host : $limshost
691 Analysis ID : $gfacID
692 Request ID : $requestID ( $db )
693 RunID : $runID
694 EditID : $editID
695 Data Type : $dataType
696 Cell/Channel/Wl : $cell / $channel / $wl
697 Status : $queuestatus
698 Cluster : $cluster
699 Job Type : $jobtype
700 GFAC Status : $status
701 GFAC Message : $gfac_message
702 Stdout : $stdout
703 ";
704
705 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
706
707 // Handle the error case where an error occurs before fetching the
708 // user's email address
709 if ( $email_address == "" ) $email_address = $admin_email;
710
711 mail( $email_address, $subject, $message, $headers );
712}
713
714function parse_xml( $xml, $type )
715{
716 $parser = new XMLReader();
717 $parser->xml( $xml );
718
719 $results = array();
720
721 while ( $parser->read() )
722 {
723 if ( $parser->name == $type )
724 {
725 while ( $parser->moveToNextAttribute() )
726 {
727 $results[ $parser->name ] = $parser->value;
728 }
729
730 break;
731 }
732 }
733
734 $parser->close();
735 return $results;
736}
737
738// Function to get information about the current job GFAC
739function get_gfac_message( $gfacID )
740{
741 global $serviceURL;
742 global $me;
743
744 $hex = "[0-9a-fA-F]";
745 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
746 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
747 {
748 // Then it's not a GFAC job
749 return false;
750 }
751
752 $url = "$serviceURL/jobstatus/$gfacID";
753 try
754 {
755 $post = new HttpRequest( $url, HttpRequest::METH_GET );
756 $http = $post->send();
757 $xml = $post->getResponseBody();
758 }
759 catch ( HttpException $e )
760 {
761 write_log( "$me: Job status not available - $gfacID" );
762 return false;
763 }
764
765 // Parse the result
766 $gfac_message = parse_message( $xml );
767
768 return $gfac_message;
769}
770
771function parse_message( $xml )
772{
773 global $status;
774 $status = "";
775 $gfac_message = "";
776
777 $parser = new XMLReader();
778 $parser->xml( $xml );
779
780 $results = array();
781
782 while( $parser->read() )
783 {
784 $type = $parser->nodeType;
785
786 if ( $type == XMLReader::ELEMENT )
787 $name = $parser->name;
788
789 else if ( $type == XMLReader::TEXT )
790 {
791 if ( $name == "status" )
792 $status = $parser->value;
793 else
794 $gfac_message = $parser->value;
795 }
796 }
797
798 $parser->close();
799 return $gfac_message;
800}
801
802function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
803{
804 global $work;
805 global $work_remote;
806 global $me;
807 global $db;
808 global $status;
809
810 // Figure out local working directory
811 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
812 $pwd = chdir( "$work/$gfacID" );
813
814 // Figure out remote directory
815 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
816
817 // Get stdout, stderr, output/analysis-results.tar
818 $output = array();
819// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
820//
821// exec( $cmd, $output, $stat );
822// if ( $stat != 0 )
823// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
824//
825// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
826// exec( $cmd, $output, $stat );
827// if ( $stat != 0 )
828// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
829
830 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
831 exec( $cmd, $output, $stat );
832 if ( $stat != 0 )
833 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
834
835 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
836 exec( $cmd, $output, $stat );
837 if ( $stat != 0 )
838 {
839 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
840 sleep( 10 );
841 write_log( "$me: RETRY" );
842 exec( $cmd, $output, $stat );
843 if ( $stat != 0 )
844 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
845 }
846
847 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
848 exec( $cmd, $output, $stat );
849 if ( $stat != 0 )
850 {
851 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
852 sleep( 10 );
853 write_log( "$me: RETRY" );
854 exec( $cmd, $output, $stat );
855 if ( $stat != 0 )
856 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
857 }
858
859 // Write the files to gfacDB
860
861 if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" );
862 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
863 if ( file_exists( "analysis-results.tar" ) )
864 $tarfile = file_get_contents( "analysis-results.tar" );
865
866 $query = "UPDATE analysis SET " .
867 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
868 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
869 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
870
871 $result = mysql_query( $query, $gfac_link );
872
873 if ( ! $result )
874 {
875 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
876 echo "Bad query\n";
877 return( -1 );
878 }
879}
880?>
Note: See TracBrowser for help on using the repository browser.