source: trunk/cleanup.php@ 24

Last change on this file since 24 was 24, checked in by gegorbet, 8 years ago

further cleanup fixes to store proper AnalysisResultData records
File size: 28.4 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $dbhost;
20 global $user;
21 global $passwd;
22 global $db;
23 global $guser;
24 global $gpasswd;
25 global $gDB;
26 global $me;
27 global $work;
28 global $email_address;
29 global $queuestatus;
30 global $jobtype;
31 global $editXMLFilename;
32 global $submittime;
33 global $status;
34 global $stderr;
35 global $stdout;
36 global $tarfile;
37 global $requestID;
38 global $submit_dir;
39 $me = 'cleanup_aira.php';
40
41 $requestID = $reqID;
42 $db = $us3_db;
43 write_log( "$me: debug db=$db; requestID=$requestID" );
44
45 $us3_link = mysql_connect( $dbhost, $user, $passwd );
46
47 if ( ! $us3_link )
48 {
49 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
50 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
51 return( -1 );
52 }
53
54 $result = mysql_select_db( $db, $us3_link );
55
56 if ( ! $result )
57 {
58 write_log( "$me: could not select DB $db" );
59 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
60 return( -1 );
61 }
62
63 // First get basic info for email messages
64 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
65 "WHERE HPCAnalysisRequestID=$requestID";
66 $result = mysql_query( $query, $us3_link );
67
68 if ( ! $result )
69 {
70 write_log( "$me: Bad query: $query" );
71 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
72 return( -1 );
73 }
74
75 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
76
77 $query = "SELECT personID FROM people " .
78 "WHERE personGUID='$investigatorGUID'";
79 $result = mysql_query( $query, $us3_link );
80
81 list( $personID ) = mysql_fetch_array( $result );
82
83 $query = "SELECT clusterName, submitTime, queueStatus, method " .
84 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
85 "WHERE h.HPCAnalysisRequestID=$requestID " .
86 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
87
88 $result = mysql_query( $query, $us3_link );
89
90 if ( ! $result )
91 {
92 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
93 return( -1 );
94 }
95
96 if ( mysql_num_rows( $result ) == 0 )
97 {
98 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
99 return( -1 );
100 }
101
102 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
103
104 // Get the GFAC ID
105 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
106 "WHERE HPCAnalysisRequestID=$requestID";
107
108 $result = mysql_query( $query, $us3_link );
109
110 if ( ! $result )
111 {
112 write_log( "$me: Bad query: $query" );
113 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
114 return( -1 );
115 }
116
117 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
118
119 // Get data from global GFAC DB then insert it into US3 DB
120
121 $result = mysql_select_db( $gDB, $gfac_link );
122
123 if ( ! $result )
124 {
125 write_log( "$me: Could not connect to DB $gDB" );
126 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
127 return( -1 );
128 }
129
130 $query = "SELECT status, cluster, id FROM analysis " .
131 "WHERE gfacID='$gfacID'";
132
133 $result = mysql_query( $query, $gfac_link );
134 if ( ! $result )
135 {
136 write_log( "$me: Could not select GFAC status for $gfacID" );
137 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
138 return( -1 );
139 }
140
141 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
142
143 if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' )
144 {
145 $clushost = $cluster;
146 $clushost = preg_replace( "/\-local/", "", $clushost );
147 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
148 }
149
150
151 $query = "SELECT id FROM analysis " .
152 "WHERE gfacID='$gfacID'";
153
154 $result = mysql_query( $query, $gfac_link );
155
156 if ( ! $result )
157 {
158 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
159 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
160 return( -1 );
161 }
162
163 list( $analysisID ) = mysql_fetch_array( $result );
164
165 // Get the request guid (LIMS submit dir name)
166 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
167 "WHERE HPCAnalysisRequestID = $requestID ";
168 $result = mysql_query( $query, $us3_link );
169
170 if ( ! $result )
171 {
172 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
173 }
174
175 list( $requestGUID ) = mysql_fetch_array( $result );
176 $output_dir = "$submit_dir/$requestGUID";
177
178 // Get stderr,stdout,tarfile from work directory
179 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
180 chdir( "$output_dir" );
181//write_log( "$me: gfacID=$gfacID" );
182//write_log( "$me: submit_dir=$submit_dir" );
183//write_log( "$me: requestGUID=$requestGUID" );
184write_log( "$me: output_dir=$output_dir" );
185
186 $stderr = "";
187 $stdout = "";
188 $tarfile = "";
189 $fn_stderr = "Ultrascan.stderr";
190 $fn_stdout = "Ultrascan.stdout";
191 $fn_tarfile = "analysis-results.tar";
192 $num_try = 0;
193 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
194 {
195 sleep( 10 );
196 $num_try++;
197 }
198
199 $ofiles = scandir( $output_dir );
200 foreach ( $ofiles as $ofile )
201 {
202 if ( preg_match( "/.*stderr$/", $ofile ) )
203 $fn_stderr = $ofile;
204 if ( preg_match( "/.*stdout$/", $ofile ) )
205 $fn_stdout = $ofile;
206//write_log( "$me: ofile=$ofile" );
207 }
208write_log( "$me: fn_stderr=$fn_stderr" );
209write_log( "$me: fn_stdout=$fn_stdout" );
210if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
211else write_log( "$me: NOT FOUND: $fn_tarfile" );
212
213 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
214 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
215 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
216
217 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
218 { // Filter "ipath_userinit" lines out of alamo stdout lines
219 $prefln = strlen( $stdout );
220 $output = array();
221 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
222 $stdout = implode( "\n", $output );
223 $posfln = strlen( $stdout );
224write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
225 }
226
227 // Save queue messages for post-mortem analysis
228 $query = "SELECT message, time FROM queue_messages " .
229 "WHERE analysisID = $analysisID " .
230 "ORDER BY time ";
231 $result = mysql_query( $query, $gfac_link );
232
233 if ( ! $result )
234 {
235 // Just log it and continue
236 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
237 }
238
239 $now = date( 'Y-m-d H:i:s' );
240 $message_log = "US3 DB: $db\n" .
241 "RequestID: $requestID\n" .
242 "GFAC ID: $gfacID\n" .
243 "Processed: $now\n\n" .
244 "Queue Messages\n\n" ;
245 if ( mysql_num_rows( $result ) > 0 )
246 {
247 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
248 $message_log .= "$time $message\n";
249 }
250
251 $query = "DELETE FROM queue_messages " .
252 "WHERE analysisID = $analysisID ";
253
254 $result = mysql_query( $query, $gfac_link );
255
256 if ( ! $result )
257 {
258 // Just log it and continue
259 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
260 }
261
262 $query = "SELECT queue_msg FROM analysis " .
263 "WHERE gfacID='$gfacID' ";
264
265 $result = mysql_query( $query, $gfac_link );
266 list( $queue_msg ) = mysql_fetch_array( $result );
267
268 // But let's allow for investigation of other large stdout and/or stderr
269 if ( strlen( $stdout ) > 20480000 ||
270 strlen( $stderr ) > 20480000 )
271 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
272
273 $message_log .= "\n\n\nStdout Contents\n\n" .
274 $stdout .
275 "\n\n\nStderr Contents\n\n" .
276 $stderr .
277 "\n\n\nGFAC Status: $status\n" .
278 "GFAC message field: $queue_msg\n";
279
280 // Delete data from GFAC DB
281 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
282
283 $result = mysql_query( $query, $gfac_link );
284
285 if ( ! $result )
286 {
287 // Just log it and continue
288 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
289 }
290
291
292 // Try to create it if necessary, and write the file
293 // Let's use FILE_APPEND, in case this is the second time around and the
294 // GFAC job status was INSERTed, rather than UPDATEd
295 if ( ! is_dir( $output_dir ) )
296 mkdir( $output_dir, 0775, true );
297 $message_filename = "$output_dir/$db-$requestID-messages.txt";
298 file_put_contents( $message_filename, $message_log, FILE_APPEND );
299 // mysql_close( $gfac_link );
300
301 /////////
302 // Insert data into HPCAnalysis
303
304 $query = "UPDATE HPCAnalysisResult SET " .
305 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
306 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
307 "queueStatus='completed' " .
308 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
309
310 $result = mysql_query( $query, $us3_link );
311
312 if ( ! $result )
313 {
314 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
315 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
316 return( -1 );
317 }
318
319 // Delete data from GFAC DB
320 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
321
322 $result = mysql_query( $query, $gfac_link );
323
324 if ( ! $result )
325 {
326 // Just log it and continue
327 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
328 }
329
330 // Expand the tar file
331
332 if ( strlen( $tarfile ) == 0 )
333 {
334 write_log( "$me: No tarfile" );
335 mail_to_user( "fail", "No results" );
336 return( -1 );
337 }
338
339 $tar_out = array();
340 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
341
342 // Insert the model files and noise files
343 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
344 $noiseIDs = array();
345 $modelGUIDs = array();
346 $mrecsIDs = array();
347 $fns_used = array();
348
349 foreach ( $files as $file )
350 {
351 $split = explode( ";", $file );
352
353 if ( count( $split ) > 1 )
354 {
355 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
356
357 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
358 list( $other, $variance ) = explode( "=", $variance );
359 list( $other, $meniscus ) = explode( "=", $meniscus );
360 }
361 else
362 $fn = $file;
363
364 if ( preg_match( "/mdl.tmp$/", $fn ) )
365 continue;
366
367 if ( in_array( $fn, $fns_used ) )
368 continue;
369
370 $fns_used[] = $fn;
371
372 if ( filesize( $fn ) < 100 )
373 {
374 write_log( "$me:fn is invalid $fn size filesize($fn)" );
375 mail_to_user( "fail", "Internal error\n$fn is invalid" );
376 return( -1 );
377 }
378
379 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
380 {
381 $xml = file_get_contents( $fn );
382 $statistics = parse_xml( $xml, 'statistics' );
383// $ntries = 0;
384//
385// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
386// { // job_statistics file not totally copied, so retry
387// sleep( 10 );
388// $xml = file_get_contents( $fn );
389// $statistics = parse_xml( $xml, 'statistics' );
390// $ntries++;
391//write_log( "$me:jobstats retry $ntries" );
392// }
393//write_log( "$me:cputime=$statistics['cputime']" );
394
395 $otherdata = parse_xml( $xml, 'id' );
396
397 $query = "UPDATE HPCAnalysisResult SET " .
398 "wallTime = {$statistics['walltime']}, " .
399 "CPUTime = {$statistics['cputime']}, " .
400 "CPUCount = {$statistics['cpucount']}, " .
401 "max_rss = {$statistics['maxmemory']}, " .
402 "startTime = '{$otherdata['starttime']}', " .
403 "endTime = '{$otherdata['endtime']}', " .
404 "mgroupcount = {$otherdata['groupcount']} " .
405 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
406 $result = mysql_query( $query, $us3_link );
407
408 if ( ! $result )
409 {
410 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
411 }
412
413 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
414
415 $file_type = "job_stats";
416 $id = 1;
417 }
418
419 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
420 {
421 $xml = file_get_contents( $fn );
422 $noise_data = parse_xml( $xml, "noise" );
423 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
424 $desc = $noise_data[ 'description' ];
425 $modelGUID = $noise_data[ 'modelGUID' ];
426 $noiseGUID = $noise_data[ 'noiseGUID' ];
427
428 $query = "INSERT INTO noise SET " .
429 "noiseGUID='$noiseGUID'," .
430 "modelGUID='$modelGUID'," .
431 "editedDataID=1, " .
432 "modelID=1, " .
433 "noiseType='$type'," .
434 "description='$desc'," .
435 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
436
437 // Add later after all files are processed: editDataID, modelID
438
439 $result = mysql_query( $query, $us3_link );
440
441 if ( ! $result )
442 {
443 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
444 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
445 return( -1 );
446 }
447
448 $id = mysql_insert_id( $us3_link );
449 $file_type = "noise";
450 $noiseIDs[] = $id;
451
452 // Keep track of modelGUIDs for later, when we replace them
453 $modelGUIDs[ $id ] = $modelGUID;
454
455 }
456
457 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
458 {
459 $xml = file_get_contents( $fn );
460 $mrecs_data = parse_xml( $xml, "modelrecords" );
461 $desc = $mrecs_data[ 'description' ];
462 $editGUID = $mrecs_data[ 'editGUID' ];
463write_log( "$me: mrecs file editGUID=$editGUID" );
464 if ( strlen( $editGUID ) < 36 )
465 $editGUID = "12345678-0123-5678-0123-567890123456";
466 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
467 $modelGUID = $mrecs_data[ 'modelGUID' ];
468
469 $query = "INSERT INTO pcsa_modelrecs SET " .
470 "editedDataID=" .
471 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
472 "modelID=0, " .
473 "mrecsGUID='$mrecGUID'," .
474 "description='$desc'," .
475 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
476
477 // Add later after all files are processed: editDataID, modelID
478
479 $result = mysql_query( $query, $us3_link );
480
481 if ( ! $result )
482 {
483 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
484 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
485 return( -1 );
486 }
487
488 $id = mysql_insert_id( $us3_link );
489 $file_type = "mrecs";
490 $mrecsIDs[] = $id;
491
492 // Keep track of modelGUIDs for later, when we replace them
493 $rmodlGUIDs[ $id ] = $modelGUID;
494//write_log( "$me: mrecs file inserted into DB : id=$id" );
495 }
496
497 else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
498 {
499 $xml = file_get_contents( $fn );
500 $model_data = parse_xml( $xml, "model" );
501 $description = $model_data[ 'description' ];
502 $modelGUID = $model_data[ 'modelGUID' ];
503 $editGUID = $model_data[ 'editGUID' ];
504
505 if ( $mc_iteration > 1 )
506 {
507 $miter = sprintf( "_mcN%03d", $mc_iteration );
508 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
509write_log( "$me: MODELUpd: O:description=$description" );
510 }
511
512 $query = "INSERT INTO model SET " .
513 "modelGUID='$modelGUID'," .
514 "editedDataID=" .
515 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
516 "description='$description'," .
517 "MCIteration='$mc_iteration'," .
518 "meniscus='$meniscus'," .
519 "variance='$variance'," .
520 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
521
522 $result = mysql_query( $query, $us3_link );
523
524 if ( ! $result )
525 {
526 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
527 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
528 return( -1 );
529 }
530
531 $modelID = mysql_insert_id( $us3_link );
532 $id = $modelID;
533 $file_type = "model";
534
535 $query = "INSERT INTO modelPerson SET " .
536 "modelID=$modelID, personID=$personID";
537 $result = mysql_query( $query, $us3_link );
538 }
539
540 else // Undetermined type: skip result data update
541 continue;
542
543 $query = "INSERT INTO HPCAnalysisResultData SET " .
544 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
545 "HPCAnalysisResultType='$file_type', " .
546 "resultID=$id";
547
548 $result = mysql_query( $query, $us3_link );
549
550 if ( ! $result )
551 {
552 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
553 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
554 return( -1 );
555 }
556 }
557
558 // Now fix up noise entries
559 // For noise files, there is, at most two: ti_noise and ri_noise
560 // In this case there will only be one modelID
561
562 foreach ( $noiseIDs as $noiseID )
563 {
564 $modelGUID = $modelGUIDs[ $noiseID ];
565 $query = "UPDATE noise SET " .
566 "editedDataID=" .
567 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
568 "modelID=" .
569 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
570 "WHERE noiseID=$noiseID";
571
572 $result = mysql_query( $query, $us3_link );
573
574 if ( ! $result )
575 {
576 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
577 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
578 return( -1 );
579 }
580 }
581
582 // Now possibly fix up mrecs entries
583
584 foreach ( $mrecsIDs as $mrecsID )
585 {
586 $modelGUID = $rmodlGUIDs[ $mrecsID ];
587 $query = "UPDATE pcsa_modelrecs SET " .
588 "modelID=" .
589 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
590 "WHERE mrecsID=$mrecsID";
591
592 $result = mysql_query( $query, $us3_link );
593
594 if ( ! $result )
595 {
596 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
597 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
598 return( -1 );
599 }
600write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
601 }
602//write_log( "$me: mrecs entries updated" );
603
604 // Copy results to LIMS submit directory (files there are deleted after 7 days)
605 global $submit_dir; // LIMS submit files dir
606
607 // Get the request guid (LIMS submit dir name)
608 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
609 "WHERE HPCAnalysisRequestID = $requestID ";
610 $result = mysql_query( $query, $us3_link );
611
612 if ( ! $result )
613 {
614 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
615 }
616
617// list( $requestGUID ) = mysql_fetch_array( $result );
618//
619// chdir( "$submit_dir/$requestGUID" );
620// $f = fopen( "analysis-results.tar", "w" );
621// fwrite( $f, $tarfile );
622// fclose( $f );
623
624 // Clean up
625// chdir ( $work );
626 // exec( "rm -rf $gfacID" );
627
628 mysql_close( $us3_link );
629
630 /////////
631 // Send email
632
633 mail_to_user( "success", "" );
634}
635
636function mail_to_user( $type, $msg )
637{
638 // Note to me. Just changed subject line to include a modified $status instead
639 // of the $type variable passed. More informative than just "fail" or "success."
640 // See how it works for awhile and then consider removing $type parameter from
641 // function.
642 global $email_address;
643 global $submittime;
644 global $queuestatus;
645 global $status;
646 global $cluster;
647 global $jobtype;
648 global $org_name;
649 global $admin_email;
650 global $db;
651 global $dbhost;
652 global $requestID;
653 global $gfacID;
654 global $editXMLFilename;
655 global $stdout;
656
657global $me;
658write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
659
660 // Get GFAC status and message
661 // function get_gfac_message() also sets global $status
662 $gfac_message = get_gfac_message( $gfacID );
663 if ( $gfac_message === false ) $gfac_message = "Job Finished";
664
665 // Create a status to put in the subject line
666 switch ( $status )
667 {
668 case "COMPLETE":
669 $subj_status = 'completed';
670 break;
671
672 case "CANCELLED":
673 case "CANCELED":
674 $subj_status = 'canceled';
675 break;
676
677 case "FAILED":
678 $subj_status = 'failed';
679 if ( preg_match( "/^US3-A/i", $gfacID ) )
680 { // For A/Thrift FAIL, get error message
681 $gfac_message = getExperimentErrors( $gfacID );
682//$gfac_message .= "Test ERROR MESSAGE";
683 }
684 break;
685
686 case "ERROR":
687 $subj_status = 'unknown error';
688 break;
689
690 default:
691 $subj_status = $status; // For now
692 break;
693
694 }
695
696 $queuestatus = $subj_status;
697 $limshost = $dbhost;
698 if ( $limshost == 'localhost' )
699 $limshost = gethostname();
700
701 // Parse the editXMLFilename
702 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
703 explode( ".", $editXMLFilename );
704
705 $headers = "From: $org_name Admin<$admin_email>" . "\n";
706 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
707
708 // Set the reply address
709 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
710 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
711
712 // Try to avoid spam filters
713 $now = time();
714 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
715 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
716 $headers .= "MIME-Version: 1.0" . "\n";
717 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
718
719 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
720 $message = "
721 Your UltraScan job is complete:
722
723 Submission Time : $submittime
724 LIMS Host : $limshost
725 Analysis ID : $gfacID
726 Request ID : $requestID ( $db )
727 RunID : $runID
728 EditID : $editID
729 Data Type : $dataType
730 Cell/Channel/Wl : $cell / $channel / $wl
731 Status : $queuestatus
732 Cluster : $cluster
733 Job Type : $jobtype
734 GFAC Status : $status
735 GFAC Message : $gfac_message
736 Stdout : $stdout
737 ";
738
739 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
740
741 // Handle the error case where an error occurs before fetching the
742 // user's email address
743 if ( $email_address == "" ) $email_address = $admin_email;
744
745 mail( $email_address, $subject, $message, $headers );
746}
747
748function parse_xml( $xml, $type )
749{
750 $parser = new XMLReader();
751 $parser->xml( $xml );
752
753 $results = array();
754
755 while ( $parser->read() )
756 {
757 if ( $parser->name == $type )
758 {
759 while ( $parser->moveToNextAttribute() )
760 {
761 $results[ $parser->name ] = $parser->value;
762 }
763
764 break;
765 }
766 }
767
768 $parser->close();
769 return $results;
770}
771
772// Function to get information about the current job GFAC
773function get_gfac_message( $gfacID )
774{
775 global $serviceURL;
776 global $me;
777
778 $hex = "[0-9a-fA-F]";
779 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
780 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
781 {
782 // Then it's not a GFAC job
783 return false;
784 }
785
786 $url = "$serviceURL/jobstatus/$gfacID";
787 try
788 {
789 $post = new HttpRequest( $url, HttpRequest::METH_GET );
790 $http = $post->send();
791 $xml = $post->getResponseBody();
792 }
793 catch ( HttpException $e )
794 {
795 write_log( "$me: Job status not available - $gfacID" );
796 return false;
797 }
798
799 // Parse the result
800 $gfac_message = parse_message( $xml );
801
802 return $gfac_message;
803}
804
805function parse_message( $xml )
806{
807 global $status;
808 $status = "";
809 $gfac_message = "";
810
811 $parser = new XMLReader();
812 $parser->xml( $xml );
813
814 $results = array();
815
816 while( $parser->read() )
817 {
818 $type = $parser->nodeType;
819
820 if ( $type == XMLReader::ELEMENT )
821 $name = $parser->name;
822
823 else if ( $type == XMLReader::TEXT )
824 {
825 if ( $name == "status" )
826 $status = $parser->value;
827 else
828 $gfac_message = $parser->value;
829 }
830 }
831
832 $parser->close();
833 return $gfac_message;
834}
835
836function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
837{
838 global $work;
839 global $work_remote;
840 global $me;
841 global $db;
842 global $status;
843
844 // Figure out local working directory
845 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
846 $pwd = chdir( "$work/$gfacID" );
847
848 // Figure out remote directory
849 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
850
851 // Get stdout, stderr, output/analysis-results.tar
852 $output = array();
853// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
854//
855// exec( $cmd, $output, $stat );
856// if ( $stat != 0 )
857// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
858//
859// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
860// exec( $cmd, $output, $stat );
861// if ( $stat != 0 )
862// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
863
864 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
865 exec( $cmd, $output, $stat );
866 if ( $stat != 0 )
867 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
868
869 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
870 exec( $cmd, $output, $stat );
871 if ( $stat != 0 )
872 {
873 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
874 sleep( 10 );
875 write_log( "$me: RETRY" );
876 exec( $cmd, $output, $stat );
877 if ( $stat != 0 )
878 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
879 }
880
881 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
882 exec( $cmd, $output, $stat );
883 if ( $stat != 0 )
884 {
885 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
886 sleep( 10 );
887 write_log( "$me: RETRY" );
888 exec( $cmd, $output, $stat );
889 if ( $stat != 0 )
890 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
891 }
892
893 // Write the files to gfacDB
894
895 if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" );
896 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
897 if ( file_exists( "analysis-results.tar" ) )
898 $tarfile = file_get_contents( "analysis-results.tar" );
899
900 $query = "UPDATE analysis SET " .
901 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
902 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
903 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
904
905 $result = mysql_query( $query, $gfac_link );
906
907 if ( ! $result )
908 {
909 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
910 echo "Bad query\n";
911 return( -1 );
912 }
913}
914?>
Note: See TracBrowser for help on using the repository browser.