source: trunk/cleanup.php@ 17

Last change on this file since 17 was 17, checked in by gegorbet, 9 years ago

misc. fixes plus mods for new lonestar5 cluster
File size: 28.2 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $dbhost;
20 global $user;
21 global $passwd;
22 global $db;
23 global $guser;
24 global $gpasswd;
25 global $gDB;
26 global $me;
27 global $work;
28 global $email_address;
29 global $queuestatus;
30 global $jobtype;
31 global $editXMLFilename;
32 global $submittime;
33 global $status;
34 global $stderr;
35 global $stdout;
36 global $tarfile;
37 global $requestID;
38 global $submit_dir;
39 $me = 'cleanup_aira.php';
40
41 $requestID = $reqID;
42 $db = $us3_db;
43 write_log( "$me: debug db=$db; requestID=$requestID" );
44
45 $us3_link = mysql_connect( $dbhost, $user, $passwd );
46
47 if ( ! $us3_link )
48 {
49 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
50 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
51 return( -1 );
52 }
53
54 $result = mysql_select_db( $db, $us3_link );
55
56 if ( ! $result )
57 {
58 write_log( "$me: could not select DB $db" );
59 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
60 return( -1 );
61 }
62
63 // First get basic info for email messages
64 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
65 "WHERE HPCAnalysisRequestID=$requestID";
66 $result = mysql_query( $query, $us3_link );
67
68 if ( ! $result )
69 {
70 write_log( "$me: Bad query: $query" );
71 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
72 return( -1 );
73 }
74
75 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
76
77 $query = "SELECT personID FROM people " .
78 "WHERE personGUID='$investigatorGUID'";
79 $result = mysql_query( $query, $us3_link );
80
81 list( $personID ) = mysql_fetch_array( $result );
82
83 $query = "SELECT clusterName, submitTime, queueStatus, method " .
84 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
85 "WHERE h.HPCAnalysisRequestID=$requestID " .
86 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
87
88 $result = mysql_query( $query, $us3_link );
89
90 if ( ! $result )
91 {
92 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
93 return( -1 );
94 }
95
96 if ( mysql_num_rows( $result ) == 0 )
97 {
98 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
99 return( -1 );
100 }
101
102 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
103
104 // Get the GFAC ID
105 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
106 "WHERE HPCAnalysisRequestID=$requestID";
107
108 $result = mysql_query( $query, $us3_link );
109
110 if ( ! $result )
111 {
112 write_log( "$me: Bad query: $query" );
113 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
114 return( -1 );
115 }
116
117 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
118
119 // Get data from global GFAC DB then insert it into US3 DB
120
121 $result = mysql_select_db( $gDB, $gfac_link );
122
123 if ( ! $result )
124 {
125 write_log( "$me: Could not connect to DB $gDB" );
126 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
127 return( -1 );
128 }
129
130 $query = "SELECT status, cluster, id FROM analysis " .
131 "WHERE gfacID='$gfacID'";
132
133 $result = mysql_query( $query, $gfac_link );
134 if ( ! $result )
135 {
136 write_log( "$me: Could not select GFAC status for $gfacID" );
137 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
138 return( -1 );
139 }
140
141 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
142
143 if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' )
144 {
145 $clushost = $cluster;
146 $clushost = preg_replace( "/\-local/", "", $clushost );
147 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
148 }
149
150
151 $query = "SELECT id FROM analysis " .
152 "WHERE gfacID='$gfacID'";
153
154 $result = mysql_query( $query, $gfac_link );
155
156 if ( ! $result )
157 {
158 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
159 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
160 return( -1 );
161 }
162
163 list( $analysisID ) = mysql_fetch_array( $result );
164
165 // Get the request guid (LIMS submit dir name)
166 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
167 "WHERE HPCAnalysisRequestID = $requestID ";
168 $result = mysql_query( $query, $us3_link );
169
170 if ( ! $result )
171 {
172 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
173 }
174
175 list( $requestGUID ) = mysql_fetch_array( $result );
176 $output_dir = "$submit_dir/$requestGUID";
177
178 // Get stderr,stdout,tarfile from work directory
179 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
180 chdir( "$output_dir" );
181//write_log( "$me: gfacID=$gfacID" );
182//write_log( "$me: submit_dir=$submit_dir" );
183//write_log( "$me: requestGUID=$requestGUID" );
184write_log( "$me: output_dir=$output_dir" );
185
186 $stderr = "";
187 $stdout = "";
188 $tarfile = "";
189 $fn_stderr = "Ultrascan.stderr";
190 $fn_stdout = "Ultrascan.stdout";
191 $fn_tarfile = "analysis-results.tar";
192 $num_try = 0;
193 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
194 {
195 sleep( 10 );
196 $num_try++;
197 }
198
199 $ofiles = scandir( $output_dir );
200 foreach ( $ofiles as $ofile )
201 {
202 if ( preg_match( "/.*stderr$/", $ofile ) )
203 $fn_stderr = $ofile;
204 if ( preg_match( "/.*stdout$/", $ofile ) )
205 $fn_stdout = $ofile;
206//write_log( "$me: ofile=$ofile" );
207 }
208write_log( "$me: fn_stderr=$fn_stderr" );
209write_log( "$me: fn_stdout=$fn_stdout" );
210if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
211else write_log( "$me: NOT FOUND: $fn_tarfile" );
212
213 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
214 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
215 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
216
217 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
218 { // Filter "ipath_userinit" lines out of alamo stdout lines
219 $prefln = strlen( $stdout );
220 $output = array();
221 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
222 $stdout = implode( "\n", $output );
223 $posfln = strlen( $stdout );
224write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
225 }
226
227 // Save queue messages for post-mortem analysis
228 $query = "SELECT message, time FROM queue_messages " .
229 "WHERE analysisID = $analysisID " .
230 "ORDER BY time ";
231 $result = mysql_query( $query, $gfac_link );
232
233 if ( ! $result )
234 {
235 // Just log it and continue
236 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
237 }
238
239 $now = date( 'Y-m-d H:i:s' );
240 $message_log = "US3 DB: $db\n" .
241 "RequestID: $requestID\n" .
242 "GFAC ID: $gfacID\n" .
243 "Processed: $now\n\n" .
244 "Queue Messages\n\n" ;
245 if ( mysql_num_rows( $result ) > 0 )
246 {
247 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
248 $message_log .= "$time $message\n";
249 }
250
251 $query = "DELETE FROM queue_messages " .
252 "WHERE analysisID = $analysisID ";
253
254 $result = mysql_query( $query, $gfac_link );
255
256 if ( ! $result )
257 {
258 // Just log it and continue
259 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
260 }
261
262 $query = "SELECT queue_msg FROM analysis " .
263 "WHERE gfacID='$gfacID' ";
264
265 $result = mysql_query( $query, $gfac_link );
266 list( $queue_msg ) = mysql_fetch_array( $result );
267
268 // But let's allow for investigation of other large stdout and/or stderr
269 if ( strlen( $stdout ) > 20480000 ||
270 strlen( $stderr ) > 20480000 )
271 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
272
273 $message_log .= "\n\n\nStdout Contents\n\n" .
274 $stdout .
275 "\n\n\nStderr Contents\n\n" .
276 $stderr .
277 "\n\n\nGFAC Status: $status\n" .
278 "GFAC message field: $queue_msg\n";
279
280 // Delete data from GFAC DB
281 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
282
283 $result = mysql_query( $query, $gfac_link );
284
285 if ( ! $result )
286 {
287 // Just log it and continue
288 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
289 }
290
291
292 // Try to create it if necessary, and write the file
293 // Let's use FILE_APPEND, in case this is the second time around and the
294 // GFAC job status was INSERTed, rather than UPDATEd
295 if ( ! is_dir( $output_dir ) )
296 mkdir( $output_dir, 0775, true );
297 $message_filename = "$output_dir/$db-$requestID-messages.txt";
298 file_put_contents( $message_filename, $message_log, FILE_APPEND );
299 // mysql_close( $gfac_link );
300
301 /////////
302 // Insert data into HPCAnalysis
303
304 $query = "UPDATE HPCAnalysisResult SET " .
305 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
306 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
307 "queueStatus='completed' " .
308 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
309
310 $result = mysql_query( $query, $us3_link );
311
312 if ( ! $result )
313 {
314 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
315 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
316 return( -1 );
317 }
318
319 // Delete data from GFAC DB
320 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
321
322 $result = mysql_query( $query, $gfac_link );
323
324 if ( ! $result )
325 {
326 // Just log it and continue
327 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
328 }
329
330 // Expand the tar file
331
332 if ( strlen( $tarfile ) == 0 )
333 {
334 write_log( "$me: No tarfile" );
335 mail_to_user( "fail", "No results" );
336 return( -1 );
337 }
338
339 $tar_out = array();
340 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
341
342 // Insert the model files and noise files
343 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
344 $noiseIDs = array();
345 $modelGUIDs = array();
346 $mrecsIDs = array();
347 $fns_used = array();
348
349 foreach ( $files as $file )
350 {
351 $split = explode( ";", $file );
352
353 if ( count( $split ) > 1 )
354 {
355 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
356
357 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
358 list( $other, $variance ) = explode( "=", $variance );
359 list( $other, $meniscus ) = explode( "=", $meniscus );
360 }
361 else
362 $fn = $file;
363
364 if ( preg_match( "/mdl.tmp$/", $fn ) )
365 continue;
366
367 if ( in_array( $fn, $fns_used ) )
368 continue;
369
370 $fns_used[] = $fn;
371
372 if ( filesize( $fn ) < 100 )
373 {
374 write_log( "$me:fn is invalid $fn size filesize($fn)" );
375 mail_to_user( "fail", "Internal error\n$fn is invalid" );
376 return( -1 );
377 }
378
379 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
380 {
381 $xml = file_get_contents( $fn );
382 $statistics = parse_xml( $xml, 'statistics' );
383// $ntries = 0;
384//
385// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
386// { // job_statistics file not totally copied, so retry
387// sleep( 10 );
388// $xml = file_get_contents( $fn );
389// $statistics = parse_xml( $xml, 'statistics' );
390// $ntries++;
391//write_log( "$me:jobstats retry $ntries" );
392// }
393//write_log( "$me:cputime=$statistics['cputime']" );
394
395 $otherdata = parse_xml( $xml, 'id' );
396
397 $query = "UPDATE HPCAnalysisResult SET " .
398 "wallTime = {$statistics['walltime']}, " .
399 "CPUTime = {$statistics['cputime']}, " .
400 "CPUCount = {$statistics['cpucount']}, " .
401 "max_rss = {$statistics['maxmemory']}, " .
402 "startTime = '{$otherdata['starttime']}', " .
403 "endTime = '{$otherdata['endtime']}', " .
404 "mgroupcount = {$otherdata['groupcount']} " .
405 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
406 $result = mysql_query( $query, $us3_link );
407
408 if ( ! $result )
409 {
410 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
411 }
412
413 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
414
415 }
416
417 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
418 {
419 $xml = file_get_contents( $fn );
420 $noise_data = parse_xml( $xml, "noise" );
421 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
422 $desc = $noise_data[ 'description' ];
423 $modelGUID = $noise_data[ 'modelGUID' ];
424 $noiseGUID = $noise_data[ 'noiseGUID' ];
425
426 $query = "INSERT INTO noise SET " .
427 "noiseGUID='$noiseGUID'," .
428 "modelGUID='$modelGUID'," .
429 "editedDataID=1, " .
430 "modelID=1, " .
431 "noiseType='$type'," .
432 "description='$desc'," .
433 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
434
435 // Add later after all files are processed: editDataID, modelID
436
437 $result = mysql_query( $query, $us3_link );
438
439 if ( ! $result )
440 {
441 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
442 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
443 return( -1 );
444 }
445
446 $id = mysql_insert_id( $us3_link );
447 $file_type = "noise";
448 $noiseIDs[] = $id;
449
450 // Keep track of modelGUIDs for later, when we replace them
451 $modelGUIDs[ $id ] = $modelGUID;
452
453 }
454
455 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
456 {
457 $xml = file_get_contents( $fn );
458 $mrecs_data = parse_xml( $xml, "modelrecords" );
459 $desc = $mrecs_data[ 'description' ];
460 $editGUID = $mrecs_data[ 'editGUID' ];
461write_log( "$me: mrecs file editGUID=$editGUID" );
462 if ( strlen( $editGUID ) < 36 )
463 $editGUID = "12345678-0123-5678-0123-567890123456";
464 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
465 $modelGUID = $mrecs_data[ 'modelGUID' ];
466
467 $query = "INSERT INTO pcsa_modelrecs SET " .
468 "editedDataID=" .
469 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
470 "modelID=0, " .
471 "mrecsGUID='$mrecGUID'," .
472 "description='$desc'," .
473 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
474
475 // Add later after all files are processed: editDataID, modelID
476
477 $result = mysql_query( $query, $us3_link );
478
479 if ( ! $result )
480 {
481 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
482 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
483 return( -1 );
484 }
485
486 $id = mysql_insert_id( $us3_link );
487 $file_type = "mrecs";
488 $mrecsIDs[] = $id;
489
490 // Keep track of modelGUIDs for later, when we replace them
491 $rmodlGUIDs[ $id ] = $modelGUID;
492//write_log( "$me: mrecs file inserted into DB : id=$id" );
493 }
494
495 else // It's a model file
496 {
497 $xml = file_get_contents( $fn );
498 $model_data = parse_xml( $xml, "model" );
499 $description = $model_data[ 'description' ];
500 $modelGUID = $model_data[ 'modelGUID' ];
501 $editGUID = $model_data[ 'editGUID' ];
502
503 if ( $mc_iteration > 1 )
504 {
505 $miter = sprintf( "_mcN%03d", $mc_iteration );
506 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
507write_log( "$me: MODELUpd: O:description=$description" );
508 }
509
510 $query = "INSERT INTO model SET " .
511 "modelGUID='$modelGUID'," .
512 "editedDataID=" .
513 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
514 "description='$description'," .
515 "MCIteration='$mc_iteration'," .
516 "meniscus='$meniscus'," .
517 "variance='$variance'," .
518 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
519
520 $result = mysql_query( $query, $us3_link );
521
522 if ( ! $result )
523 {
524 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
525 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
526 return( -1 );
527 }
528
529 $modelID = mysql_insert_id( $us3_link );
530 $id = $modelID;
531 $file_type = "model";
532
533 $query = "INSERT INTO modelPerson SET " .
534 "modelID=$modelID, personID=$personID";
535 $result = mysql_query( $query, $us3_link );
536 }
537
538 $query = "INSERT INTO HPCAnalysisResultData SET " .
539 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
540 "HPCAnalysisResultType='$file_type', " .
541 "resultID=$id";
542
543 $result = mysql_query( $query, $us3_link );
544
545 if ( ! $result )
546 {
547 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
548 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
549 return( -1 );
550 }
551 }
552
553 // Now fix up noise entries
554 // For noise files, there is, at most two: ti_noise and ri_noise
555 // In this case there will only be one modelID
556
557 foreach ( $noiseIDs as $noiseID )
558 {
559 $modelGUID = $modelGUIDs[ $noiseID ];
560 $query = "UPDATE noise SET " .
561 "editedDataID=" .
562 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
563 "modelID=" .
564 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
565 "WHERE noiseID=$noiseID";
566
567 $result = mysql_query( $query, $us3_link );
568
569 if ( ! $result )
570 {
571 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
572 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
573 return( -1 );
574 }
575 }
576
577 // Now possibly fix up mrecs entries
578
579 foreach ( $mrecsIDs as $mrecsID )
580 {
581 $modelGUID = $rmodlGUIDs[ $mrecsID ];
582 $query = "UPDATE pcsa_modelrecs SET " .
583 "modelID=" .
584 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
585 "WHERE mrecsID=$mrecsID";
586
587 $result = mysql_query( $query, $us3_link );
588
589 if ( ! $result )
590 {
591 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
592 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
593 return( -1 );
594 }
595write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
596 }
597//write_log( "$me: mrecs entries updated" );
598
599 // Copy results to LIMS submit directory (files there are deleted after 7 days)
600 global $submit_dir; // LIMS submit files dir
601
602 // Get the request guid (LIMS submit dir name)
603 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
604 "WHERE HPCAnalysisRequestID = $requestID ";
605 $result = mysql_query( $query, $us3_link );
606
607 if ( ! $result )
608 {
609 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
610 }
611
612// list( $requestGUID ) = mysql_fetch_array( $result );
613//
614// chdir( "$submit_dir/$requestGUID" );
615// $f = fopen( "analysis-results.tar", "w" );
616// fwrite( $f, $tarfile );
617// fclose( $f );
618
619 // Clean up
620// chdir ( $work );
621 // exec( "rm -rf $gfacID" );
622
623 mysql_close( $us3_link );
624
625 /////////
626 // Send email
627
628 mail_to_user( "success", "" );
629}
630
631function mail_to_user( $type, $msg )
632{
633 // Note to me. Just changed subject line to include a modified $status instead
634 // of the $type variable passed. More informative than just "fail" or "success."
635 // See how it works for awhile and then consider removing $type parameter from
636 // function.
637 global $email_address;
638 global $submittime;
639 global $queuestatus;
640 global $status;
641 global $cluster;
642 global $jobtype;
643 global $org_name;
644 global $admin_email;
645 global $db;
646 global $dbhost;
647 global $requestID;
648 global $gfacID;
649 global $editXMLFilename;
650 global $stdout;
651
652global $me;
653write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
654
655 // Get GFAC status and message
656 // function get_gfac_message() also sets global $status
657 $gfac_message = get_gfac_message( $gfacID );
658 if ( $gfac_message === false ) $gfac_message = "Job Finished";
659
660 // Create a status to put in the subject line
661 switch ( $status )
662 {
663 case "COMPLETE":
664 $subj_status = 'completed';
665 break;
666
667 case "CANCELLED":
668 case "CANCELED":
669 $subj_status = 'canceled';
670 break;
671
672 case "FAILED":
673 $subj_status = 'failed';
674 if ( preg_match( "/^US3-A/i", $gfacID ) )
675 { // For A/Thrift FAIL, get error message
676 $gfac_message = getExperimentErrors( $gfacID );
677//$gfac_message .= "Test ERROR MESSAGE";
678 }
679 break;
680
681 case "ERROR":
682 $subj_status = 'unknown error';
683 break;
684
685 default:
686 $subj_status = $status; // For now
687 break;
688
689 }
690
691 $queuestatus = $subj_status;
692 $limshost = $dbhost;
693 if ( $limshost == 'localhost' )
694 $limshost = gethostname();
695
696 // Parse the editXMLFilename
697 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
698 explode( ".", $editXMLFilename );
699
700 $headers = "From: $org_name Admin<$admin_email>" . "\n";
701 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
702
703 // Set the reply address
704 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
705 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
706
707 // Try to avoid spam filters
708 $now = time();
709 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
710 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
711 $headers .= "MIME-Version: 1.0" . "\n";
712 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
713
714 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
715 $message = "
716 Your UltraScan job is complete:
717
718 Submission Time : $submittime
719 LIMS Host : $limshost
720 Analysis ID : $gfacID
721 Request ID : $requestID ( $db )
722 RunID : $runID
723 EditID : $editID
724 Data Type : $dataType
725 Cell/Channel/Wl : $cell / $channel / $wl
726 Status : $queuestatus
727 Cluster : $cluster
728 Job Type : $jobtype
729 GFAC Status : $status
730 GFAC Message : $gfac_message
731 Stdout : $stdout
732 ";
733
734 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
735
736 // Handle the error case where an error occurs before fetching the
737 // user's email address
738 if ( $email_address == "" ) $email_address = $admin_email;
739
740 mail( $email_address, $subject, $message, $headers );
741}
742
743function parse_xml( $xml, $type )
744{
745 $parser = new XMLReader();
746 $parser->xml( $xml );
747
748 $results = array();
749
750 while ( $parser->read() )
751 {
752 if ( $parser->name == $type )
753 {
754 while ( $parser->moveToNextAttribute() )
755 {
756 $results[ $parser->name ] = $parser->value;
757 }
758
759 break;
760 }
761 }
762
763 $parser->close();
764 return $results;
765}
766
767// Function to get information about the current job GFAC
768function get_gfac_message( $gfacID )
769{
770 global $serviceURL;
771 global $me;
772
773 $hex = "[0-9a-fA-F]";
774 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
775 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
776 {
777 // Then it's not a GFAC job
778 return false;
779 }
780
781 $url = "$serviceURL/jobstatus/$gfacID";
782 try
783 {
784 $post = new HttpRequest( $url, HttpRequest::METH_GET );
785 $http = $post->send();
786 $xml = $post->getResponseBody();
787 }
788 catch ( HttpException $e )
789 {
790 write_log( "$me: Job status not available - $gfacID" );
791 return false;
792 }
793
794 // Parse the result
795 $gfac_message = parse_message( $xml );
796
797 return $gfac_message;
798}
799
800function parse_message( $xml )
801{
802 global $status;
803 $status = "";
804 $gfac_message = "";
805
806 $parser = new XMLReader();
807 $parser->xml( $xml );
808
809 $results = array();
810
811 while( $parser->read() )
812 {
813 $type = $parser->nodeType;
814
815 if ( $type == XMLReader::ELEMENT )
816 $name = $parser->name;
817
818 else if ( $type == XMLReader::TEXT )
819 {
820 if ( $name == "status" )
821 $status = $parser->value;
822 else
823 $gfac_message = $parser->value;
824 }
825 }
826
827 $parser->close();
828 return $gfac_message;
829}
830
831function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
832{
833 global $work;
834 global $work_remote;
835 global $me;
836 global $db;
837 global $status;
838
839 // Figure out local working directory
840 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
841 $pwd = chdir( "$work/$gfacID" );
842
843 // Figure out remote directory
844 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
845
846 // Get stdout, stderr, output/analysis-results.tar
847 $output = array();
848// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
849//
850// exec( $cmd, $output, $stat );
851// if ( $stat != 0 )
852// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
853//
854// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
855// exec( $cmd, $output, $stat );
856// if ( $stat != 0 )
857// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
858
859 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
860 exec( $cmd, $output, $stat );
861 if ( $stat != 0 )
862 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
863
864 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
865 exec( $cmd, $output, $stat );
866 if ( $stat != 0 )
867 {
868 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
869 sleep( 10 );
870 write_log( "$me: RETRY" );
871 exec( $cmd, $output, $stat );
872 if ( $stat != 0 )
873 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
874 }
875
876 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
877 exec( $cmd, $output, $stat );
878 if ( $stat != 0 )
879 {
880 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
881 sleep( 10 );
882 write_log( "$me: RETRY" );
883 exec( $cmd, $output, $stat );
884 if ( $stat != 0 )
885 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
886 }
887
888 // Write the files to gfacDB
889
890 if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" );
891 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
892 if ( file_exists( "analysis-results.tar" ) )
893 $tarfile = file_get_contents( "analysis-results.tar" );
894
895 $query = "UPDATE analysis SET " .
896 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
897 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
898 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
899
900 $result = mysql_query( $query, $gfac_link );
901
902 if ( ! $result )
903 {
904 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
905 echo "Bad query\n";
906 return( -1 );
907 }
908}
909?>
Note: See TracBrowser for help on using the repository browser.