source: trunk/cleanup.php@ 23

Last change on this file since 23 was 23, checked in by gegorbet, 8 years ago

fix to avoid extra AnalysisResultData in cleanup, lonestar mods to cluster_status
File size: 28.3 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $dbhost;
20 global $user;
21 global $passwd;
22 global $db;
23 global $guser;
24 global $gpasswd;
25 global $gDB;
26 global $me;
27 global $work;
28 global $email_address;
29 global $queuestatus;
30 global $jobtype;
31 global $editXMLFilename;
32 global $submittime;
33 global $status;
34 global $stderr;
35 global $stdout;
36 global $tarfile;
37 global $requestID;
38 global $submit_dir;
39 $me = 'cleanup_aira.php';
40
41 $requestID = $reqID;
42 $db = $us3_db;
43 write_log( "$me: debug db=$db; requestID=$requestID" );
44
45 $us3_link = mysql_connect( $dbhost, $user, $passwd );
46
47 if ( ! $us3_link )
48 {
49 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
50 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
51 return( -1 );
52 }
53
54 $result = mysql_select_db( $db, $us3_link );
55
56 if ( ! $result )
57 {
58 write_log( "$me: could not select DB $db" );
59 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
60 return( -1 );
61 }
62
63 // First get basic info for email messages
64 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
65 "WHERE HPCAnalysisRequestID=$requestID";
66 $result = mysql_query( $query, $us3_link );
67
68 if ( ! $result )
69 {
70 write_log( "$me: Bad query: $query" );
71 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
72 return( -1 );
73 }
74
75 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
76
77 $query = "SELECT personID FROM people " .
78 "WHERE personGUID='$investigatorGUID'";
79 $result = mysql_query( $query, $us3_link );
80
81 list( $personID ) = mysql_fetch_array( $result );
82
83 $query = "SELECT clusterName, submitTime, queueStatus, method " .
84 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
85 "WHERE h.HPCAnalysisRequestID=$requestID " .
86 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
87
88 $result = mysql_query( $query, $us3_link );
89
90 if ( ! $result )
91 {
92 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
93 return( -1 );
94 }
95
96 if ( mysql_num_rows( $result ) == 0 )
97 {
98 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
99 return( -1 );
100 }
101
102 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
103
104 // Get the GFAC ID
105 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
106 "WHERE HPCAnalysisRequestID=$requestID";
107
108 $result = mysql_query( $query, $us3_link );
109
110 if ( ! $result )
111 {
112 write_log( "$me: Bad query: $query" );
113 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
114 return( -1 );
115 }
116
117 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
118
119 // Get data from global GFAC DB then insert it into US3 DB
120
121 $result = mysql_select_db( $gDB, $gfac_link );
122
123 if ( ! $result )
124 {
125 write_log( "$me: Could not connect to DB $gDB" );
126 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
127 return( -1 );
128 }
129
130 $query = "SELECT status, cluster, id FROM analysis " .
131 "WHERE gfacID='$gfacID'";
132
133 $result = mysql_query( $query, $gfac_link );
134 if ( ! $result )
135 {
136 write_log( "$me: Could not select GFAC status for $gfacID" );
137 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
138 return( -1 );
139 }
140
141 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
142
143 if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' )
144 {
145 $clushost = $cluster;
146 $clushost = preg_replace( "/\-local/", "", $clushost );
147 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
148 }
149
150
151 $query = "SELECT id FROM analysis " .
152 "WHERE gfacID='$gfacID'";
153
154 $result = mysql_query( $query, $gfac_link );
155
156 if ( ! $result )
157 {
158 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
159 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
160 return( -1 );
161 }
162
163 list( $analysisID ) = mysql_fetch_array( $result );
164
165 // Get the request guid (LIMS submit dir name)
166 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
167 "WHERE HPCAnalysisRequestID = $requestID ";
168 $result = mysql_query( $query, $us3_link );
169
170 if ( ! $result )
171 {
172 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
173 }
174
175 list( $requestGUID ) = mysql_fetch_array( $result );
176 $output_dir = "$submit_dir/$requestGUID";
177
178 // Get stderr,stdout,tarfile from work directory
179 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
180 chdir( "$output_dir" );
181//write_log( "$me: gfacID=$gfacID" );
182//write_log( "$me: submit_dir=$submit_dir" );
183//write_log( "$me: requestGUID=$requestGUID" );
184write_log( "$me: output_dir=$output_dir" );
185
186 $stderr = "";
187 $stdout = "";
188 $tarfile = "";
189 $fn_stderr = "Ultrascan.stderr";
190 $fn_stdout = "Ultrascan.stdout";
191 $fn_tarfile = "analysis-results.tar";
192 $num_try = 0;
193 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
194 {
195 sleep( 10 );
196 $num_try++;
197 }
198
199 $ofiles = scandir( $output_dir );
200 foreach ( $ofiles as $ofile )
201 {
202 if ( preg_match( "/.*stderr$/", $ofile ) )
203 $fn_stderr = $ofile;
204 if ( preg_match( "/.*stdout$/", $ofile ) )
205 $fn_stdout = $ofile;
206//write_log( "$me: ofile=$ofile" );
207 }
208write_log( "$me: fn_stderr=$fn_stderr" );
209write_log( "$me: fn_stdout=$fn_stdout" );
210if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
211else write_log( "$me: NOT FOUND: $fn_tarfile" );
212
213 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
214 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
215 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
216
217 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
218 { // Filter "ipath_userinit" lines out of alamo stdout lines
219 $prefln = strlen( $stdout );
220 $output = array();
221 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
222 $stdout = implode( "\n", $output );
223 $posfln = strlen( $stdout );
224write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
225 }
226
227 // Save queue messages for post-mortem analysis
228 $query = "SELECT message, time FROM queue_messages " .
229 "WHERE analysisID = $analysisID " .
230 "ORDER BY time ";
231 $result = mysql_query( $query, $gfac_link );
232
233 if ( ! $result )
234 {
235 // Just log it and continue
236 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
237 }
238
239 $now = date( 'Y-m-d H:i:s' );
240 $message_log = "US3 DB: $db\n" .
241 "RequestID: $requestID\n" .
242 "GFAC ID: $gfacID\n" .
243 "Processed: $now\n\n" .
244 "Queue Messages\n\n" ;
245 if ( mysql_num_rows( $result ) > 0 )
246 {
247 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
248 $message_log .= "$time $message\n";
249 }
250
251 $query = "DELETE FROM queue_messages " .
252 "WHERE analysisID = $analysisID ";
253
254 $result = mysql_query( $query, $gfac_link );
255
256 if ( ! $result )
257 {
258 // Just log it and continue
259 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
260 }
261
262 $query = "SELECT queue_msg FROM analysis " .
263 "WHERE gfacID='$gfacID' ";
264
265 $result = mysql_query( $query, $gfac_link );
266 list( $queue_msg ) = mysql_fetch_array( $result );
267
268 // But let's allow for investigation of other large stdout and/or stderr
269 if ( strlen( $stdout ) > 20480000 ||
270 strlen( $stderr ) > 20480000 )
271 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
272
273 $message_log .= "\n\n\nStdout Contents\n\n" .
274 $stdout .
275 "\n\n\nStderr Contents\n\n" .
276 $stderr .
277 "\n\n\nGFAC Status: $status\n" .
278 "GFAC message field: $queue_msg\n";
279
280 // Delete data from GFAC DB
281 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
282
283 $result = mysql_query( $query, $gfac_link );
284
285 if ( ! $result )
286 {
287 // Just log it and continue
288 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
289 }
290
291
292 // Try to create it if necessary, and write the file
293 // Let's use FILE_APPEND, in case this is the second time around and the
294 // GFAC job status was INSERTed, rather than UPDATEd
295 if ( ! is_dir( $output_dir ) )
296 mkdir( $output_dir, 0775, true );
297 $message_filename = "$output_dir/$db-$requestID-messages.txt";
298 file_put_contents( $message_filename, $message_log, FILE_APPEND );
299 // mysql_close( $gfac_link );
300
301 /////////
302 // Insert data into HPCAnalysis
303
304 $query = "UPDATE HPCAnalysisResult SET " .
305 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
306 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
307 "queueStatus='completed' " .
308 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
309
310 $result = mysql_query( $query, $us3_link );
311
312 if ( ! $result )
313 {
314 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
315 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
316 return( -1 );
317 }
318
319 // Delete data from GFAC DB
320 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
321
322 $result = mysql_query( $query, $gfac_link );
323
324 if ( ! $result )
325 {
326 // Just log it and continue
327 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
328 }
329
330 // Expand the tar file
331
332 if ( strlen( $tarfile ) == 0 )
333 {
334 write_log( "$me: No tarfile" );
335 mail_to_user( "fail", "No results" );
336 return( -1 );
337 }
338
339 $tar_out = array();
340 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
341
342 // Insert the model files and noise files
343 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
344 $noiseIDs = array();
345 $modelGUIDs = array();
346 $mrecsIDs = array();
347 $fns_used = array();
348
349 foreach ( $files as $file )
350 {
351 $split = explode( ";", $file );
352
353 if ( count( $split ) > 1 )
354 {
355 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
356
357 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
358 list( $other, $variance ) = explode( "=", $variance );
359 list( $other, $meniscus ) = explode( "=", $meniscus );
360 }
361 else
362 $fn = $file;
363
364 if ( preg_match( "/mdl.tmp$/", $fn ) )
365 continue;
366
367 if ( in_array( $fn, $fns_used ) )
368 continue;
369
370 $fns_used[] = $fn;
371
372 if ( filesize( $fn ) < 100 )
373 {
374 write_log( "$me:fn is invalid $fn size filesize($fn)" );
375 mail_to_user( "fail", "Internal error\n$fn is invalid" );
376 return( -1 );
377 }
378
379 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
380 {
381 $xml = file_get_contents( $fn );
382 $statistics = parse_xml( $xml, 'statistics' );
383// $ntries = 0;
384//
385// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
386// { // job_statistics file not totally copied, so retry
387// sleep( 10 );
388// $xml = file_get_contents( $fn );
389// $statistics = parse_xml( $xml, 'statistics' );
390// $ntries++;
391//write_log( "$me:jobstats retry $ntries" );
392// }
393//write_log( "$me:cputime=$statistics['cputime']" );
394
395 $otherdata = parse_xml( $xml, 'id' );
396
397 $query = "UPDATE HPCAnalysisResult SET " .
398 "wallTime = {$statistics['walltime']}, " .
399 "CPUTime = {$statistics['cputime']}, " .
400 "CPUCount = {$statistics['cpucount']}, " .
401 "max_rss = {$statistics['maxmemory']}, " .
402 "startTime = '{$otherdata['starttime']}', " .
403 "endTime = '{$otherdata['endtime']}', " .
404 "mgroupcount = {$otherdata['groupcount']} " .
405 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
406 $result = mysql_query( $query, $us3_link );
407
408 if ( ! $result )
409 {
410 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
411 }
412
413 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
414
415 $file_type = "jobstats";
416 }
417
418 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
419 {
420 $xml = file_get_contents( $fn );
421 $noise_data = parse_xml( $xml, "noise" );
422 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
423 $desc = $noise_data[ 'description' ];
424 $modelGUID = $noise_data[ 'modelGUID' ];
425 $noiseGUID = $noise_data[ 'noiseGUID' ];
426
427 $query = "INSERT INTO noise SET " .
428 "noiseGUID='$noiseGUID'," .
429 "modelGUID='$modelGUID'," .
430 "editedDataID=1, " .
431 "modelID=1, " .
432 "noiseType='$type'," .
433 "description='$desc'," .
434 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
435
436 // Add later after all files are processed: editDataID, modelID
437
438 $result = mysql_query( $query, $us3_link );
439
440 if ( ! $result )
441 {
442 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
443 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
444 return( -1 );
445 }
446
447 $id = mysql_insert_id( $us3_link );
448 $file_type = "noise";
449 $noiseIDs[] = $id;
450
451 // Keep track of modelGUIDs for later, when we replace them
452 $modelGUIDs[ $id ] = $modelGUID;
453
454 }
455
456 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
457 {
458 $xml = file_get_contents( $fn );
459 $mrecs_data = parse_xml( $xml, "modelrecords" );
460 $desc = $mrecs_data[ 'description' ];
461 $editGUID = $mrecs_data[ 'editGUID' ];
462write_log( "$me: mrecs file editGUID=$editGUID" );
463 if ( strlen( $editGUID ) < 36 )
464 $editGUID = "12345678-0123-5678-0123-567890123456";
465 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
466 $modelGUID = $mrecs_data[ 'modelGUID' ];
467
468 $query = "INSERT INTO pcsa_modelrecs SET " .
469 "editedDataID=" .
470 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
471 "modelID=0, " .
472 "mrecsGUID='$mrecGUID'," .
473 "description='$desc'," .
474 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
475
476 // Add later after all files are processed: editDataID, modelID
477
478 $result = mysql_query( $query, $us3_link );
479
480 if ( ! $result )
481 {
482 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
483 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
484 return( -1 );
485 }
486
487 $id = mysql_insert_id( $us3_link );
488 $file_type = "mrecs";
489 $mrecsIDs[] = $id;
490
491 // Keep track of modelGUIDs for later, when we replace them
492 $rmodlGUIDs[ $id ] = $modelGUID;
493//write_log( "$me: mrecs file inserted into DB : id=$id" );
494 }
495
496 else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
497 {
498 $xml = file_get_contents( $fn );
499 $model_data = parse_xml( $xml, "model" );
500 $description = $model_data[ 'description' ];
501 $modelGUID = $model_data[ 'modelGUID' ];
502 $editGUID = $model_data[ 'editGUID' ];
503
504 if ( $mc_iteration > 1 )
505 {
506 $miter = sprintf( "_mcN%03d", $mc_iteration );
507 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
508write_log( "$me: MODELUpd: O:description=$description" );
509 }
510
511 $query = "INSERT INTO model SET " .
512 "modelGUID='$modelGUID'," .
513 "editedDataID=" .
514 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
515 "description='$description'," .
516 "MCIteration='$mc_iteration'," .
517 "meniscus='$meniscus'," .
518 "variance='$variance'," .
519 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
520
521 $result = mysql_query( $query, $us3_link );
522
523 if ( ! $result )
524 {
525 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
526 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
527 return( -1 );
528 }
529
530 $modelID = mysql_insert_id( $us3_link );
531 $id = $modelID;
532 $file_type = "model";
533
534 $query = "INSERT INTO modelPerson SET " .
535 "modelID=$modelID, personID=$personID";
536 $result = mysql_query( $query, $us3_link );
537 }
538
539 $query = "INSERT INTO HPCAnalysisResultData SET " .
540 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
541 "HPCAnalysisResultType='$file_type', " .
542 "resultID=$id";
543
544 $result = mysql_query( $query, $us3_link );
545
546 if ( ! $result )
547 {
548 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
549 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
550 return( -1 );
551 }
552 }
553
554 // Now fix up noise entries
555 // For noise files, there is, at most two: ti_noise and ri_noise
556 // In this case there will only be one modelID
557
558 foreach ( $noiseIDs as $noiseID )
559 {
560 $modelGUID = $modelGUIDs[ $noiseID ];
561 $query = "UPDATE noise SET " .
562 "editedDataID=" .
563 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
564 "modelID=" .
565 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
566 "WHERE noiseID=$noiseID";
567
568 $result = mysql_query( $query, $us3_link );
569
570 if ( ! $result )
571 {
572 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
573 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
574 return( -1 );
575 }
576 }
577
578 // Now possibly fix up mrecs entries
579
580 foreach ( $mrecsIDs as $mrecsID )
581 {
582 $modelGUID = $rmodlGUIDs[ $mrecsID ];
583 $query = "UPDATE pcsa_modelrecs SET " .
584 "modelID=" .
585 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
586 "WHERE mrecsID=$mrecsID";
587
588 $result = mysql_query( $query, $us3_link );
589
590 if ( ! $result )
591 {
592 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
593 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
594 return( -1 );
595 }
596write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
597 }
598//write_log( "$me: mrecs entries updated" );
599
600 // Copy results to LIMS submit directory (files there are deleted after 7 days)
601 global $submit_dir; // LIMS submit files dir
602
603 // Get the request guid (LIMS submit dir name)
604 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
605 "WHERE HPCAnalysisRequestID = $requestID ";
606 $result = mysql_query( $query, $us3_link );
607
608 if ( ! $result )
609 {
610 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
611 }
612
613// list( $requestGUID ) = mysql_fetch_array( $result );
614//
615// chdir( "$submit_dir/$requestGUID" );
616// $f = fopen( "analysis-results.tar", "w" );
617// fwrite( $f, $tarfile );
618// fclose( $f );
619
620 // Clean up
621// chdir ( $work );
622 // exec( "rm -rf $gfacID" );
623
624 mysql_close( $us3_link );
625
626 /////////
627 // Send email
628
629 mail_to_user( "success", "" );
630}
631
632function mail_to_user( $type, $msg )
633{
634 // Note to me. Just changed subject line to include a modified $status instead
635 // of the $type variable passed. More informative than just "fail" or "success."
636 // See how it works for awhile and then consider removing $type parameter from
637 // function.
638 global $email_address;
639 global $submittime;
640 global $queuestatus;
641 global $status;
642 global $cluster;
643 global $jobtype;
644 global $org_name;
645 global $admin_email;
646 global $db;
647 global $dbhost;
648 global $requestID;
649 global $gfacID;
650 global $editXMLFilename;
651 global $stdout;
652
653global $me;
654write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
655
656 // Get GFAC status and message
657 // function get_gfac_message() also sets global $status
658 $gfac_message = get_gfac_message( $gfacID );
659 if ( $gfac_message === false ) $gfac_message = "Job Finished";
660
661 // Create a status to put in the subject line
662 switch ( $status )
663 {
664 case "COMPLETE":
665 $subj_status = 'completed';
666 break;
667
668 case "CANCELLED":
669 case "CANCELED":
670 $subj_status = 'canceled';
671 break;
672
673 case "FAILED":
674 $subj_status = 'failed';
675 if ( preg_match( "/^US3-A/i", $gfacID ) )
676 { // For A/Thrift FAIL, get error message
677 $gfac_message = getExperimentErrors( $gfacID );
678//$gfac_message .= "Test ERROR MESSAGE";
679 }
680 break;
681
682 case "ERROR":
683 $subj_status = 'unknown error';
684 break;
685
686 default:
687 $subj_status = $status; // For now
688 break;
689
690 }
691
692 $queuestatus = $subj_status;
693 $limshost = $dbhost;
694 if ( $limshost == 'localhost' )
695 $limshost = gethostname();
696
697 // Parse the editXMLFilename
698 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
699 explode( ".", $editXMLFilename );
700
701 $headers = "From: $org_name Admin<$admin_email>" . "\n";
702 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
703
704 // Set the reply address
705 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
706 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
707
708 // Try to avoid spam filters
709 $now = time();
710 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
711 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
712 $headers .= "MIME-Version: 1.0" . "\n";
713 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
714
715 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
716 $message = "
717 Your UltraScan job is complete:
718
719 Submission Time : $submittime
720 LIMS Host : $limshost
721 Analysis ID : $gfacID
722 Request ID : $requestID ( $db )
723 RunID : $runID
724 EditID : $editID
725 Data Type : $dataType
726 Cell/Channel/Wl : $cell / $channel / $wl
727 Status : $queuestatus
728 Cluster : $cluster
729 Job Type : $jobtype
730 GFAC Status : $status
731 GFAC Message : $gfac_message
732 Stdout : $stdout
733 ";
734
735 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
736
737 // Handle the error case where an error occurs before fetching the
738 // user's email address
739 if ( $email_address == "" ) $email_address = $admin_email;
740
741 mail( $email_address, $subject, $message, $headers );
742}
743
744function parse_xml( $xml, $type )
745{
746 $parser = new XMLReader();
747 $parser->xml( $xml );
748
749 $results = array();
750
751 while ( $parser->read() )
752 {
753 if ( $parser->name == $type )
754 {
755 while ( $parser->moveToNextAttribute() )
756 {
757 $results[ $parser->name ] = $parser->value;
758 }
759
760 break;
761 }
762 }
763
764 $parser->close();
765 return $results;
766}
767
768// Function to get information about the current job GFAC
769function get_gfac_message( $gfacID )
770{
771 global $serviceURL;
772 global $me;
773
774 $hex = "[0-9a-fA-F]";
775 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
776 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
777 {
778 // Then it's not a GFAC job
779 return false;
780 }
781
782 $url = "$serviceURL/jobstatus/$gfacID";
783 try
784 {
785 $post = new HttpRequest( $url, HttpRequest::METH_GET );
786 $http = $post->send();
787 $xml = $post->getResponseBody();
788 }
789 catch ( HttpException $e )
790 {
791 write_log( "$me: Job status not available - $gfacID" );
792 return false;
793 }
794
795 // Parse the result
796 $gfac_message = parse_message( $xml );
797
798 return $gfac_message;
799}
800
801function parse_message( $xml )
802{
803 global $status;
804 $status = "";
805 $gfac_message = "";
806
807 $parser = new XMLReader();
808 $parser->xml( $xml );
809
810 $results = array();
811
812 while( $parser->read() )
813 {
814 $type = $parser->nodeType;
815
816 if ( $type == XMLReader::ELEMENT )
817 $name = $parser->name;
818
819 else if ( $type == XMLReader::TEXT )
820 {
821 if ( $name == "status" )
822 $status = $parser->value;
823 else
824 $gfac_message = $parser->value;
825 }
826 }
827
828 $parser->close();
829 return $gfac_message;
830}
831
832function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
833{
834 global $work;
835 global $work_remote;
836 global $me;
837 global $db;
838 global $status;
839
840 // Figure out local working directory
841 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
842 $pwd = chdir( "$work/$gfacID" );
843
844 // Figure out remote directory
845 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
846
847 // Get stdout, stderr, output/analysis-results.tar
848 $output = array();
849// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
850//
851// exec( $cmd, $output, $stat );
852// if ( $stat != 0 )
853// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
854//
855// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
856// exec( $cmd, $output, $stat );
857// if ( $stat != 0 )
858// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
859
860 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
861 exec( $cmd, $output, $stat );
862 if ( $stat != 0 )
863 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
864
865 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
866 exec( $cmd, $output, $stat );
867 if ( $stat != 0 )
868 {
869 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
870 sleep( 10 );
871 write_log( "$me: RETRY" );
872 exec( $cmd, $output, $stat );
873 if ( $stat != 0 )
874 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
875 }
876
877 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
878 exec( $cmd, $output, $stat );
879 if ( $stat != 0 )
880 {
881 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
882 sleep( 10 );
883 write_log( "$me: RETRY" );
884 exec( $cmd, $output, $stat );
885 if ( $stat != 0 )
886 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
887 }
888
889 // Write the files to gfacDB
890
891 if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" );
892 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
893 if ( file_exists( "analysis-results.tar" ) )
894 $tarfile = file_get_contents( "analysis-results.tar" );
895
896 $query = "UPDATE analysis SET " .
897 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
898 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
899 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
900
901 $result = mysql_query( $query, $gfac_link );
902
903 if ( ! $result )
904 {
905 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
906 echo "Bad query\n";
907 return( -1 );
908 }
909}
910?>
Note: See TracBrowser for help on using the repository browser.