source: trunk/cleanup.php@ 15

Last change on this file since 15 was 13, checked in by gegorbet, 9 years ago

fix AIRA cleanup.php to update any mrecs record with modelID
File size: 28.1 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $dbhost;
20 global $user;
21 global $passwd;
22 global $db;
23 global $guser;
24 global $gpasswd;
25 global $gDB;
26 global $me;
27 global $work;
28 global $email_address;
29 global $queuestatus;
30 global $jobtype;
31 global $editXMLFilename;
32 global $submittime;
33 global $status;
34 global $stderr;
35 global $stdout;
36 global $tarfile;
37 global $requestID;
38 global $submit_dir;
39 $me = 'cleanup_aira.php';
40
41 $requestID = $reqID;
42 $db = $us3_db;
43 write_log( "$me: debug db=$db; requestID=$requestID" );
44
45 $us3_link = mysql_connect( $dbhost, $user, $passwd );
46
47 if ( ! $us3_link )
48 {
49 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
50 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
51 return( -1 );
52 }
53
54 $result = mysql_select_db( $db, $us3_link );
55
56 if ( ! $result )
57 {
58 write_log( "$me: could not select DB $db" );
59 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
60 return( -1 );
61 }
62
63 // First get basic info for email messages
64 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
65 "WHERE HPCAnalysisRequestID=$requestID";
66 $result = mysql_query( $query, $us3_link );
67
68 if ( ! $result )
69 {
70 write_log( "$me: Bad query: $query" );
71 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
72 return( -1 );
73 }
74
75 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
76
77 $query = "SELECT personID FROM people " .
78 "WHERE personGUID='$investigatorGUID'";
79 $result = mysql_query( $query, $us3_link );
80
81 list( $personID ) = mysql_fetch_array( $result );
82
83 $query = "SELECT clusterName, submitTime, queueStatus, method " .
84 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
85 "WHERE h.HPCAnalysisRequestID=$requestID " .
86 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
87
88 $result = mysql_query( $query, $us3_link );
89
90 if ( ! $result )
91 {
92 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
93 return( -1 );
94 }
95
96 if ( mysql_num_rows( $result ) == 0 )
97 {
98 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
99 return( -1 );
100 }
101
102 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
103
104 // Get the GFAC ID
105 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
106 "WHERE HPCAnalysisRequestID=$requestID";
107
108 $result = mysql_query( $query, $us3_link );
109
110 if ( ! $result )
111 {
112 write_log( "$me: Bad query: $query" );
113 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
114 return( -1 );
115 }
116
117 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
118
119 // Get data from global GFAC DB then insert it into US3 DB
120
121 $result = mysql_select_db( $gDB, $gfac_link );
122
123 if ( ! $result )
124 {
125 write_log( "$me: Could not connect to DB $gDB" );
126 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
127 return( -1 );
128 }
129
130 $query = "SELECT status, cluster, id FROM analysis " .
131 "WHERE gfacID='$gfacID'";
132
133 $result = mysql_query( $query, $gfac_link );
134 if ( ! $result )
135 {
136 write_log( "$me: Could not select GFAC status for $gfacID" );
137 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
138 return( -1 );
139 }
140
141 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
142
143 if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' )
144 {
145 $clushost = $cluster;
146 $clushost = preg_replace( "/\-local/", "", $clushost );
147 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
148 }
149
150
151 $query = "SELECT id FROM analysis " .
152 "WHERE gfacID='$gfacID'";
153
154 $result = mysql_query( $query, $gfac_link );
155
156 if ( ! $result )
157 {
158 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
159 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
160 return( -1 );
161 }
162
163 list( $analysisID ) = mysql_fetch_array( $result );
164
165 // Get the request guid (LIMS submit dir name)
166 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
167 "WHERE HPCAnalysisRequestID = $requestID ";
168 $result = mysql_query( $query, $us3_link );
169
170 if ( ! $result )
171 {
172 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
173 }
174
175 list( $requestGUID ) = mysql_fetch_array( $result );
176 $output_dir = "$submit_dir/$requestGUID";
177
178 // Get stderr,stdout,tarfile from work directory
179 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
180 chdir( "$output_dir" );
181//write_log( "$me: gfacID=$gfacID" );
182//write_log( "$me: submit_dir=$submit_dir" );
183//write_log( "$me: requestGUID=$requestGUID" );
184write_log( "$me: output_dir=$output_dir" );
185
186 $stderr = "";
187 $stdout = "";
188 $tarfile = "";
189 $fn_stderr = "Ultrascan.stderr";
190 $fn_stdout = "Ultrascan.stdout";
191 $fn_tarfile = "analysis-results.tar";
192 $num_try = 0;
193 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
194 {
195 sleep( 10 );
196 $num_try++;
197 }
198
199 $ofiles = scandir( $output_dir );
200 foreach ( $ofiles as $ofile )
201 {
202 if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
203 $fn_stderr = $ofile;
204 if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
205 $fn_stdout = $ofile;
206//write_log( "$me: ofile=$ofile" );
207 }
208write_log( "$me: fn_stderr=$fn_stderr" );
209write_log( "$me: fn_stdout=$fn_stdout" );
210if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
211else write_log( "$me: NOT FOUND: $fn_tarfile" );
212
213 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
214 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
215 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
216
217 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
218 { // Filter "ipath_userinit" lines out of alamo stdout lines
219 $prefln = strlen( $stdout );
220 $output = array();
221 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
222 $stdout = implode( "\n", $output );
223 $posfln = strlen( $stdout );
224write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
225 }
226
227 // Save queue messages for post-mortem analysis
228 $query = "SELECT message, time FROM queue_messages " .
229 "WHERE analysisID = $analysisID " .
230 "ORDER BY time ";
231 $result = mysql_query( $query, $gfac_link );
232
233 if ( ! $result )
234 {
235 // Just log it and continue
236 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
237 }
238
239 $now = date( 'Y-m-d H:i:s' );
240 $message_log = "US3 DB: $db\n" .
241 "RequestID: $requestID\n" .
242 "GFAC ID: $gfacID\n" .
243 "Processed: $now\n\n" .
244 "Queue Messages\n\n" ;
245 if ( mysql_num_rows( $result ) > 0 )
246 {
247 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
248 $message_log .= "$time $message\n";
249 }
250
251 $query = "DELETE FROM queue_messages " .
252 "WHERE analysisID = $analysisID ";
253
254 $result = mysql_query( $query, $gfac_link );
255
256 if ( ! $result )
257 {
258 // Just log it and continue
259 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
260 }
261
262 $query = "SELECT queue_msg FROM analysis " .
263 "WHERE gfacID='$gfacID' ";
264
265 $result = mysql_query( $query, $gfac_link );
266 list( $queue_msg ) = mysql_fetch_array( $result );
267
268 // But let's allow for investigation of other large stdout and/or stderr
269 if ( strlen( $stdout ) > 20480000 ||
270 strlen( $stderr ) > 20480000 )
271 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
272
273 $message_log .= "\n\n\nStdout Contents\n\n" .
274 $stdout .
275 "\n\n\nStderr Contents\n\n" .
276 $stderr .
277 "\n\n\nGFAC Status: $status\n" .
278 "GFAC message field: $queue_msg\n";
279
280 // Delete data from GFAC DB
281 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
282
283 $result = mysql_query( $query, $gfac_link );
284
285 if ( ! $result )
286 {
287 // Just log it and continue
288 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
289 }
290
291
292 // Try to create it if necessary, and write the file
293 // Let's use FILE_APPEND, in case this is the second time around and the
294 // GFAC job status was INSERTed, rather than UPDATEd
295 if ( ! is_dir( $output_dir ) )
296 mkdir( $output_dir, 0775, true );
297 $message_filename = "$output_dir/$db-$requestID-messages.txt";
298 file_put_contents( $message_filename, $message_log, FILE_APPEND );
299 // mysql_close( $gfac_link );
300
301 /////////
302 // Insert data into HPCAnalysis
303
304 $query = "UPDATE HPCAnalysisResult SET " .
305 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
306 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
307 "queueStatus='completed' " .
308 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
309
310 $result = mysql_query( $query, $us3_link );
311
312 if ( ! $result )
313 {
314 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
315 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
316 return( -1 );
317 }
318
319 // Delete data from GFAC DB
320 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
321
322 $result = mysql_query( $query, $gfac_link );
323
324 if ( ! $result )
325 {
326 // Just log it and continue
327 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
328 }
329
330 // Expand the tar file
331
332 if ( strlen( $tarfile ) == 0 )
333 {
334 write_log( "$me: No tarfile" );
335 mail_to_user( "fail", "No results" );
336 return( -1 );
337 }
338
339 $tar_out = array();
340 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
341
342 // Insert the model files and noise files
343 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
344 $noiseIDs = array();
345 $modelGUIDs = array();
346
347 foreach ( $files as $file )
348 {
349 $split = explode( ";", $file );
350
351 if ( count( $split ) > 1 )
352 {
353 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
354
355 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
356 list( $other, $variance ) = explode( "=", $variance );
357 list( $other, $meniscus ) = explode( "=", $meniscus );
358 }
359 else
360 $fn = $file;
361
362 if ( preg_match( "/mdl.tmp$/", $fn ) )
363 continue;
364
365 if ( filesize( $fn ) < 100 )
366 {
367 write_log( "$me:fn is invalid $fn size filesize($fn)" );
368 mail_to_user( "fail", "Internal error\n$fn is invalid" );
369 return( -1 );
370 }
371
372 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
373 {
374 $xml = file_get_contents( $fn );
375 $statistics = parse_xml( $xml, 'statistics' );
376// $ntries = 0;
377//
378// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
379// { // job_statistics file not totally copied, so retry
380// sleep( 10 );
381// $xml = file_get_contents( $fn );
382// $statistics = parse_xml( $xml, 'statistics' );
383// $ntries++;
384//write_log( "$me:jobstats retry $ntries" );
385// }
386//write_log( "$me:cputime=$statistics['cputime']" );
387
388 $otherdata = parse_xml( $xml, 'id' );
389
390 $query = "UPDATE HPCAnalysisResult SET " .
391 "wallTime = {$statistics['walltime']}, " .
392 "CPUTime = {$statistics['cputime']}, " .
393 "CPUCount = {$statistics['cpucount']}, " .
394 "max_rss = {$statistics['maxmemory']}, " .
395 "startTime = '{$otherdata['starttime']}', " .
396 "endTime = '{$otherdata['endtime']}', " .
397 "mgroupcount = {$otherdata['groupcount']} " .
398 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
399 $result = mysql_query( $query, $us3_link );
400
401 if ( ! $result )
402 {
403 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
404 }
405
406 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
407
408 }
409
410 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
411 {
412 $xml = file_get_contents( $fn );
413 $noise_data = parse_xml( $xml, "noise" );
414 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
415 $desc = $noise_data[ 'description' ];
416 $modelGUID = $noise_data[ 'modelGUID' ];
417 $noiseGUID = $noise_data[ 'noiseGUID' ];
418
419 $query = "INSERT INTO noise SET " .
420 "noiseGUID='$noiseGUID'," .
421 "modelGUID='$modelGUID'," .
422 "editedDataID=1, " .
423 "modelID=1, " .
424 "noiseType='$type'," .
425 "description='$desc'," .
426 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
427
428 // Add later after all files are processed: editDataID, modelID
429
430 $result = mysql_query( $query, $us3_link );
431
432 if ( ! $result )
433 {
434 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
435 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
436 return( -1 );
437 }
438
439 $id = mysql_insert_id( $us3_link );
440 $file_type = "noise";
441 $noiseIDs[] = $id;
442
443 // Keep track of modelGUIDs for later, when we replace them
444 $modelGUIDs[ $id ] = $modelGUID;
445
446 }
447
448 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
449 {
450 $xml = file_get_contents( $fn );
451 $mrecs_data = parse_xml( $xml, "modelrecords" );
452 $desc = $mrecs_data[ 'description' ];
453 $editGUID = $mrecs_data[ 'editGUID' ];
454write_log( "$me: mrecs file editGUID=$editGUID" );
455 if ( strlen( $editGUID ) < 36 )
456 $editGUID = "12345678-0123-5678-0123-567890123456";
457 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
458 $modelGUID = $mrecs_data[ 'modelGUID' ];
459
460 $query = "INSERT INTO pcsa_modelrecs SET " .
461 "editedDataID=" .
462 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
463 "modelID=0, " .
464 "mrecsGUID='$mrecGUID'," .
465 "description='$desc'," .
466 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
467
468 // Add later after all files are processed: editDataID, modelID
469
470 $result = mysql_query( $query, $us3_link );
471
472 if ( ! $result )
473 {
474 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
475 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
476 return( -1 );
477 }
478
479 $id = mysql_insert_id( $us3_link );
480 $file_type = "mrecs";
481 $mrecsIDs[] = $id;
482
483 // Keep track of modelGUIDs for later, when we replace them
484 $rmodlGUIDs[ $id ] = $modelGUID;
485//write_log( "$me: mrecs file inserted into DB : id=$id" );
486 }
487
488 else // It's a model file
489 {
490 $xml = file_get_contents( $fn );
491 $model_data = parse_xml( $xml, "model" );
492 $description = $model_data[ 'description' ];
493 $modelGUID = $model_data[ 'modelGUID' ];
494 $editGUID = $model_data[ 'editGUID' ];
495
496 if ( $mc_iteration > 1 )
497 {
498 $miter = sprintf( "_mcN%03d", $mc_iteration );
499 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
500write_log( "$me: MODELUpd: O:description=$description" );
501 }
502
503 $query = "INSERT INTO model SET " .
504 "modelGUID='$modelGUID'," .
505 "editedDataID=" .
506 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
507 "description='$description'," .
508 "MCIteration='$mc_iteration'," .
509 "meniscus='$meniscus'," .
510 "variance='$variance'," .
511 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
512
513 $result = mysql_query( $query, $us3_link );
514
515 if ( ! $result )
516 {
517 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
518 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
519 return( -1 );
520 }
521
522 $modelID = mysql_insert_id( $us3_link );
523 $id = $modelID;
524 $file_type = "model";
525
526 $query = "INSERT INTO modelPerson SET " .
527 "modelID=$modelID, personID=$personID";
528 $result = mysql_query( $query, $us3_link );
529 }
530
531 $query = "INSERT INTO HPCAnalysisResultData SET " .
532 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
533 "HPCAnalysisResultType='$file_type', " .
534 "resultID=$id";
535
536 $result = mysql_query( $query, $us3_link );
537
538 if ( ! $result )
539 {
540 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
541 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
542 return( -1 );
543 }
544 }
545
546 // Now fix up noise entries
547 // For noise files, there is, at most two: ti_noise and ri_noise
548 // In this case there will only be one modelID
549
550 foreach ( $noiseIDs as $noiseID )
551 {
552 $modelGUID = $modelGUIDs[ $noiseID ];
553 $query = "UPDATE noise SET " .
554 "editedDataID=" .
555 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
556 "modelID=" .
557 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
558 "WHERE noiseID=$noiseID";
559
560 $result = mysql_query( $query, $us3_link );
561
562 if ( ! $result )
563 {
564 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
565 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
566 return( -1 );
567 }
568 }
569
570 // Now possibly fix up mrecs entries
571
572 foreach ( $mrecsIDs as $mrecsID )
573 {
574 $modelGUID = $rmodlGUIDs[ $mrecsID ];
575 $query = "UPDATE pcsa_modelrecs SET " .
576 "modelID=" .
577 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
578 "WHERE mrecsID=$mrecsID";
579
580 $result = mysql_query( $query, $us3_link );
581
582 if ( ! $result )
583 {
584 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
585 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
586 return( -1 );
587 }
588write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
589 }
590//write_log( "$me: mrecs entries updated" );
591
592 // Copy results to LIMS submit directory (files there are deleted after 7 days)
593 global $submit_dir; // LIMS submit files dir
594
595 // Get the request guid (LIMS submit dir name)
596 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
597 "WHERE HPCAnalysisRequestID = $requestID ";
598 $result = mysql_query( $query, $us3_link );
599
600 if ( ! $result )
601 {
602 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
603 }
604
605// list( $requestGUID ) = mysql_fetch_array( $result );
606//
607// chdir( "$submit_dir/$requestGUID" );
608// $f = fopen( "analysis-results.tar", "w" );
609// fwrite( $f, $tarfile );
610// fclose( $f );
611
612 // Clean up
613// chdir ( $work );
614 // exec( "rm -rf $gfacID" );
615
616 mysql_close( $us3_link );
617
618 /////////
619 // Send email
620
621 mail_to_user( "success", "" );
622}
623
624function mail_to_user( $type, $msg )
625{
626 // Note to me. Just changed subject line to include a modified $status instead
627 // of the $type variable passed. More informative than just "fail" or "success."
628 // See how it works for awhile and then consider removing $type parameter from
629 // function.
630 global $email_address;
631 global $submittime;
632 global $queuestatus;
633 global $status;
634 global $cluster;
635 global $jobtype;
636 global $org_name;
637 global $admin_email;
638 global $db;
639 global $dbhost;
640 global $requestID;
641 global $gfacID;
642 global $editXMLFilename;
643 global $stdout;
644
645global $me;
646write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
647
648 // Get GFAC status and message
649 // function get_gfac_message() also sets global $status
650 $gfac_message = get_gfac_message( $gfacID );
651 if ( $gfac_message === false ) $gfac_message = "Job Finished";
652
653 // Create a status to put in the subject line
654 switch ( $status )
655 {
656 case "COMPLETE":
657 $subj_status = 'completed';
658 break;
659
660 case "CANCELLED":
661 case "CANCELED":
662 $subj_status = 'canceled';
663 break;
664
665 case "FAILED":
666 $subj_status = 'failed';
667 if ( preg_match( "/^US3-A/i", $gfacID ) )
668 { // For A/Thrift FAIL, get error message
669 $gfac_message = getExperimentErrors( $gfacID );
670//$gfac_message .= "Test ERROR MESSAGE";
671 }
672 break;
673
674 case "ERROR":
675 $subj_status = 'unknown error';
676 break;
677
678 default:
679 $subj_status = $status; // For now
680 break;
681
682 }
683
684 $queuestatus = $subj_status;
685 $limshost = $dbhost;
686 if ( $limshost == 'localhost' )
687 $limshost = gethostname();
688
689 // Parse the editXMLFilename
690 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
691 explode( ".", $editXMLFilename );
692
693 $headers = "From: $org_name Admin<$admin_email>" . "\n";
694 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
695
696 // Set the reply address
697 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
698 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
699
700 // Try to avoid spam filters
701 $now = time();
702 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
703 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
704 $headers .= "MIME-Version: 1.0" . "\n";
705 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
706
707 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
708 $message = "
709 Your UltraScan job is complete:
710
711 Submission Time : $submittime
712 LIMS Host : $limshost
713 Analysis ID : $gfacID
714 Request ID : $requestID ( $db )
715 RunID : $runID
716 EditID : $editID
717 Data Type : $dataType
718 Cell/Channel/Wl : $cell / $channel / $wl
719 Status : $queuestatus
720 Cluster : $cluster
721 Job Type : $jobtype
722 GFAC Status : $status
723 GFAC Message : $gfac_message
724 Stdout : $stdout
725 ";
726
727 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
728
729 // Handle the error case where an error occurs before fetching the
730 // user's email address
731 if ( $email_address == "" ) $email_address = $admin_email;
732
733 mail( $email_address, $subject, $message, $headers );
734}
735
736function parse_xml( $xml, $type )
737{
738 $parser = new XMLReader();
739 $parser->xml( $xml );
740
741 $results = array();
742
743 while ( $parser->read() )
744 {
745 if ( $parser->name == $type )
746 {
747 while ( $parser->moveToNextAttribute() )
748 {
749 $results[ $parser->name ] = $parser->value;
750 }
751
752 break;
753 }
754 }
755
756 $parser->close();
757 return $results;
758}
759
760// Function to get information about the current job GFAC
761function get_gfac_message( $gfacID )
762{
763 global $serviceURL;
764 global $me;
765
766 $hex = "[0-9a-fA-F]";
767 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
768 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
769 {
770 // Then it's not a GFAC job
771 return false;
772 }
773
774 $url = "$serviceURL/jobstatus/$gfacID";
775 try
776 {
777 $post = new HttpRequest( $url, HttpRequest::METH_GET );
778 $http = $post->send();
779 $xml = $post->getResponseBody();
780 }
781 catch ( HttpException $e )
782 {
783 write_log( "$me: Job status not available - $gfacID" );
784 return false;
785 }
786
787 // Parse the result
788 $gfac_message = parse_message( $xml );
789
790 return $gfac_message;
791}
792
793function parse_message( $xml )
794{
795 global $status;
796 $status = "";
797 $gfac_message = "";
798
799 $parser = new XMLReader();
800 $parser->xml( $xml );
801
802 $results = array();
803
804 while( $parser->read() )
805 {
806 $type = $parser->nodeType;
807
808 if ( $type == XMLReader::ELEMENT )
809 $name = $parser->name;
810
811 else if ( $type == XMLReader::TEXT )
812 {
813 if ( $name == "status" )
814 $status = $parser->value;
815 else
816 $gfac_message = $parser->value;
817 }
818 }
819
820 $parser->close();
821 return $gfac_message;
822}
823
824function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
825{
826 global $work;
827 global $work_remote;
828 global $me;
829 global $db;
830 global $status;
831
832 // Figure out local working directory
833 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
834 $pwd = chdir( "$work/$gfacID" );
835
836 // Figure out remote directory
837 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
838
839 // Get stdout, stderr, output/analysis-results.tar
840 $output = array();
841// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
842//
843// exec( $cmd, $output, $stat );
844// if ( $stat != 0 )
845// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
846//
847// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
848// exec( $cmd, $output, $stat );
849// if ( $stat != 0 )
850// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
851
852 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
853 exec( $cmd, $output, $stat );
854 if ( $stat != 0 )
855 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
856
857 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
858 exec( $cmd, $output, $stat );
859 if ( $stat != 0 )
860 {
861 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
862 sleep( 10 );
863 write_log( "$me: RETRY" );
864 exec( $cmd, $output, $stat );
865 if ( $stat != 0 )
866 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
867 }
868
869 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
870 exec( $cmd, $output, $stat );
871 if ( $stat != 0 )
872 {
873 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
874 sleep( 10 );
875 write_log( "$me: RETRY" );
876 exec( $cmd, $output, $stat );
877 if ( $stat != 0 )
878 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
879 }
880
881 // Write the files to gfacDB
882
883 if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" );
884 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
885 if ( file_exists( "analysis-results.tar" ) )
886 $tarfile = file_get_contents( "analysis-results.tar" );
887
888 $query = "UPDATE analysis SET " .
889 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
890 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
891 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
892
893 $result = mysql_query( $query, $gfac_link );
894
895 if ( ! $result )
896 {
897 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
898 echo "Bad query\n";
899 return( -1 );
900 }
901}
902?>
Note: See TracBrowser for help on using the repository browser.