source: trunk/cleanup_aira.php@ 8

Last change on this file since 8 was 8, checked in by gegorbet, 10 years ago

fixes for PCSA with Airavata/Thrift
  • Property svn:keywords set to LastChangedDate Author
File size: 27.6 KB
RevLine 
[7]1<?php
2/*
3 * cleanup_aira.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$us3bin = exec( "ls -d ~us3/bin" );
11include_once "$us3bin/listen-config.php";
12include "/srv/www/htdocs/common/class/experiment_status.php";
13include "/srv/www/htdocs/common/class/experiment_errors.php";
14$me = 'cleanup_aira.php';
15$email_address = '';
16$queuestatus = '';
17$jobtype = '';
18$db = '';
19$editXMLFilename = '';
20$status = '';
21
22function aira_cleanup( $us3_db, $reqID, $gfac_link )
23{
24 global $dbhost;
25 global $user;
26 global $passwd;
27 global $db;
28 global $guser;
29 global $gpasswd;
30 global $gDB;
31 global $me;
32 global $work;
33 global $email_address;
34 global $queuestatus;
35 global $jobtype;
36 global $editXMLFilename;
37 global $submittime;
38 global $status;
39 global $stderr;
40 global $stdout;
41 global $tarfile;
42 global $requestID;
43 global $submit_dir;
44 $me = 'cleanup_aira.php';
45
46 $requestID = $reqID;
47 $db = $us3_db;
48 write_log( "$me: debug db=$db; requestID=$requestID" );
49
50 $us3_link = mysql_connect( $dbhost, $user, $passwd );
51
52 if ( ! $us3_link )
53 {
54 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
55 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
56 return( -1 );
57 }
58
59 $result = mysql_select_db( $db, $us3_link );
60
61 if ( ! $result )
62 {
63 write_log( "$me: could not select DB $db" );
64 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
65 return( -1 );
66 }
67
68 // First get basic info for email messages
69 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
70 "WHERE HPCAnalysisRequestID=$requestID";
71 $result = mysql_query( $query, $us3_link );
72
73 if ( ! $result )
74 {
75 write_log( "$me: Bad query: $query" );
76 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
77 return( -1 );
78 }
79
80 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
81
82 $query = "SELECT personID FROM people " .
83 "WHERE personGUID='$investigatorGUID'";
84 $result = mysql_query( $query, $us3_link );
85
86 list( $personID ) = mysql_fetch_array( $result );
87
88 $query = "SELECT clusterName, submitTime, queueStatus, method " .
89 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
90 "WHERE h.HPCAnalysisRequestID=$requestID " .
91 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
92
93 $result = mysql_query( $query, $us3_link );
94
95 if ( ! $result )
96 {
97 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
98 return( -1 );
99 }
100
101 if ( mysql_num_rows( $result ) == 0 )
102 {
103 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
104 return( -1 );
105 }
106
107 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
108
109 // Get the GFAC ID
110 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
111 "WHERE HPCAnalysisRequestID=$requestID";
112
113 $result = mysql_query( $query, $us3_link );
114
115 if ( ! $result )
116 {
117 write_log( "$me: Bad query: $query" );
118 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
119 return( -1 );
120 }
121
122 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
123
124 // Get data from global GFAC DB then insert it into US3 DB
125
126 $result = mysql_select_db( $gDB, $gfac_link );
127
128 if ( ! $result )
129 {
130 write_log( "$me: Could not connect to DB $gDB" );
131 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
132 return( -1 );
133 }
134
135 $query = "SELECT status, cluster, id FROM analysis " .
136 "WHERE gfacID='$gfacID'";
137
138 $result = mysql_query( $query, $gfac_link );
139 if ( ! $result )
140 {
141 write_log( "$me: Could not select GFAC status for $gfacID" );
142 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
143 return( -1 );
144 }
145
146 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
147
148 if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' )
149 {
150 $clushost = $cluster;
151 $clushost = preg_replace( "/\-local/", "", $clushost );
152 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
153 }
154
155
156 $query = "SELECT id FROM analysis " .
157 "WHERE gfacID='$gfacID'";
158
159 $result = mysql_query( $query, $gfac_link );
160
161 if ( ! $result )
162 {
163 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
164 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
165 return( -1 );
166 }
167
168 list( $analysisID ) = mysql_fetch_array( $result );
169
170 // Get the request guid (LIMS submit dir name)
171 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
172 "WHERE HPCAnalysisRequestID = $requestID ";
173 $result = mysql_query( $query, $us3_link );
174
175 if ( ! $result )
176 {
177 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
178 }
179
180 list( $requestGUID ) = mysql_fetch_array( $result );
181 $output_dir = "$submit_dir/$requestGUID";
182
183 // Get stderr,stdout,tarfile from work directory
184 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
185 chdir( "$output_dir" );
186//write_log( "$me: gfacID=$gfacID" );
187//write_log( "$me: submit_dir=$submit_dir" );
188//write_log( "$me: requestGUID=$requestGUID" );
189write_log( "$me: output_dir=$output_dir" );
190
191 $stderr = "";
192 $stdout = "";
193 $tarfile = "";
194 $fn_stderr = "Ultrascan.stderr";
195 $fn_stdout = "Ultrascan.stdout";
196 $fn_tarfile = "analysis-results.tar";
197 $num_try = 0;
198 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
199 {
200 sleep( 10 );
201 $num_try++;
202 }
203
204 $ofiles = scandir( $output_dir );
205 foreach ( $ofiles as $ofile )
206 {
207 if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
208 $fn_stderr = $ofile;
209 if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
210 $fn_stdout = $ofile;
211//write_log( "$me: ofile=$ofile" );
212 }
213write_log( "$me: fn_stderr=$fn_stderr" );
214write_log( "$me: fn_stdout=$fn_stdout" );
215if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
216else write_log( "$me: NOT FOUND: $fn_tarfile" );
217
218 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
219 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
220 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
221
222 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
223 { // Filter "ipath_userinit" lines out of alamo stdout lines
224 $prefln = strlen( $stdout );
225 $output = array();
226 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
227 $stdout = implode( "\n", $output );
228 $posfln = strlen( $stdout );
229write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
230 }
231
232 // Save queue messages for post-mortem analysis
233 $query = "SELECT message, time FROM queue_messages " .
234 "WHERE analysisID = $analysisID " .
235 "ORDER BY time ";
236 $result = mysql_query( $query, $gfac_link );
237
238 if ( ! $result )
239 {
240 // Just log it and continue
241 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
242 }
243
244 $now = date( 'Y-m-d H:i:s' );
245 $message_log = "US3 DB: $db\n" .
246 "RequestID: $requestID\n" .
247 "GFAC ID: $gfacID\n" .
248 "Processed: $now\n\n" .
249 "Queue Messages\n\n" ;
250 if ( mysql_num_rows( $result ) > 0 )
251 {
252 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
253 $message_log .= "$time $message\n";
254 }
255
256 $query = "DELETE FROM queue_messages " .
257 "WHERE analysisID = $analysisID ";
258
259 $result = mysql_query( $query, $gfac_link );
260
261 if ( ! $result )
262 {
263 // Just log it and continue
264 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
265 }
266
267 $query = "SELECT queue_msg FROM analysis " .
268 "WHERE gfacID='$gfacID' ";
269
270 $result = mysql_query( $query, $gfac_link );
271 list( $queue_msg ) = mysql_fetch_array( $result );
272
273 // But let's allow for investigation of other large stdout and/or stderr
274 if ( strlen( $stdout ) > 20480000 ||
275 strlen( $stderr ) > 20480000 )
276 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
277
278 $message_log .= "\n\n\nStdout Contents\n\n" .
279 $stdout .
280 "\n\n\nStderr Contents\n\n" .
281 $stderr .
282 "\n\n\nGFAC Status: $status\n" .
283 "GFAC message field: $queue_msg\n";
284
285 // Delete data from GFAC DB
286 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
287
288 $result = mysql_query( $query, $gfac_link );
289
290 if ( ! $result )
291 {
292 // Just log it and continue
293 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
294 }
295
296
297 // Try to create it if necessary, and write the file
298 // Let's use FILE_APPEND, in case this is the second time around and the
299 // GFAC job status was INSERTed, rather than UPDATEd
300 if ( ! is_dir( $output_dir ) )
301 mkdir( $output_dir, 0775, true );
302 $message_filename = "$output_dir/$db-$requestID-messages.txt";
303 file_put_contents( $message_filename, $message_log, FILE_APPEND );
304 // mysql_close( $gfac_link );
305
306 /////////
307 // Insert data into HPCAnalysis
308
309 $query = "UPDATE HPCAnalysisResult SET " .
310 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
311 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
312 "queueStatus='completed' " .
313 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
314
315 $result = mysql_query( $query, $us3_link );
316
317 if ( ! $result )
318 {
319 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
320 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
321 return( -1 );
322 }
323
324 // Delete data from GFAC DB
325 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
326
327 $result = mysql_query( $query, $gfac_link );
328
329 if ( ! $result )
330 {
331 // Just log it and continue
332 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
333 }
334
335 // Expand the tar file
336
337 if ( strlen( $tarfile ) == 0 )
338 {
339 write_log( "$me: No tarfile" );
340 mail_to_user( "fail", "No results" );
341 return( -1 );
342 }
343
344 $tar_out = array();
345 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
346
347 // Insert the model files and noise files
348 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
349 $noiseIDs = array();
350 $modelGUIDs = array();
351
352 foreach ( $files as $file )
353 {
354 $split = explode( ";", $file );
355
356 if ( count( $split ) > 1 )
357 {
358 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
359
360 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
361 list( $other, $variance ) = explode( "=", $variance );
362 list( $other, $meniscus ) = explode( "=", $meniscus );
363 }
364 else
365 $fn = $file;
366
367 if ( preg_match( "/mdl.tmp$/", $fn ) )
368 continue;
369
370 if ( filesize( $fn ) < 100 )
371 {
372 write_log( "$me:fn is invalid $fn size filesize($fn)" );
373 mail_to_user( "fail", "Internal error\n$fn is invalid" );
374 return( -1 );
375 }
376
377 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
378 {
379 $xml = file_get_contents( $fn );
380 $statistics = parse_xml( $xml, 'statistics' );
381// $ntries = 0;
382//
383// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
384// { // job_statistics file not totally copied, so retry
385// sleep( 10 );
386// $xml = file_get_contents( $fn );
387// $statistics = parse_xml( $xml, 'statistics' );
388// $ntries++;
389//write_log( "$me:jobstats retry $ntries" );
390// }
391//write_log( "$me:cputime=$statistics['cputime']" );
392
393 $otherdata = parse_xml( $xml, 'id' );
394
395 $query = "UPDATE HPCAnalysisResult SET " .
396 "wallTime = {$statistics['walltime']}, " .
397 "CPUTime = {$statistics['cputime']}, " .
398 "CPUCount = {$statistics['cpucount']}, " .
399 "max_rss = {$statistics['maxmemory']}, " .
400 "startTime = '{$otherdata['starttime']}', " .
401 "endTime = '{$otherdata['endtime']}', " .
402 "mgroupcount = {$otherdata['groupcount']} " .
403 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
404 $result = mysql_query( $query, $us3_link );
405
406 if ( ! $result )
407 {
408 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
409 }
410
411 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
412
413 }
414
415 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
416 {
417 $xml = file_get_contents( $fn );
418 $noise_data = parse_xml( $xml, "noise" );
419 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
420 $desc = $noise_data[ 'description' ];
421 $modelGUID = $noise_data[ 'modelGUID' ];
422 $noiseGUID = $noise_data[ 'noiseGUID' ];
423
424 $query = "INSERT INTO noise SET " .
425 "noiseGUID='$noiseGUID'," .
426 "modelGUID='$modelGUID'," .
427 "editedDataID=1, " .
428 "modelID=1, " .
429 "noiseType='$type'," .
430 "description='$desc'," .
431 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
432
433 // Add later after all files are processed: editDataID, modelID
434
435 $result = mysql_query( $query, $us3_link );
436
437 if ( ! $result )
438 {
439 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
440 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
441 return( -1 );
442 }
443
444 $id = mysql_insert_id( $us3_link );
445 $file_type = "noise";
446 $noiseIDs[] = $id;
447
448 // Keep track of modelGUIDs for later, when we replace them
449 $modelGUIDs[ $id ] = $modelGUID;
450
451 }
[8]452
453 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
[7]454 {
455 $xml = file_get_contents( $fn );
[8]456 $mrecs_data = parse_xml( $xml, "modelrecords" );
457 $desc = $mrecs_data[ 'description' ];
458 $editGUID = $mrecs_data[ 'editGUID' ];
459write_log( "$me: mrecs file editGUID=$editGUID" );
460 if ( strlen( $editGUID ) < 36 )
461 $editGUID = "12345678-0123-5678-0123-567890123456";
462 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
463 $modelGUID = $mrecs_data[ 'modelGUID' ];
464
465 $query = "INSERT INTO pcsa_modelrecs SET " .
466 "editedDataID=" .
467 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
468 "modelID=0, " .
469 "mrecsGUID='$mrecGUID'," .
470 "description='$desc'," .
471 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
472
473 // Add later after all files are processed: editDataID, modelID
474
475 $result = mysql_query( $query, $us3_link );
476
477 if ( ! $result )
478 {
479 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
480 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
481 return( -1 );
482 }
483
484 $id = mysql_insert_id( $us3_link );
485 $file_type = "mrecs";
486 $mrecsIDs[] = $id;
487
488 // Keep track of modelGUIDs for later, when we replace them
489 $rmodlGUIDs[ $id ] = $modelGUID;
490//write_log( "$me: mrecs file inserted into DB : id=$id" );
491 }
492
493 else // It's a model file
494 {
495 $xml = file_get_contents( $fn );
[7]496 $model_data = parse_xml( $xml, "model" );
497 $description = $model_data[ 'description' ];
498 $modelGUID = $model_data[ 'modelGUID' ];
499 $editGUID = $model_data[ 'editGUID' ];
500
501 if ( $mc_iteration > 1 )
502 {
503 $miter = sprintf( "_mcN%03d", $mc_iteration );
504 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
505write_log( "$me: MODELUpd: O:description=$description" );
506 }
507
508 $query = "INSERT INTO model SET " .
509 "modelGUID='$modelGUID'," .
510 "editedDataID=" .
511 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
512 "description='$description'," .
513 "MCIteration='$mc_iteration'," .
514 "meniscus='$meniscus'," .
515 "variance='$variance'," .
516 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
517
518 $result = mysql_query( $query, $us3_link );
519
520 if ( ! $result )
521 {
522 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
523 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
524 return( -1 );
525 }
526
527 $modelID = mysql_insert_id( $us3_link );
528 $id = $modelID;
529 $file_type = "model";
530
531 $query = "INSERT INTO modelPerson SET " .
532 "modelID=$modelID, personID=$personID";
533 $result = mysql_query( $query, $us3_link );
534 }
535
536 $query = "INSERT INTO HPCAnalysisResultData SET " .
537 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
538 "HPCAnalysisResultType='$file_type', " .
539 "resultID=$id";
540
541 $result = mysql_query( $query, $us3_link );
542
543 if ( ! $result )
544 {
545 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
546 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
547 return( -1 );
548 }
549 }
550
551 // Now fix up noise entries
552 // For noise files, there is, at most two: ti_noise and ri_noise
553 // In this case there will only be one modelID
554
555 foreach ( $noiseIDs as $noiseID )
556 {
557 $modelGUID = $modelGUIDs[ $noiseID ];
558 $query = "UPDATE noise SET " .
559 "editedDataID=" .
560 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
561 "modelID=" .
562 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
563 "WHERE noiseID=$noiseID";
564
565 $result = mysql_query( $query, $us3_link );
566
567 if ( ! $result )
568 {
569 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
570 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
571 return( -1 );
572 }
573 }
574
575 // Copy results to LIMS submit directory (files there are deleted after 7 days)
576 global $submit_dir; // LIMS submit files dir
577
578 // Get the request guid (LIMS submit dir name)
579 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
580 "WHERE HPCAnalysisRequestID = $requestID ";
581 $result = mysql_query( $query, $us3_link );
582
583 if ( ! $result )
584 {
585 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
586 }
587
588// list( $requestGUID ) = mysql_fetch_array( $result );
589//
590// chdir( "$submit_dir/$requestGUID" );
591// $f = fopen( "analysis-results.tar", "w" );
592// fwrite( $f, $tarfile );
593// fclose( $f );
594
595 // Clean up
596// chdir ( $work );
597 // exec( "rm -rf $gfacID" );
598
599 mysql_close( $us3_link );
600
601 /////////
602 // Send email
603
604 mail_to_user( "success", "" );
605}
606
607function mail_to_user( $type, $msg )
608{
609 // Note to me. Just changed subject line to include a modified $status instead
610 // of the $type variable passed. More informative than just "fail" or "success."
611 // See how it works for awhile and then consider removing $type parameter from
612 // function.
613 global $email_address;
614 global $submittime;
615 global $queuestatus;
616 global $status;
617 global $cluster;
618 global $jobtype;
619 global $org_name;
620 global $admin_email;
621 global $db;
622 global $dbhost;
623 global $requestID;
624 global $gfacID;
625 global $editXMLFilename;
626 global $stdout;
627
628global $me;
629write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
630
631 // Get GFAC status and message
632 // function get_gfac_message() also sets global $status
633 $gfac_message = get_gfac_message( $gfacID );
634 if ( $gfac_message === false ) $gfac_message = "Job Finished";
635
636 // Create a status to put in the subject line
637 switch ( $status )
638 {
639 case "COMPLETE":
640 $subj_status = 'completed';
641 break;
642
643 case "CANCELLED":
644 case "CANCELED":
645 $subj_status = 'canceled';
646 break;
647
648 case "FAILED":
649 $subj_status = 'failed';
650 if ( preg_match( "/^US3-AIRA/i", $gfacID ) )
651 { // For A/Thrift FAIL, get error message
652 $gfac_message = getExperimentErrors( $gfacID );
653//$gfac_message .= "Test ERROR MESSAGE";
654 }
655 break;
656
657 case "ERROR":
658 $subj_status = 'unknown error';
659 break;
660
661 default:
662 $subj_status = $status; // For now
663 break;
664
665 }
666
667 $queuestatus = $subj_status;
668 $limshost = $dbhost;
669 if ( $limshost == 'localhost' )
670 $limshost = gethostname();
671
672 // Parse the editXMLFilename
673 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
674 explode( ".", $editXMLFilename );
675
676 $headers = "From: $org_name Admin<$admin_email>" . "\n";
677 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
678
679 // Set the reply address
680 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
681 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
682
683 // Try to avoid spam filters
684 $now = time();
685 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
686 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
687 $headers .= "MIME-Version: 1.0" . "\n";
688 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
689
690 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
691 $message = "
692 Your UltraScan job is complete:
693
694 Submission Time : $submittime
695 LIMS Host : $limshost
696 Analysis ID : $gfacID
697 Request ID : $requestID ( $db )
698 RunID : $runID
699 EditID : $editID
700 Data Type : $dataType
701 Cell/Channel/Wl : $cell / $channel / $wl
702 Status : $queuestatus
703 Cluster : $cluster
704 Job Type : $jobtype
705 GFAC Status : $status
706 GFAC Message : $gfac_message
707 Stdout : $stdout
708 ";
709
710 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
711
712 // Handle the error case where an error occurs before fetching the
713 // user's email address
714 if ( $email_address == "" ) $email_address = $admin_email;
715
716 mail( $email_address, $subject, $message, $headers );
717}
718
719function parse_xml( $xml, $type )
720{
721 $parser = new XMLReader();
722 $parser->xml( $xml );
723
724 $results = array();
725
726 while ( $parser->read() )
727 {
728 if ( $parser->name == $type )
729 {
730 while ( $parser->moveToNextAttribute() )
731 {
732 $results[ $parser->name ] = $parser->value;
733 }
734
735 break;
736 }
737 }
738
739 $parser->close();
740 return $results;
741}
742
743// Function to get information about the current job GFAC
744function get_gfac_message( $gfacID )
745{
746 global $serviceURL;
747 global $me;
748
749 $hex = "[0-9a-fA-F]";
750 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
751 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
752 {
753 // Then it's not a GFAC job
754 return false;
755 }
756
757 $url = "$serviceURL/jobstatus/$gfacID";
758 try
759 {
760 $post = new HttpRequest( $url, HttpRequest::METH_GET );
761 $http = $post->send();
762 $xml = $post->getResponseBody();
763 }
764 catch ( HttpException $e )
765 {
766 write_log( "$me: Job status not available - $gfacID" );
767 return false;
768 }
769
770 // Parse the result
771 $gfac_message = parse_message( $xml );
772
773 return $gfac_message;
774}
775
776function parse_message( $xml )
777{
778 global $status;
779 $status = "";
780 $gfac_message = "";
781
782 $parser = new XMLReader();
783 $parser->xml( $xml );
784
785 $results = array();
786
787 while( $parser->read() )
788 {
789 $type = $parser->nodeType;
790
791 if ( $type == XMLReader::ELEMENT )
792 $name = $parser->name;
793
794 else if ( $type == XMLReader::TEXT )
795 {
796 if ( $name == "status" )
797 $status = $parser->value;
798 else
799 $gfac_message = $parser->value;
800 }
801 }
802
803 $parser->close();
804 return $gfac_message;
805}
806
807function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
808{
809 global $work;
810 global $work_remote;
811 global $me;
812 global $db;
813 global $status;
814
815 // Figure out local working directory
816 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
817 $pwd = chdir( "$work/$gfacID" );
818
819 // Figure out remote directory
820 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
821
822 // Get stdout, stderr, output/analysis-results.tar
823 $output = array();
824// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
825//
826// exec( $cmd, $output, $stat );
827// if ( $stat != 0 )
828// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
829//
830// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
831// exec( $cmd, $output, $stat );
832// if ( $stat != 0 )
833// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
834
835 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
836 exec( $cmd, $output, $stat );
837 if ( $stat != 0 )
838 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
839
840 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
841 exec( $cmd, $output, $stat );
842 if ( $stat != 0 )
843 {
844 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
845 sleep( 10 );
846 write_log( "$me: RETRY" );
847 exec( $cmd, $output, $stat );
848 if ( $stat != 0 )
849 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
850 }
851
852 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
853 exec( $cmd, $output, $stat );
854 if ( $stat != 0 )
855 {
856 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
857 sleep( 10 );
858 write_log( "$me: RETRY" );
859 exec( $cmd, $output, $stat );
860 if ( $stat != 0 )
861 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
862 }
863
864 // Write the files to gfacDB
865
866 if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" );
867 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
868 if ( file_exists( "analysis-results.tar" ) )
869 $tarfile = file_get_contents( "analysis-results.tar" );
870
871 $query = "UPDATE analysis SET " .
872 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
873 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
874 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
875
876 $result = mysql_query( $query, $gfac_link );
877
878 if ( ! $result )
879 {
880 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
881 echo "Bad query\n";
882 return( -1 );
883 }
884}
885?>
Note: See TracBrowser for help on using the repository browser.