source: trunk/cleanup_aira.php@ 7

Last change on this file since 7 was 7, checked in by gegorbet, 10 years ago

airvata/thrift, us3-listen, update_notice changes
  • Property svn:keywords set to LastChangedDate Author
File size: 25.9 KB
RevLine 
[7]1<?php
2/*
3 * cleanup_aira.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$us3bin = exec( "ls -d ~us3/bin" );
11include_once "$us3bin/listen-config.php";
12include "/srv/www/htdocs/common/class/experiment_status.php";
13include "/srv/www/htdocs/common/class/experiment_errors.php";
14$me = 'cleanup_aira.php';
15$email_address = '';
16$queuestatus = '';
17$jobtype = '';
18$db = '';
19$editXMLFilename = '';
20$status = '';
21
22function aira_cleanup( $us3_db, $reqID, $gfac_link )
23{
24 global $dbhost;
25 global $user;
26 global $passwd;
27 global $db;
28 global $guser;
29 global $gpasswd;
30 global $gDB;
31 global $me;
32 global $work;
33 global $email_address;
34 global $queuestatus;
35 global $jobtype;
36 global $editXMLFilename;
37 global $submittime;
38 global $status;
39 global $stderr;
40 global $stdout;
41 global $tarfile;
42 global $requestID;
43 global $submit_dir;
44 $me = 'cleanup_aira.php';
45
46 $requestID = $reqID;
47 $db = $us3_db;
48 write_log( "$me: debug db=$db; requestID=$requestID" );
49
50 $us3_link = mysql_connect( $dbhost, $user, $passwd );
51
52 if ( ! $us3_link )
53 {
54 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
55 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
56 return( -1 );
57 }
58
59 $result = mysql_select_db( $db, $us3_link );
60
61 if ( ! $result )
62 {
63 write_log( "$me: could not select DB $db" );
64 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
65 return( -1 );
66 }
67
68 // First get basic info for email messages
69 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
70 "WHERE HPCAnalysisRequestID=$requestID";
71 $result = mysql_query( $query, $us3_link );
72
73 if ( ! $result )
74 {
75 write_log( "$me: Bad query: $query" );
76 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
77 return( -1 );
78 }
79
80 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
81
82 $query = "SELECT personID FROM people " .
83 "WHERE personGUID='$investigatorGUID'";
84 $result = mysql_query( $query, $us3_link );
85
86 list( $personID ) = mysql_fetch_array( $result );
87
88 $query = "SELECT clusterName, submitTime, queueStatus, method " .
89 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
90 "WHERE h.HPCAnalysisRequestID=$requestID " .
91 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
92
93 $result = mysql_query( $query, $us3_link );
94
95 if ( ! $result )
96 {
97 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
98 return( -1 );
99 }
100
101 if ( mysql_num_rows( $result ) == 0 )
102 {
103 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
104 return( -1 );
105 }
106
107 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
108
109 // Get the GFAC ID
110 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
111 "WHERE HPCAnalysisRequestID=$requestID";
112
113 $result = mysql_query( $query, $us3_link );
114
115 if ( ! $result )
116 {
117 write_log( "$me: Bad query: $query" );
118 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
119 return( -1 );
120 }
121
122 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
123
124 // Get data from global GFAC DB then insert it into US3 DB
125
126 $result = mysql_select_db( $gDB, $gfac_link );
127
128 if ( ! $result )
129 {
130 write_log( "$me: Could not connect to DB $gDB" );
131 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
132 return( -1 );
133 }
134
135 $query = "SELECT status, cluster, id FROM analysis " .
136 "WHERE gfacID='$gfacID'";
137
138 $result = mysql_query( $query, $gfac_link );
139 if ( ! $result )
140 {
141 write_log( "$me: Could not select GFAC status for $gfacID" );
142 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
143 return( -1 );
144 }
145
146 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
147
148 if ( $cluster == 'bcf-local' || $cluster == 'alamo-local' )
149 {
150 $clushost = $cluster;
151 $clushost = preg_replace( "/\-local/", "", $clushost );
152 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
153 }
154
155
156 $query = "SELECT id FROM analysis " .
157 "WHERE gfacID='$gfacID'";
158
159 $result = mysql_query( $query, $gfac_link );
160
161 if ( ! $result )
162 {
163 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
164 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
165 return( -1 );
166 }
167
168 list( $analysisID ) = mysql_fetch_array( $result );
169
170 // Get the request guid (LIMS submit dir name)
171 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
172 "WHERE HPCAnalysisRequestID = $requestID ";
173 $result = mysql_query( $query, $us3_link );
174
175 if ( ! $result )
176 {
177 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
178 }
179
180 list( $requestGUID ) = mysql_fetch_array( $result );
181 $output_dir = "$submit_dir/$requestGUID";
182
183 // Get stderr,stdout,tarfile from work directory
184 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
185 chdir( "$output_dir" );
186//write_log( "$me: gfacID=$gfacID" );
187//write_log( "$me: submit_dir=$submit_dir" );
188//write_log( "$me: requestGUID=$requestGUID" );
189write_log( "$me: output_dir=$output_dir" );
190
191 $stderr = "";
192 $stdout = "";
193 $tarfile = "";
194 $fn_stderr = "Ultrascan.stderr";
195 $fn_stdout = "Ultrascan.stdout";
196 $fn_tarfile = "analysis-results.tar";
197 $num_try = 0;
198 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
199 {
200 sleep( 10 );
201 $num_try++;
202 }
203
204 $ofiles = scandir( $output_dir );
205 foreach ( $ofiles as $ofile )
206 {
207 if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
208 $fn_stderr = $ofile;
209 if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
210 $fn_stdout = $ofile;
211//write_log( "$me: ofile=$ofile" );
212 }
213write_log( "$me: fn_stderr=$fn_stderr" );
214write_log( "$me: fn_stdout=$fn_stdout" );
215if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
216else write_log( "$me: NOT FOUND: $fn_tarfile" );
217
218 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
219 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
220 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
221
222 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
223 { // Filter "ipath_userinit" lines out of alamo stdout lines
224 $prefln = strlen( $stdout );
225 $output = array();
226 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
227 $stdout = implode( "\n", $output );
228 $posfln = strlen( $stdout );
229write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
230 }
231
232 // Save queue messages for post-mortem analysis
233 $query = "SELECT message, time FROM queue_messages " .
234 "WHERE analysisID = $analysisID " .
235 "ORDER BY time ";
236 $result = mysql_query( $query, $gfac_link );
237
238 if ( ! $result )
239 {
240 // Just log it and continue
241 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
242 }
243
244 $now = date( 'Y-m-d H:i:s' );
245 $message_log = "US3 DB: $db\n" .
246 "RequestID: $requestID\n" .
247 "GFAC ID: $gfacID\n" .
248 "Processed: $now\n\n" .
249 "Queue Messages\n\n" ;
250 if ( mysql_num_rows( $result ) > 0 )
251 {
252 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
253 $message_log .= "$time $message\n";
254 }
255
256 $query = "DELETE FROM queue_messages " .
257 "WHERE analysisID = $analysisID ";
258
259 $result = mysql_query( $query, $gfac_link );
260
261 if ( ! $result )
262 {
263 // Just log it and continue
264 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
265 }
266
267 $query = "SELECT queue_msg FROM analysis " .
268 "WHERE gfacID='$gfacID' ";
269
270 $result = mysql_query( $query, $gfac_link );
271 list( $queue_msg ) = mysql_fetch_array( $result );
272
273 // But let's allow for investigation of other large stdout and/or stderr
274 if ( strlen( $stdout ) > 20480000 ||
275 strlen( $stderr ) > 20480000 )
276 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
277
278 $message_log .= "\n\n\nStdout Contents\n\n" .
279 $stdout .
280 "\n\n\nStderr Contents\n\n" .
281 $stderr .
282 "\n\n\nGFAC Status: $status\n" .
283 "GFAC message field: $queue_msg\n";
284
285 // Delete data from GFAC DB
286 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
287
288 $result = mysql_query( $query, $gfac_link );
289
290 if ( ! $result )
291 {
292 // Just log it and continue
293 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
294 }
295
296
297 // Try to create it if necessary, and write the file
298 // Let's use FILE_APPEND, in case this is the second time around and the
299 // GFAC job status was INSERTed, rather than UPDATEd
300 if ( ! is_dir( $output_dir ) )
301 mkdir( $output_dir, 0775, true );
302 $message_filename = "$output_dir/$db-$requestID-messages.txt";
303 file_put_contents( $message_filename, $message_log, FILE_APPEND );
304 // mysql_close( $gfac_link );
305
306 /////////
307 // Insert data into HPCAnalysis
308
309 $query = "UPDATE HPCAnalysisResult SET " .
310 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
311 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
312 "queueStatus='completed' " .
313 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
314
315 $result = mysql_query( $query, $us3_link );
316
317 if ( ! $result )
318 {
319 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
320 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
321 return( -1 );
322 }
323
324 // Delete data from GFAC DB
325 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
326
327 $result = mysql_query( $query, $gfac_link );
328
329 if ( ! $result )
330 {
331 // Just log it and continue
332 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
333 }
334
335 // Expand the tar file
336
337 if ( strlen( $tarfile ) == 0 )
338 {
339 write_log( "$me: No tarfile" );
340 mail_to_user( "fail", "No results" );
341 return( -1 );
342 }
343
344 $tar_out = array();
345 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
346
347 // Insert the model files and noise files
348 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
349 $noiseIDs = array();
350 $modelGUIDs = array();
351
352 foreach ( $files as $file )
353 {
354 $split = explode( ";", $file );
355
356 if ( count( $split ) > 1 )
357 {
358 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
359
360 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
361 list( $other, $variance ) = explode( "=", $variance );
362 list( $other, $meniscus ) = explode( "=", $meniscus );
363 }
364 else
365 $fn = $file;
366
367 if ( preg_match( "/mdl.tmp$/", $fn ) )
368 continue;
369
370 if ( filesize( $fn ) < 100 )
371 {
372 write_log( "$me:fn is invalid $fn size filesize($fn)" );
373 mail_to_user( "fail", "Internal error\n$fn is invalid" );
374 return( -1 );
375 }
376
377 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
378 {
379 $xml = file_get_contents( $fn );
380 $statistics = parse_xml( $xml, 'statistics' );
381// $ntries = 0;
382//
383// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
384// { // job_statistics file not totally copied, so retry
385// sleep( 10 );
386// $xml = file_get_contents( $fn );
387// $statistics = parse_xml( $xml, 'statistics' );
388// $ntries++;
389//write_log( "$me:jobstats retry $ntries" );
390// }
391//write_log( "$me:cputime=$statistics['cputime']" );
392
393 $otherdata = parse_xml( $xml, 'id' );
394
395 $query = "UPDATE HPCAnalysisResult SET " .
396 "wallTime = {$statistics['walltime']}, " .
397 "CPUTime = {$statistics['cputime']}, " .
398 "CPUCount = {$statistics['cpucount']}, " .
399 "max_rss = {$statistics['maxmemory']}, " .
400 "startTime = '{$otherdata['starttime']}', " .
401 "endTime = '{$otherdata['endtime']}', " .
402 "mgroupcount = {$otherdata['groupcount']} " .
403 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
404 $result = mysql_query( $query, $us3_link );
405
406 if ( ! $result )
407 {
408 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
409 }
410
411 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
412
413 }
414
415 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
416 {
417 $xml = file_get_contents( $fn );
418 $noise_data = parse_xml( $xml, "noise" );
419 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
420 $desc = $noise_data[ 'description' ];
421 $modelGUID = $noise_data[ 'modelGUID' ];
422 $noiseGUID = $noise_data[ 'noiseGUID' ];
423
424 $query = "INSERT INTO noise SET " .
425 "noiseGUID='$noiseGUID'," .
426 "modelGUID='$modelGUID'," .
427 "editedDataID=1, " .
428 "modelID=1, " .
429 "noiseType='$type'," .
430 "description='$desc'," .
431 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
432
433 // Add later after all files are processed: editDataID, modelID
434
435 $result = mysql_query( $query, $us3_link );
436
437 if ( ! $result )
438 {
439 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
440 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
441 return( -1 );
442 }
443
444 $id = mysql_insert_id( $us3_link );
445 $file_type = "noise";
446 $noiseIDs[] = $id;
447
448 // Keep track of modelGUIDs for later, when we replace them
449 $modelGUIDs[ $id ] = $modelGUID;
450
451 }
452 else // It's a model file
453 {
454 $xml = file_get_contents( $fn );
455 $model_data = parse_xml( $xml, "model" );
456 $description = $model_data[ 'description' ];
457 $modelGUID = $model_data[ 'modelGUID' ];
458 $editGUID = $model_data[ 'editGUID' ];
459
460 if ( $mc_iteration > 1 )
461 {
462 $miter = sprintf( "_mcN%03d", $mc_iteration );
463 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
464write_log( "$me: MODELUpd: O:description=$description" );
465 }
466
467 $query = "INSERT INTO model SET " .
468 "modelGUID='$modelGUID'," .
469 "editedDataID=" .
470 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
471 "description='$description'," .
472 "MCIteration='$mc_iteration'," .
473 "meniscus='$meniscus'," .
474 "variance='$variance'," .
475 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
476
477 $result = mysql_query( $query, $us3_link );
478
479 if ( ! $result )
480 {
481 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
482 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
483 return( -1 );
484 }
485
486 $modelID = mysql_insert_id( $us3_link );
487 $id = $modelID;
488 $file_type = "model";
489
490 $query = "INSERT INTO modelPerson SET " .
491 "modelID=$modelID, personID=$personID";
492 $result = mysql_query( $query, $us3_link );
493 }
494
495 $query = "INSERT INTO HPCAnalysisResultData SET " .
496 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
497 "HPCAnalysisResultType='$file_type', " .
498 "resultID=$id";
499
500 $result = mysql_query( $query, $us3_link );
501
502 if ( ! $result )
503 {
504 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
505 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
506 return( -1 );
507 }
508 }
509
510 // Now fix up noise entries
511 // For noise files, there is, at most two: ti_noise and ri_noise
512 // In this case there will only be one modelID
513
514 foreach ( $noiseIDs as $noiseID )
515 {
516 $modelGUID = $modelGUIDs[ $noiseID ];
517 $query = "UPDATE noise SET " .
518 "editedDataID=" .
519 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
520 "modelID=" .
521 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
522 "WHERE noiseID=$noiseID";
523
524 $result = mysql_query( $query, $us3_link );
525
526 if ( ! $result )
527 {
528 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
529 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
530 return( -1 );
531 }
532 }
533
534 // Copy results to LIMS submit directory (files there are deleted after 7 days)
535 global $submit_dir; // LIMS submit files dir
536
537 // Get the request guid (LIMS submit dir name)
538 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
539 "WHERE HPCAnalysisRequestID = $requestID ";
540 $result = mysql_query( $query, $us3_link );
541
542 if ( ! $result )
543 {
544 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
545 }
546
547// list( $requestGUID ) = mysql_fetch_array( $result );
548//
549// chdir( "$submit_dir/$requestGUID" );
550// $f = fopen( "analysis-results.tar", "w" );
551// fwrite( $f, $tarfile );
552// fclose( $f );
553
554 // Clean up
555// chdir ( $work );
556 // exec( "rm -rf $gfacID" );
557
558 mysql_close( $us3_link );
559
560 /////////
561 // Send email
562
563 mail_to_user( "success", "" );
564}
565
566function mail_to_user( $type, $msg )
567{
568 // Note to me. Just changed subject line to include a modified $status instead
569 // of the $type variable passed. More informative than just "fail" or "success."
570 // See how it works for awhile and then consider removing $type parameter from
571 // function.
572 global $email_address;
573 global $submittime;
574 global $queuestatus;
575 global $status;
576 global $cluster;
577 global $jobtype;
578 global $org_name;
579 global $admin_email;
580 global $db;
581 global $dbhost;
582 global $requestID;
583 global $gfacID;
584 global $editXMLFilename;
585 global $stdout;
586
587global $me;
588write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
589
590 // Get GFAC status and message
591 // function get_gfac_message() also sets global $status
592 $gfac_message = get_gfac_message( $gfacID );
593 if ( $gfac_message === false ) $gfac_message = "Job Finished";
594
595 // Create a status to put in the subject line
596 switch ( $status )
597 {
598 case "COMPLETE":
599 $subj_status = 'completed';
600 break;
601
602 case "CANCELLED":
603 case "CANCELED":
604 $subj_status = 'canceled';
605 break;
606
607 case "FAILED":
608 $subj_status = 'failed';
609 if ( preg_match( "/^US3-AIRA/i", $gfacID ) )
610 { // For A/Thrift FAIL, get error message
611 $gfac_message = getExperimentErrors( $gfacID );
612//$gfac_message .= "Test ERROR MESSAGE";
613 }
614 break;
615
616 case "ERROR":
617 $subj_status = 'unknown error';
618 break;
619
620 default:
621 $subj_status = $status; // For now
622 break;
623
624 }
625
626 $queuestatus = $subj_status;
627 $limshost = $dbhost;
628 if ( $limshost == 'localhost' )
629 $limshost = gethostname();
630
631 // Parse the editXMLFilename
632 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
633 explode( ".", $editXMLFilename );
634
635 $headers = "From: $org_name Admin<$admin_email>" . "\n";
636 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
637
638 // Set the reply address
639 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
640 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
641
642 // Try to avoid spam filters
643 $now = time();
644 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
645 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
646 $headers .= "MIME-Version: 1.0" . "\n";
647 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
648
649 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
650 $message = "
651 Your UltraScan job is complete:
652
653 Submission Time : $submittime
654 LIMS Host : $limshost
655 Analysis ID : $gfacID
656 Request ID : $requestID ( $db )
657 RunID : $runID
658 EditID : $editID
659 Data Type : $dataType
660 Cell/Channel/Wl : $cell / $channel / $wl
661 Status : $queuestatus
662 Cluster : $cluster
663 Job Type : $jobtype
664 GFAC Status : $status
665 GFAC Message : $gfac_message
666 Stdout : $stdout
667 ";
668
669 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
670
671 // Handle the error case where an error occurs before fetching the
672 // user's email address
673 if ( $email_address == "" ) $email_address = $admin_email;
674
675 mail( $email_address, $subject, $message, $headers );
676}
677
678function parse_xml( $xml, $type )
679{
680 $parser = new XMLReader();
681 $parser->xml( $xml );
682
683 $results = array();
684
685 while ( $parser->read() )
686 {
687 if ( $parser->name == $type )
688 {
689 while ( $parser->moveToNextAttribute() )
690 {
691 $results[ $parser->name ] = $parser->value;
692 }
693
694 break;
695 }
696 }
697
698 $parser->close();
699 return $results;
700}
701
702// Function to get information about the current job GFAC
703function get_gfac_message( $gfacID )
704{
705 global $serviceURL;
706 global $me;
707
708 $hex = "[0-9a-fA-F]";
709 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
710 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
711 {
712 // Then it's not a GFAC job
713 return false;
714 }
715
716 $url = "$serviceURL/jobstatus/$gfacID";
717 try
718 {
719 $post = new HttpRequest( $url, HttpRequest::METH_GET );
720 $http = $post->send();
721 $xml = $post->getResponseBody();
722 }
723 catch ( HttpException $e )
724 {
725 write_log( "$me: Job status not available - $gfacID" );
726 return false;
727 }
728
729 // Parse the result
730 $gfac_message = parse_message( $xml );
731
732 return $gfac_message;
733}
734
735function parse_message( $xml )
736{
737 global $status;
738 $status = "";
739 $gfac_message = "";
740
741 $parser = new XMLReader();
742 $parser->xml( $xml );
743
744 $results = array();
745
746 while( $parser->read() )
747 {
748 $type = $parser->nodeType;
749
750 if ( $type == XMLReader::ELEMENT )
751 $name = $parser->name;
752
753 else if ( $type == XMLReader::TEXT )
754 {
755 if ( $name == "status" )
756 $status = $parser->value;
757 else
758 $gfac_message = $parser->value;
759 }
760 }
761
762 $parser->close();
763 return $gfac_message;
764}
765
766function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
767{
768 global $work;
769 global $work_remote;
770 global $me;
771 global $db;
772 global $status;
773
774 // Figure out local working directory
775 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
776 $pwd = chdir( "$work/$gfacID" );
777
778 // Figure out remote directory
779 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
780
781 // Get stdout, stderr, output/analysis-results.tar
782 $output = array();
783// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
784//
785// exec( $cmd, $output, $stat );
786// if ( $stat != 0 )
787// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
788//
789// $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
790// exec( $cmd, $output, $stat );
791// if ( $stat != 0 )
792// write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
793
794 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
795 exec( $cmd, $output, $stat );
796 if ( $stat != 0 )
797 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
798
799 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
800 exec( $cmd, $output, $stat );
801 if ( $stat != 0 )
802 {
803 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
804 sleep( 10 );
805 write_log( "$me: RETRY" );
806 exec( $cmd, $output, $stat );
807 if ( $stat != 0 )
808 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
809 }
810
811 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
812 exec( $cmd, $output, $stat );
813 if ( $stat != 0 )
814 {
815 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
816 sleep( 10 );
817 write_log( "$me: RETRY" );
818 exec( $cmd, $output, $stat );
819 if ( $stat != 0 )
820 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
821 }
822
823 // Write the files to gfacDB
824
825 if ( file_exists( "stderr" ) ) $stderr = file_get_contents( "stderr" );
826 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
827 if ( file_exists( "analysis-results.tar" ) )
828 $tarfile = file_get_contents( "analysis-results.tar" );
829
830 $query = "UPDATE analysis SET " .
831 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
832 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
833 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
834
835 $result = mysql_query( $query, $gfac_link );
836
837 if ( ! $result )
838 {
839 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
840 echo "Bad query\n";
841 return( -1 );
842 }
843}
844?>
Note: See TracBrowser for help on using the repository browser.