source: trunk/cleanup.php@ 25

Last change on this file since 25 was 25, checked in by gegorbet, 7 years ago

merge us3iab/airavata gridctl code
File size: 30.0 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $org_domain;
20 global $dbhost;
21 global $user;
22 global $passwd;
23 global $db;
24 global $guser;
25 global $gpasswd;
26 global $gDB;
27 global $me;
28 global $work;
29 global $email_address;
30 global $queuestatus;
31 global $jobtype;
32 global $editXMLFilename;
33 global $submittime;
34 global $status;
35 global $stderr;
36 global $stdout;
37 global $tarfile;
38 global $requestID;
39 global $submit_dir;
40 $me = 'cleanup_aira.php';
41
42 $requestID = $reqID;
43 $db = $us3_db;
44 write_log( "$me: debug db=$db; requestID=$requestID" );
45
46 $us3_link = mysql_connect( $dbhost, $user, $passwd );
47
48 if ( ! $us3_link )
49 {
50 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
51 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
52 return( -1 );
53 }
54
55 $result = mysql_select_db( $db, $us3_link );
56
57 if ( ! $result )
58 {
59 write_log( "$me: could not select DB $db" );
60 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
61 return( -1 );
62 }
63
64 // First get basic info for email messages
65 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
66 "WHERE HPCAnalysisRequestID=$requestID";
67 $result = mysql_query( $query, $us3_link );
68
69 if ( ! $result )
70 {
71 write_log( "$me: Bad query: $query" );
72 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
73 return( -1 );
74 }
75
76 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
77
78 $query = "SELECT personID FROM people " .
79 "WHERE personGUID='$investigatorGUID'";
80 $result = mysql_query( $query, $us3_link );
81
82 list( $personID ) = mysql_fetch_array( $result );
83
84 $query = "SELECT clusterName, submitTime, queueStatus, method " .
85 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
86 "WHERE h.HPCAnalysisRequestID=$requestID " .
87 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
88
89 $result = mysql_query( $query, $us3_link );
90
91 if ( ! $result )
92 {
93 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
94 return( -1 );
95 }
96
97 if ( mysql_num_rows( $result ) == 0 )
98 {
99 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
100 return( -1 );
101 }
102
103 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
104
105 // Get the GFAC ID
106 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
107 "WHERE HPCAnalysisRequestID=$requestID";
108
109 $result = mysql_query( $query, $us3_link );
110
111 if ( ! $result )
112 {
113 write_log( "$me: Bad query: $query" );
114 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
115 return( -1 );
116 }
117
118 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
119
120 // Get data from global GFAC DB then insert it into US3 DB
121
122 $result = mysql_select_db( $gDB, $gfac_link );
123
124 if ( ! $result )
125 {
126 write_log( "$me: Could not connect to DB $gDB" );
127 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
128 return( -1 );
129 }
130
131 $query = "SELECT status, cluster, id FROM analysis " .
132 "WHERE gfacID='$gfacID'";
133
134 $result = mysql_query( $query, $gfac_link );
135 if ( ! $result )
136 {
137 write_log( "$me: Could not select GFAC status for $gfacID" );
138 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
139 return( -1 );
140 }
141
142 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
143
144 $is_us3iab = preg_match( "/us3iab/", $cluster );
145 $is_local = preg_match( "/-local/", $cluster );
146
147 if ( $is_us3iab || $is_local )
148 {
149 $clushost = $cluster;
150 $clushost = preg_replace( "/\-local/", "", $clushost );
151 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
152 }
153
154
155 $query = "SELECT id FROM analysis " .
156 "WHERE gfacID='$gfacID'";
157
158 $result = mysql_query( $query, $gfac_link );
159
160 if ( ! $result )
161 {
162 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
163 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
164 return( -1 );
165 }
166
167 list( $analysisID ) = mysql_fetch_array( $result );
168
169 // Get the request guid (LIMS submit dir name)
170 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
171 "WHERE HPCAnalysisRequestID = $requestID ";
172 $result = mysql_query( $query, $us3_link );
173
174 if ( ! $result )
175 {
176 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
177 }
178
179 list( $requestGUID ) = mysql_fetch_array( $result );
180 $output_dir = "$submit_dir/$requestGUID";
181
182 // Get stderr,stdout,tarfile from work directory
183 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
184 chdir( "$output_dir" );
185//write_log( "$me: gfacID=$gfacID" );
186//write_log( "$me: submit_dir=$submit_dir" );
187//write_log( "$me: requestGUID=$requestGUID" );
188write_log( "$me: output_dir=$output_dir" );
189
190 $stderr = "";
191 $stdout = "";
192 $tarfile = "";
193 $fn_stderr = "Ultrascan.stderr";
194 $fn_stdout = "Ultrascan.stdout";
195 $fn_tarfile = "analysis-results.tar";
196 $num_try = 0;
197$len1 = strlen( $fn_tarfile );
198write_log( "$me: fn_tarfile=$fn_tarfile len(tar)=$len1" );
199 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
200 {
201 sleep( 10 );
202 $num_try++;
203write_log( "$me: tar-exists: num_try=$num_try" );
204 }
205
206 $ofiles = scandir( $output_dir );
207 foreach ( $ofiles as $ofile )
208 {
209 if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
210 $fn_stderr = $ofile;
211 if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
212 $fn_stdout = $ofile;
213//write_log( "$me: ofile=$ofile" );
214 }
215write_log( "$me: fn_stderr=$fn_stderr" );
216write_log( "$me: fn_stdout=$fn_stdout" );
217if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
218else write_log( "$me: NOT FOUND: $fn_tarfile" );
219
220 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
221 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
222 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
223
224 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
225 { // Filter "ipath_userinit" lines out of alamo stdout lines
226 $prefln = strlen( $stdout );
227 $output = array();
228 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
229 $stdout = implode( "\n", $output );
230 $posfln = strlen( $stdout );
231write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
232 }
233
234 // Save queue messages for post-mortem analysis
235 $query = "SELECT message, time FROM queue_messages " .
236 "WHERE analysisID = $analysisID " .
237 "ORDER BY time ";
238 $result = mysql_query( $query, $gfac_link );
239
240 if ( ! $result )
241 {
242 // Just log it and continue
243 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
244 }
245
246 $now = date( 'Y-m-d H:i:s' );
247 $message_log = "US3 DB: $db\n" .
248 "RequestID: $requestID\n" .
249 "GFAC ID: $gfacID\n" .
250 "Processed: $now\n\n" .
251 "Queue Messages\n\n" ;
252 if ( mysql_num_rows( $result ) > 0 )
253 {
254 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
255 $message_log .= "$time $message\n";
256 }
257
258 $query = "DELETE FROM queue_messages " .
259 "WHERE analysisID = $analysisID ";
260
261 $result = mysql_query( $query, $gfac_link );
262
263 if ( ! $result )
264 {
265 // Just log it and continue
266 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
267 }
268
269 $query = "SELECT queue_msg FROM analysis " .
270 "WHERE gfacID='$gfacID' ";
271
272 $result = mysql_query( $query, $gfac_link );
273 list( $queue_msg ) = mysql_fetch_array( $result );
274
275 // But let's allow for investigation of other large stdout and/or stderr
276 if ( strlen( $stdout ) > 20480000 ||
277 strlen( $stderr ) > 20480000 )
278 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
279
280 $message_log .= "\n\n\nStdout Contents\n\n" .
281 $stdout .
282 "\n\n\nStderr Contents\n\n" .
283 $stderr .
284 "\n\n\nGFAC Status: $status\n" .
285 "GFAC message field: $queue_msg\n";
286
287 // Delete data from GFAC DB
288 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
289
290 $result = mysql_query( $query, $gfac_link );
291
292 if ( ! $result )
293 {
294 // Just log it and continue
295 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
296 }
297
298
299 // Try to create it if necessary, and write the file
300 // Let's use FILE_APPEND, in case this is the second time around and the
301 // GFAC job status was INSERTed, rather than UPDATEd
302 if ( ! is_dir( $output_dir ) )
303 mkdir( $output_dir, 0775, true );
304 $message_filename = "$output_dir/$db-$requestID-messages.txt";
305 file_put_contents( $message_filename, $message_log, FILE_APPEND );
306 // mysql_close( $gfac_link );
307
308 /////////
309 // Insert data into HPCAnalysis
310
311 $query = "UPDATE HPCAnalysisResult SET " .
312 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
313 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
314 "queueStatus='completed' " .
315 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
316
317 $result = mysql_query( $query, $us3_link );
318
319 if ( ! $result )
320 {
321 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
322 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
323 return( -1 );
324 }
325
326 // Delete data from GFAC DB
327 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
328
329 $result = mysql_query( $query, $gfac_link );
330
331 if ( ! $result )
332 {
333 // Just log it and continue
334 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
335 }
336
337 // Expand the tar file
338
339 if ( strlen( $tarfile ) == 0 )
340 {
341 write_log( "$me: No tarfile" );
342 mail_to_user( "fail", "No results" );
343 return( -1 );
344 }
345
346 $tar_out = array();
347 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
348
349 // Insert the model files and noise files
350 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
351 $noiseIDs = array();
352 $modelGUIDs = array();
353 $mrecsIDs = array();
354 $fns_used = array();
355
356 foreach ( $files as $file )
357 {
358 $split = explode( ";", $file );
359
360 if ( count( $split ) > 1 )
361 {
362 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
363
364 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
365 list( $other, $variance ) = explode( "=", $variance );
366 list( $other, $meniscus ) = explode( "=", $meniscus );
367 }
368 else
369 $fn = $file;
370
371 if ( preg_match( "/mdl.tmp$/", $fn ) )
372 continue;
373
374 if ( in_array( $fn, $fns_used ) )
375 continue;
376
377 $fns_used[] = $fn;
378
379 if ( filesize( $fn ) < 100 )
380 {
381 write_log( "$me:fn is invalid $fn size filesize($fn)" );
382 mail_to_user( "fail", "Internal error\n$fn is invalid" );
383 return( -1 );
384 }
385
386 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
387 {
388 $xml = file_get_contents( $fn );
389 $statistics = parse_xml( $xml, 'statistics' );
390// $ntries = 0;
391//
392// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
393// { // job_statistics file not totally copied, so retry
394// sleep( 10 );
395// $xml = file_get_contents( $fn );
396// $statistics = parse_xml( $xml, 'statistics' );
397// $ntries++;
398//write_log( "$me:jobstats retry $ntries" );
399// }
400//write_log( "$me:cputime=$statistics['cputime']" );
401
402 $otherdata = parse_xml( $xml, 'id' );
403
404 $query = "UPDATE HPCAnalysisResult SET " .
405 "wallTime = {$statistics['walltime']}, " .
406 "CPUTime = {$statistics['cputime']}, " .
407 "CPUCount = {$statistics['cpucount']}, " .
408 "max_rss = {$statistics['maxmemory']}, " .
409 "startTime = '{$otherdata['starttime']}', " .
410 "endTime = '{$otherdata['endtime']}', " .
411 "mgroupcount = {$otherdata['groupcount']} " .
412 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
413 $result = mysql_query( $query, $us3_link );
414
415 if ( ! $result )
416 {
417 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
418 }
419
420 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
421
422 $file_type = "job_stats";
423 $id = 1;
424 }
425
426 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
427 {
428 $xml = file_get_contents( $fn );
429 $noise_data = parse_xml( $xml, "noise" );
430 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
431 $desc = $noise_data[ 'description' ];
432 $modelGUID = $noise_data[ 'modelGUID' ];
433 $noiseGUID = $noise_data[ 'noiseGUID' ];
434
435 $query = "INSERT INTO noise SET " .
436 "noiseGUID='$noiseGUID'," .
437 "modelGUID='$modelGUID'," .
438 "editedDataID=1, " .
439 "modelID=1, " .
440 "noiseType='$type'," .
441 "description='$desc'," .
442 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
443
444 // Add later after all files are processed: editDataID, modelID
445
446 $result = mysql_query( $query, $us3_link );
447
448 if ( ! $result )
449 {
450 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
451 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
452 return( -1 );
453 }
454
455 $id = mysql_insert_id( $us3_link );
456 $file_type = "noise";
457 $noiseIDs[] = $id;
458
459 // Keep track of modelGUIDs for later, when we replace them
460 $modelGUIDs[ $id ] = $modelGUID;
461
462 }
463
464 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
465 {
466 $xml = file_get_contents( $fn );
467 $mrecs_data = parse_xml( $xml, "modelrecords" );
468 $desc = $mrecs_data[ 'description' ];
469 $editGUID = $mrecs_data[ 'editGUID' ];
470write_log( "$me: mrecs file editGUID=$editGUID" );
471 if ( strlen( $editGUID ) < 36 )
472 $editGUID = "12345678-0123-5678-0123-567890123456";
473 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
474 $modelGUID = $mrecs_data[ 'modelGUID' ];
475
476 $query = "INSERT INTO pcsa_modelrecs SET " .
477 "editedDataID=" .
478 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
479 "modelID=0, " .
480 "mrecsGUID='$mrecGUID'," .
481 "description='$desc'," .
482 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
483
484 // Add later after all files are processed: editDataID, modelID
485
486 $result = mysql_query( $query, $us3_link );
487
488 if ( ! $result )
489 {
490 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
491 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
492 return( -1 );
493 }
494
495 $id = mysql_insert_id( $us3_link );
496 $file_type = "mrecs";
497 $mrecsIDs[] = $id;
498
499 // Keep track of modelGUIDs for later, when we replace them
500 $rmodlGUIDs[ $id ] = $modelGUID;
501//write_log( "$me: mrecs file inserted into DB : id=$id" );
502 }
503
504 else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
505 {
506 $xml = file_get_contents( $fn );
507 $model_data = parse_xml( $xml, "model" );
508 $description = $model_data[ 'description' ];
509 $modelGUID = $model_data[ 'modelGUID' ];
510 $editGUID = $model_data[ 'editGUID' ];
511
512 if ( $mc_iteration > 1 )
513 {
514 $miter = sprintf( "_mcN%03d", $mc_iteration );
515 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
516write_log( "$me: MODELUpd: O:description=$description" );
517 }
518
519 $query = "INSERT INTO model SET " .
520 "modelGUID='$modelGUID'," .
521 "editedDataID=" .
522 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
523 "description='$description'," .
524 "MCIteration='$mc_iteration'," .
525 "meniscus='$meniscus'," .
526 "variance='$variance'," .
527 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
528
529 $result = mysql_query( $query, $us3_link );
530
531 if ( ! $result )
532 {
533 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
534 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
535 return( -1 );
536 }
537
538 $modelID = mysql_insert_id( $us3_link );
539 $id = $modelID;
540 $file_type = "model";
541
542 $query = "INSERT INTO modelPerson SET " .
543 "modelID=$modelID, personID=$personID";
544 $result = mysql_query( $query, $us3_link );
545 }
546
547 else // Undetermined type: skip result data update
548 continue;
549
550 $query = "INSERT INTO HPCAnalysisResultData SET " .
551 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
552 "HPCAnalysisResultType='$file_type', " .
553 "resultID=$id";
554
555 $result = mysql_query( $query, $us3_link );
556
557 if ( ! $result )
558 {
559 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
560 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
561 return( -1 );
562 }
563 }
564
565 // Now fix up noise entries
566 // For noise files, there is, at most two: ti_noise and ri_noise
567 // In this case there will only be one modelID
568
569 foreach ( $noiseIDs as $noiseID )
570 {
571 $modelGUID = $modelGUIDs[ $noiseID ];
572 $query = "UPDATE noise SET " .
573 "editedDataID=" .
574 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
575 "modelID=" .
576 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
577 "WHERE noiseID=$noiseID";
578
579 $result = mysql_query( $query, $us3_link );
580
581 if ( ! $result )
582 {
583 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
584 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
585 return( -1 );
586 }
587 }
588
589 // Now possibly fix up mrecs entries
590
591 foreach ( $mrecsIDs as $mrecsID )
592 {
593 $modelGUID = $rmodlGUIDs[ $mrecsID ];
594 $query = "UPDATE pcsa_modelrecs SET " .
595 "modelID=" .
596 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
597 "WHERE mrecsID=$mrecsID";
598
599 $result = mysql_query( $query, $us3_link );
600
601 if ( ! $result )
602 {
603 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
604 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
605 return( -1 );
606 }
607write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
608 }
609//write_log( "$me: mrecs entries updated" );
610
611 // Copy results to LIMS submit directory (files there are deleted after 7 days)
612 global $submit_dir; // LIMS submit files dir
613
614 // Get the request guid (LIMS submit dir name)
615 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
616 "WHERE HPCAnalysisRequestID = $requestID ";
617 $result = mysql_query( $query, $us3_link );
618
619 if ( ! $result )
620 {
621 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
622 }
623
624// list( $requestGUID ) = mysql_fetch_array( $result );
625//
626// chdir( "$submit_dir/$requestGUID" );
627// $f = fopen( "analysis-results.tar", "w" );
628// fwrite( $f, $tarfile );
629// fclose( $f );
630
631 // Clean up
632// chdir ( $work );
633 // exec( "rm -rf $gfacID" );
634
635 mysql_close( $us3_link );
636
637 /////////
638 // Send email
639
640 mail_to_user( "success", "" );
641}
642
643function mail_to_user( $type, $msg )
644{
645 // Note to me. Just changed subject line to include a modified $status instead
646 // of the $type variable passed. More informative than just "fail" or "success."
647 // See how it works for awhile and then consider removing $type parameter from
648 // function.
649 global $email_address;
650 global $submittime;
651 global $queuestatus;
652 global $status;
653 global $cluster;
654 global $jobtype;
655 global $org_name;
656 global $admin_email;
657 global $db;
658 global $dbhost;
659 global $requestID;
660 global $gfacID;
661 global $editXMLFilename;
662 global $stdout;
663 global $org_domain;
664
665global $me;
666write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
667
668 // Get GFAC status and message
669 // function get_gfac_message() also sets global $status
670 $gfac_message = get_gfac_message( $gfacID );
671 if ( $gfac_message === false ) $gfac_message = "Job Finished";
672
673 // Create a status to put in the subject line
674 switch ( $status )
675 {
676 case "COMPLETE":
677 $subj_status = 'completed';
678 break;
679
680 case "CANCELLED":
681 case "CANCELED":
682 $subj_status = 'canceled';
683 break;
684
685 case "FAILED":
686 $subj_status = 'failed';
687 if ( preg_match( "/^US3-A/i", $gfacID ) )
688 { // For A/Thrift FAIL, get error message
689 $gfac_message = getExperimentErrors( $gfacID );
690//$gfac_message .= "Test ERROR MESSAGE";
691 }
692 break;
693
694 case "ERROR":
695 $subj_status = 'unknown error';
696 break;
697
698 default:
699 $subj_status = $status; // For now
700 break;
701
702 }
703
704 $queuestatus = $subj_status;
705 $limshost = $dbhost;
706 if ( $limshost == 'localhost' )
707 {
708 $limshost = gethostname();
709 if ( ! preg_match( "/\./", $limshost ) )
710 $limshost = $limshost . $org_domain;
711 }
712
713 // Parse the editXMLFilename
714 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
715 explode( ".", $editXMLFilename );
716
717 $headers = "From: $org_name Admin<$admin_email>" . "\n";
718 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
719
720 // Set the reply address
721 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
722 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
723
724 // Try to avoid spam filters
725 $now = time();
726 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
727 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
728 $headers .= "MIME-Version: 1.0" . "\n";
729 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
730
731 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
732 $message = "
733 Your UltraScan job is complete:
734
735 Submission Time : $submittime
736 LIMS Host : $limshost
737 Analysis ID : $gfacID
738 Request ID : $requestID ( $db )
739 RunID : $runID
740 EditID : $editID
741 Data Type : $dataType
742 Cell/Channel/Wl : $cell / $channel / $wl
743 Status : $queuestatus
744 Cluster : $cluster
745 Job Type : $jobtype
746 GFAC Status : $status
747 GFAC Message : $gfac_message
748 Stdout : $stdout
749 ";
750
751 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
752
753 // Handle the error case where an error occurs before fetching the
754 // user's email address
755 if ( $email_address == "" ) $email_address = $admin_email;
756
757 mail( $email_address, $subject, $message, $headers );
758}
759
760function parse_xml( $xml, $type )
761{
762 $parser = new XMLReader();
763 $parser->xml( $xml );
764
765 $results = array();
766
767 while ( $parser->read() )
768 {
769 if ( $parser->name == $type )
770 {
771 while ( $parser->moveToNextAttribute() )
772 {
773 $results[ $parser->name ] = $parser->value;
774 }
775
776 break;
777 }
778 }
779
780 $parser->close();
781 return $results;
782}
783
784// Function to get information about the current job GFAC
785function get_gfac_message( $gfacID )
786{
787 global $serviceURL;
788 global $me;
789
790 $hex = "[0-9a-fA-F]";
791 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
792 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
793 {
794 // Then it's not a GFAC job
795 return false;
796 }
797
798 $url = "$serviceURL/jobstatus/$gfacID";
799 try
800 {
801 $post = new HttpRequest( $url, HttpRequest::METH_GET );
802 $http = $post->send();
803 $xml = $post->getResponseBody();
804 }
805 catch ( HttpException $e )
806 {
807 write_log( "$me: Job status not available - $gfacID" );
808 return false;
809 }
810
811 // Parse the result
812 $gfac_message = parse_message( $xml );
813
814 return $gfac_message;
815}
816
817function parse_message( $xml )
818{
819 global $status;
820 $status = "";
821 $gfac_message = "";
822
823 $parser = new XMLReader();
824 $parser->xml( $xml );
825
826 $results = array();
827
828 while( $parser->read() )
829 {
830 $type = $parser->nodeType;
831
832 if ( $type == XMLReader::ELEMENT )
833 $name = $parser->name;
834
835 else if ( $type == XMLReader::TEXT )
836 {
837 if ( $name == "status" )
838 $status = $parser->value;
839 else
840 $gfac_message = $parser->value;
841 }
842 }
843
844 $parser->close();
845 return $gfac_message;
846}
847
848function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
849{
850 global $work;
851 global $work_remote;
852 global $me;
853 global $db;
854 global $status;
855 $is_us3iab = preg_match( "/us3iab/", $cluster );
856
857 // Figure out remote directory
858 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
859write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
860
861 // Get stdout, stderr, output/analysis-results.tar
862 $output = array();
863
864 if ( $is_us3iab == 0 )
865 {
866 // Figure out local working directory
867 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
868 $pwd = chdir( "$work/$gfacID" );
869
870 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
871
872 exec( $cmd, $output, $stat );
873 if ( $stat != 0 )
874 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
875
876 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
877
878 exec( $cmd, $output, $stat );
879 if ( $stat != 0 )
880 {
881 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
882 sleep( 10 );
883 write_log( "$me: RETRY" );
884 exec( $cmd, $output, $stat );
885 if ( $stat != 0 )
886 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
887 }
888
889 $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
890
891 exec( $cmd, $output, $stat );
892 if ( $stat != 0 )
893 {
894 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
895 sleep( 10 );
896 write_log( "$me: RETRY" );
897 exec( $cmd, $output, $stat );
898 if ( $stat != 0 )
899 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
900 }
901 }
902 else
903 {
904 $pwd = chdir( "$remoteDir" );
905write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
906 }
907
908
909 // Write the files to gfacDB
910
911 if ( file_exists( "stderr" ) )
912 { // Filter stderr to not have libnnls debug lines
913 exec( "mv stderr stderr+nnls", $output, $stat );
914 exec( "grep -vi nnls stderr+nnls >stderr", $output, $stat );
915 $stderr = file_get_contents( "stderr" );
916 }
917 else
918 $stderr = "";
919 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
920 $fn1_tarfile = "analysis-results.tar";
921 $fn2_tarfile = "output/" . $fn1_tarfile;
922 if ( file_exists( $fn1_tarfile ) )
923 $tarfile = file_get_contents( $fn1_tarfile );
924 else if ( file_exists( $fn2_tarfile ) )
925 $tarfile = file_get_contents( $fn2_tarfile );
926
927 $lense = strlen( $stderr );
928 if ( $lense > 1000000 )
929 { // Replace exceptionally large stderr with smaller version
930 exec( "mv stderr stderr-orig", $output, $stat );
931 exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
932 exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
933 exec( "cat stderr-h stderr-t >stderr", $output, $stat );
934 $stderr = file_get_contents( "stderr" );
935 }
936$lent = strlen( $tarfile );
937write_log( "$me: tarfile size: $lent");
938$lene = strlen( $stderr );
939write_log( "$me: stderr size: $lene (was $lense)");
940$leno = strlen( $stdout );
941write_log( "$me: stdout size: $leno");
942$estarf=mysql_real_escape_string($tarfile,$gfac_link);
943$lenf = strlen($estarf);
944write_log( "$me: es-tarfile size: $lenf");
945$esstdo=mysql_real_escape_string($stdout,$gfac_link);
946$leno = strlen($esstdo);
947write_log( "$me: es-stdout size: $leno");
948$esstde=mysql_real_escape_string($stderr,$gfac_link);
949$lene = strlen($esstde);
950write_log( "$me: es-stderr size: $lene");
951 $query = "UPDATE analysis SET " .
952 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
953 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
954 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
955
956 $result = mysql_query( $query, $gfac_link );
957
958 if ( ! $result )
959 {
960 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
961 echo "Bad query\n";
962 return( -1 );
963 }
964}
965?>
Note: See TracBrowser for help on using the repository browser.