source: trunk/cleanup.php@ 31

Last change on this file since 31 was 31, checked in by gegorbet, 7 years ago

global-fit and jetstream,stampede2 mods
File size: 31.2 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $org_domain;
20 global $dbhost;
21 global $user;
22 global $passwd;
23 global $db;
24 global $guser;
25 global $gpasswd;
26 global $gDB;
27 global $me;
28 global $work;
29 global $email_address;
30 global $queuestatus;
31 global $jobtype;
32 global $editXMLFilename;
33 global $submittime;
34 global $status;
35 global $stderr;
36 global $stdout;
37 global $tarfile;
38 global $requestID;
39 global $submit_dir;
40 $me = 'cleanup_aira.php';
41
42 $requestID = $reqID;
43 $db = $us3_db;
44 write_log( "$me: debug db=$db; requestID=$requestID" );
45
46 $us3_link = mysql_connect( $dbhost, $user, $passwd );
47
48 if ( ! $us3_link )
49 {
50 write_log( "$me: could not connect: $dbhost, $user, $passwd" );
51 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
52 return( -1 );
53 }
54
55 $result = mysql_select_db( $db, $us3_link );
56
57 if ( ! $result )
58 {
59 write_log( "$me: could not select DB $db" );
60 mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
61 return( -1 );
62 }
63
64 // First get basic info for email messages
65 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
66 "WHERE HPCAnalysisRequestID=$requestID";
67 $result = mysql_query( $query, $us3_link );
68
69 if ( ! $result )
70 {
71 write_log( "$me: Bad query: $query" );
72 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
73 return( -1 );
74 }
75
76 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
77
78 $query = "SELECT personID FROM people " .
79 "WHERE personGUID='$investigatorGUID'";
80 $result = mysql_query( $query, $us3_link );
81
82 list( $personID ) = mysql_fetch_array( $result );
83
84 $query = "SELECT clusterName, submitTime, queueStatus, method " .
85 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
86 "WHERE h.HPCAnalysisRequestID=$requestID " .
87 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
88
89 $result = mysql_query( $query, $us3_link );
90
91 if ( ! $result )
92 {
93 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
94 return( -1 );
95 }
96
97 if ( mysql_num_rows( $result ) == 0 )
98 {
99 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
100 return( -1 );
101 }
102
103 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
104
105 // Get the GFAC ID
106 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
107 "WHERE HPCAnalysisRequestID=$requestID";
108
109 $result = mysql_query( $query, $us3_link );
110
111 if ( ! $result )
112 {
113 write_log( "$me: Bad query: $query" );
114 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
115 return( -1 );
116 }
117
118 list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
119
120 // Get data from global GFAC DB then insert it into US3 DB
121
122 $result = mysql_select_db( $gDB, $gfac_link );
123
124 if ( ! $result )
125 {
126 write_log( "$me: Could not connect to DB $gDB" );
127 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
128 return( -1 );
129 }
130
131 $query = "SELECT status, cluster, id FROM analysis " .
132 "WHERE gfacID='$gfacID'";
133
134 $result = mysql_query( $query, $gfac_link );
135 if ( ! $result )
136 {
137 write_log( "$me: Could not select GFAC status for $gfacID" );
138 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
139 return( -1 );
140 }
141
142 list( $status, $cluster, $id ) = mysql_fetch_array( $result );
143
144 $is_us3iab = preg_match( "/us3iab/", $cluster );
145 $is_local = preg_match( "/-local/", $cluster );
146
147 if ( $is_us3iab || $is_local )
148 {
149 $clushost = $cluster;
150 $clushost = preg_replace( "/\-local/", "", $clushost );
151 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
152 }
153
154
155 $query = "SELECT id FROM analysis " .
156 "WHERE gfacID='$gfacID'";
157
158 $result = mysql_query( $query, $gfac_link );
159
160 if ( ! $result )
161 {
162 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
163 mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
164 return( -1 );
165 }
166
167 list( $analysisID ) = mysql_fetch_array( $result );
168
169 // Get the request guid (LIMS submit dir name)
170 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
171 "WHERE HPCAnalysisRequestID = $requestID ";
172 $result = mysql_query( $query, $us3_link );
173
174 if ( ! $result )
175 {
176 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
177 }
178
179 list( $requestGUID ) = mysql_fetch_array( $result );
180 $output_dir = "$submit_dir/$requestGUID";
181
182 // Get stderr,stdout,tarfile from work directory
183 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
184 chdir( "$output_dir" );
185//write_log( "$me: gfacID=$gfacID" );
186//write_log( "$me: submit_dir=$submit_dir" );
187//write_log( "$me: requestGUID=$requestGUID" );
188write_log( "$me: output_dir=$output_dir" );
189
190 $stderr = "";
191 $stdout = "";
192 $tarfile = "";
193 $fn_stderr = "Ultrascan.stderr";
194 $fn_stdout = "Ultrascan.stdout";
195 $fn_tarfile = "analysis-results.tar";
196 $secwait = 10;
197 $num_try = 0;
198write_log( "$me: fn_tarfile=$fn_tarfile" );
199 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
200 {
201 sleep( $secwait );
202 $num_try++;
203 $secwait *= 2;
204write_log( "$me: tar-exists: num_try=$num_try" );
205 }
206
207 $ofiles = scandir( $output_dir );
208 foreach ( $ofiles as $ofile )
209 {
210 if ( preg_match( "/^.*stderr$/", $ofile ) )
211 $fn_stderr = $ofile;
212 if ( preg_match( "/^.*stdout$/", $ofile ) )
213 $fn_stdout = $ofile;
214//write_log( "$me: ofile=$ofile" );
215 }
216write_log( "$me: fn_stderr=$fn_stderr" );
217write_log( "$me: fn_stdout=$fn_stdout" );
218if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
219else write_log( "$me: NOT FOUND: $fn_tarfile" );
220
221 $stderr = '';
222 $stdout = '';
223 $tarfile = '';
224 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
225 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
226 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
227write_log( "$me(0): length contents stderr,stdout,tarfile -- "
228 . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
229 // If stdout,stderr have no content, retry after delay
230 if ( strlen( $stdout ) == 0 || strlen( $stderr ) == 0 )
231 {
232 sleep( 20 );
233 if ( file_exists( $fn_stderr ) )
234 $stderr = file_get_contents( $fn_stderr );
235 if ( file_exists( $fn_stdout ) )
236 $stdout = file_get_contents( $fn_stdout );
237 }
238
239write_log( "$me: length contents stderr,stdout,tarfile -- "
240 . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
241
242 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
243 { // Filter "ipath_userinit" lines out of alamo stdout lines
244 $prefln = strlen( $stdout );
245 $output = array();
246 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
247 $stdout = implode( "\n", $output );
248 $posfln = strlen( $stdout );
249write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
250 }
251
252 // Save queue messages for post-mortem analysis
253 $query = "SELECT message, time FROM queue_messages " .
254 "WHERE analysisID = $analysisID " .
255 "ORDER BY time ";
256 $result = mysql_query( $query, $gfac_link );
257
258 if ( ! $result )
259 {
260 // Just log it and continue
261 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
262 }
263
264 $now = date( 'Y-m-d H:i:s' );
265 $message_log = "US3 DB: $db\n" .
266 "RequestID: $requestID\n" .
267 "GFAC ID: $gfacID\n" .
268 "Processed: $now\n\n" .
269 "Queue Messages\n\n" ;
270 if ( mysql_num_rows( $result ) > 0 )
271 {
272 while ( list( $message, $time ) = mysql_fetch_array( $result ) )
273 $message_log .= "$time $message\n";
274 }
275
276 $query = "DELETE FROM queue_messages " .
277 "WHERE analysisID = $analysisID ";
278
279 $result = mysql_query( $query, $gfac_link );
280
281 if ( ! $result )
282 {
283 // Just log it and continue
284 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
285 }
286
287 $query = "SELECT queue_msg FROM analysis " .
288 "WHERE gfacID='$gfacID' ";
289
290 $result = mysql_query( $query, $gfac_link );
291 list( $queue_msg ) = mysql_fetch_array( $result );
292
293 // But let's allow for investigation of other large stdout and/or stderr
294 if ( strlen( $stdout ) > 20480000 ||
295 strlen( $stderr ) > 20480000 )
296 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
297
298 $message_log .= "\n\n\nStdout Contents\n\n" .
299 $stdout .
300 "\n\n\nStderr Contents\n\n" .
301 $stderr .
302 "\n\n\nGFAC Status: $status\n" .
303 "GFAC message field: $queue_msg\n";
304
305 // Delete data from GFAC DB
306 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
307
308 $result = mysql_query( $query, $gfac_link );
309
310 if ( ! $result )
311 {
312 // Just log it and continue
313 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
314 }
315
316
317 // Try to create it if necessary, and write the file
318 // Let's use FILE_APPEND, in case this is the second time around and the
319 // GFAC job status was INSERTed, rather than UPDATEd
320 if ( ! is_dir( $output_dir ) )
321 mkdir( $output_dir, 0775, true );
322 $message_filename = "$output_dir/$db-$requestID-messages.txt";
323 file_put_contents( $message_filename, $message_log, FILE_APPEND );
324 // mysql_close( $gfac_link );
325
326 /////////
327 // Insert data into HPCAnalysis
328
329 $query = "UPDATE HPCAnalysisResult SET " .
330 "stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
331 "stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
332 "queueStatus='completed' " .
333 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
334
335 $result = mysql_query( $query, $us3_link );
336
337 if ( ! $result )
338 {
339 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
340 mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
341 return( -1 );
342 }
343
344 // Delete data from GFAC DB
345 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
346
347 $result = mysql_query( $query, $gfac_link );
348
349 if ( ! $result )
350 {
351 // Just log it and continue
352 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
353 }
354
355 // Expand the tar file
356
357 if ( strlen( $tarfile ) == 0 )
358 {
359 write_log( "$me: No tarfile" );
360 mail_to_user( "fail", "No results" );
361 return( -1 );
362 }
363
364 $tar_out = array();
365 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
366
367 // Insert the model files and noise files
368 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
369 $noiseIDs = array();
370 $modelGUIDs = array();
371 $mrecsIDs = array();
372 $fns_used = array();
373
374 foreach ( $files as $file )
375 {
376 $split = explode( ";", $file );
377
378 if ( count( $split ) > 1 )
379 {
380 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
381
382 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
383 list( $other, $variance ) = explode( "=", $variance );
384 list( $other, $meniscus ) = explode( "=", $meniscus );
385 }
386 else
387 $fn = $file;
388
389 if ( preg_match( "/mdl.tmp$/", $fn ) )
390 continue;
391
392 if ( in_array( $fn, $fns_used ) )
393 continue;
394
395 $fns_used[] = $fn;
396
397 if ( filesize( $fn ) < 100 )
398 {
399 write_log( "$me:fn is invalid $fn size filesize($fn)" );
400 mail_to_user( "fail", "Internal error\n$fn is invalid" );
401 return( -1 );
402 }
403
404 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
405 {
406 $xml = file_get_contents( $fn );
407 $statistics = parse_xml( $xml, 'statistics' );
408// $ntries = 0;
409//
410// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
411// { // job_statistics file not totally copied, so retry
412// sleep( 10 );
413// $xml = file_get_contents( $fn );
414// $statistics = parse_xml( $xml, 'statistics' );
415// $ntries++;
416//write_log( "$me:jobstats retry $ntries" );
417// }
418//write_log( "$me:cputime=$statistics['cputime']" );
419
420 $otherdata = parse_xml( $xml, 'id' );
421
422 $query = "UPDATE HPCAnalysisResult SET " .
423 "wallTime = {$statistics['walltime']}, " .
424 "CPUTime = {$statistics['cputime']}, " .
425 "CPUCount = {$statistics['cpucount']}, " .
426 "max_rss = {$statistics['maxmemory']}, " .
427 "startTime = '{$otherdata['starttime']}', " .
428 "endTime = '{$otherdata['endtime']}', " .
429 "mgroupcount = {$otherdata['groupcount']} " .
430 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
431 $result = mysql_query( $query, $us3_link );
432
433 if ( ! $result )
434 {
435 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
436 }
437
438 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
439
440 $file_type = "job_stats";
441 $id = 1;
442 }
443
444 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
445 {
446 $xml = file_get_contents( $fn );
447 $noise_data = parse_xml( $xml, "noise" );
448 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
449 $desc = $noise_data[ 'description' ];
450 $modelGUID = $noise_data[ 'modelGUID' ];
451 $noiseGUID = $noise_data[ 'noiseGUID' ];
452
453 $query = "INSERT INTO noise SET " .
454 "noiseGUID='$noiseGUID'," .
455 "modelGUID='$modelGUID'," .
456 "editedDataID=1, " .
457 "modelID=1, " .
458 "noiseType='$type'," .
459 "description='$desc'," .
460 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
461
462 // Add later after all files are processed: editDataID, modelID
463
464 $result = mysql_query( $query, $us3_link );
465
466 if ( ! $result )
467 {
468 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
469 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
470 return( -1 );
471 }
472
473 $id = mysql_insert_id( $us3_link );
474 $file_type = "noise";
475 $noiseIDs[] = $id;
476
477 // Keep track of modelGUIDs for later, when we replace them
478 $modelGUIDs[ $id ] = $modelGUID;
479
480 }
481
482 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
483 {
484 $xml = file_get_contents( $fn );
485 $mrecs_data = parse_xml( $xml, "modelrecords" );
486 $desc = $mrecs_data[ 'description' ];
487 $editGUID = $mrecs_data[ 'editGUID' ];
488write_log( "$me: mrecs file editGUID=$editGUID" );
489 if ( strlen( $editGUID ) < 36 )
490 $editGUID = "12345678-0123-5678-0123-567890123456";
491 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
492 $modelGUID = $mrecs_data[ 'modelGUID' ];
493
494 $query = "INSERT INTO pcsa_modelrecs SET " .
495 "editedDataID=" .
496 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
497 "modelID=0, " .
498 "mrecsGUID='$mrecGUID'," .
499 "description='$desc'," .
500 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
501
502 // Add later after all files are processed: editDataID, modelID
503
504 $result = mysql_query( $query, $us3_link );
505
506 if ( ! $result )
507 {
508 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
509 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
510 return( -1 );
511 }
512
513 $id = mysql_insert_id( $us3_link );
514 $file_type = "mrecs";
515 $mrecsIDs[] = $id;
516
517 // Keep track of modelGUIDs for later, when we replace them
518 $rmodlGUIDs[ $id ] = $modelGUID;
519//write_log( "$me: mrecs file inserted into DB : id=$id" );
520 }
521
522 else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
523 {
524 $xml = file_get_contents( $fn );
525 $model_data = parse_xml( $xml, "model" );
526 $description = $model_data[ 'description' ];
527 $modelGUID = $model_data[ 'modelGUID' ];
528 $editGUID = $model_data[ 'editGUID' ];
529
530 if ( $mc_iteration > 1 )
531 {
532 $miter = sprintf( "_mcN%03d", $mc_iteration );
533 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
534write_log( "$me: MODELUpd: O:description=$description" );
535 }
536
537 $query = "INSERT INTO model SET " .
538 "modelGUID='$modelGUID'," .
539 "editedDataID=" .
540 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
541 "description='$description'," .
542 "MCIteration='$mc_iteration'," .
543 "meniscus='$meniscus'," .
544 "variance='$variance'," .
545 "xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
546
547 $result = mysql_query( $query, $us3_link );
548
549 if ( ! $result )
550 {
551 write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
552 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
553 return( -1 );
554 }
555
556 $modelID = mysql_insert_id( $us3_link );
557 $id = $modelID;
558 $file_type = "model";
559
560 $query = "INSERT INTO modelPerson SET " .
561 "modelID=$modelID, personID=$personID";
562 $result = mysql_query( $query, $us3_link );
563 }
564
565 else // Undetermined type: skip result data update
566 continue;
567
568 $query = "INSERT INTO HPCAnalysisResultData SET " .
569 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
570 "HPCAnalysisResultType='$file_type', " .
571 "resultID=$id";
572
573 $result = mysql_query( $query, $us3_link );
574
575 if ( ! $result )
576 {
577 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
578 mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
579 return( -1 );
580 }
581 }
582
583 // Now fix up noise entries
584 // For noise files, there is, at most two: ti_noise and ri_noise
585 // In this case there will only be one modelID
586
587 foreach ( $noiseIDs as $noiseID )
588 {
589 $modelGUID = $modelGUIDs[ $noiseID ];
590 $query = "UPDATE noise SET " .
591 "editedDataID=" .
592 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
593 "modelID=" .
594 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
595 "WHERE noiseID=$noiseID";
596
597 $result = mysql_query( $query, $us3_link );
598
599 if ( ! $result )
600 {
601 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
602 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
603 return( -1 );
604 }
605 }
606
607 // Now possibly fix up mrecs entries
608
609 foreach ( $mrecsIDs as $mrecsID )
610 {
611 $modelGUID = $rmodlGUIDs[ $mrecsID ];
612 $query = "UPDATE pcsa_modelrecs SET " .
613 "modelID=" .
614 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
615 "WHERE mrecsID=$mrecsID";
616
617 $result = mysql_query( $query, $us3_link );
618
619 if ( ! $result )
620 {
621 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
622 mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
623 return( -1 );
624 }
625write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
626 }
627//write_log( "$me: mrecs entries updated" );
628
629 // Copy results to LIMS submit directory (files there are deleted after 7 days)
630 global $submit_dir; // LIMS submit files dir
631
632 // Get the request guid (LIMS submit dir name)
633 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
634 "WHERE HPCAnalysisRequestID = $requestID ";
635 $result = mysql_query( $query, $us3_link );
636
637 if ( ! $result )
638 {
639 write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
640 }
641
642// list( $requestGUID ) = mysql_fetch_array( $result );
643//
644// chdir( "$submit_dir/$requestGUID" );
645// $f = fopen( "analysis-results.tar", "w" );
646// fwrite( $f, $tarfile );
647// fclose( $f );
648
649 // Clean up
650// chdir ( $work );
651 // exec( "rm -rf $gfacID" );
652
653 mysql_close( $us3_link );
654
655 /////////
656 // Send email
657
658 mail_to_user( "success", "" );
659}
660
661function mail_to_user( $type, $msg )
662{
663 // Note to me. Just changed subject line to include a modified $status instead
664 // of the $type variable passed. More informative than just "fail" or "success."
665 // See how it works for awhile and then consider removing $type parameter from
666 // function.
667 global $email_address;
668 global $submittime;
669 global $queuestatus;
670 global $status;
671 global $cluster;
672 global $jobtype;
673 global $org_name;
674 global $admin_email;
675 global $db;
676 global $dbhost;
677 global $requestID;
678 global $gfacID;
679 global $editXMLFilename;
680 global $stdout;
681 global $org_domain;
682
683global $me;
684write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
685
686 // Get GFAC status and message
687 // function get_gfac_message() also sets global $status
688 $gfac_message = get_gfac_message( $gfacID );
689 if ( $gfac_message === false ) $gfac_message = "Job Finished";
690
691 // Create a status to put in the subject line
692 switch ( $status )
693 {
694 case "COMPLETE":
695 $subj_status = 'completed';
696 break;
697
698 case "CANCELLED":
699 case "CANCELED":
700 $subj_status = 'canceled';
701 break;
702
703 case "FAILED":
704 $subj_status = 'failed';
705 if ( preg_match( "/^US3-A/i", $gfacID ) )
706 { // For A/Thrift FAIL, get error message
707 $gfac_message = getExperimentErrors( $gfacID );
708//$gfac_message .= "Test ERROR MESSAGE";
709 }
710 break;
711
712 case "ERROR":
713 $subj_status = 'unknown error';
714 break;
715
716 default:
717 $subj_status = $status; // For now
718 break;
719
720 }
721
722 $queuestatus = $subj_status;
723 $limshost = $dbhost;
724 if ( $limshost == 'localhost' )
725 {
726 $limshost = gethostname();
727 if ( ! preg_match( "/\./", $limshost ) )
728 $limshost = $limshost . $org_domain;
729 }
730
731 // Parse the editXMLFilename
732 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
733 explode( ".", $editXMLFilename );
734
735 $headers = "From: $org_name Admin<$admin_email>" . "\n";
736 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
737
738 // Set the reply address
739 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
740 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
741
742 // Try to avoid spam filters
743 $now = time();
744 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
745 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
746 $headers .= "MIME-Version: 1.0" . "\n";
747 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
748
749 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
750 $message = "
751 Your UltraScan job is complete:
752
753 Submission Time : $submittime
754 LIMS Host : $limshost
755 Analysis ID : $gfacID
756 Request ID : $requestID ( $db )
757 RunID : $runID
758 EditID : $editID
759 Data Type : $dataType
760 Cell/Channel/Wl : $cell / $channel / $wl
761 Status : $queuestatus
762 Cluster : $cluster
763 Job Type : $jobtype
764 GFAC Status : $status
765 GFAC Message : $gfac_message
766 Stdout : $stdout
767 ";
768
769 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
770
771 // Handle the error case where an error occurs before fetching the
772 // user's email address
773 if ( $email_address == "" ) $email_address = $admin_email;
774
775 mail( $email_address, $subject, $message, $headers );
776}
777
778function parse_xml( $xml, $type )
779{
780 $parser = new XMLReader();
781 $parser->xml( $xml );
782
783 $results = array();
784
785 while ( $parser->read() )
786 {
787 if ( $parser->name == $type )
788 {
789 while ( $parser->moveToNextAttribute() )
790 {
791 $results[ $parser->name ] = $parser->value;
792 }
793
794 break;
795 }
796 }
797
798 $parser->close();
799 return $results;
800}
801
802// Function to get information about the current job GFAC
803function get_gfac_message( $gfacID )
804{
805 global $serviceURL;
806 global $me;
807
808 $hex = "[0-9a-fA-F]";
809 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
810 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
811 {
812 // Then it's not a GFAC job
813 return false;
814 }
815
816 $url = "$serviceURL/jobstatus/$gfacID";
817 try
818 {
819 $post = new HttpRequest( $url, HttpRequest::METH_GET );
820 $http = $post->send();
821 $xml = $post->getResponseBody();
822 }
823 catch ( HttpException $e )
824 {
825 write_log( "$me: Job status not available - $gfacID" );
826 return false;
827 }
828
829 // Parse the result
830 $gfac_message = parse_message( $xml );
831
832 return $gfac_message;
833}
834
835function parse_message( $xml )
836{
837 global $status;
838 $status = "";
839 $gfac_message = "";
840
841 $parser = new XMLReader();
842 $parser->xml( $xml );
843
844 $results = array();
845
846 while( $parser->read() )
847 {
848 $type = $parser->nodeType;
849
850 if ( $type == XMLReader::ELEMENT )
851 $name = $parser->name;
852
853 else if ( $type == XMLReader::TEXT )
854 {
855 if ( $name == "status" )
856 $status = $parser->value;
857 else
858 $gfac_message = $parser->value;
859 }
860 }
861
862 $parser->close();
863 return $gfac_message;
864}
865
866function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
867{
868 global $work;
869 global $work_remote;
870 global $me;
871 global $db;
872 global $status;
873 $is_us3iab = preg_match( "/us3iab/", $cluster );
874 $is_jetstr = preg_match( "/jetstream/", $cluster );
875
876 // Figure out remote directory
877 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
878//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
879
880 // Get stdout, stderr, output/analysis-results.tar
881 $output = array();
882
883 if ( $is_us3iab == 0 )
884 {
885 // For "-local", recompute remote work directory
886 $clushost = "$cluster.uthscsa.edu";
887 $lworkdir = "~us3/lims/work/local";
888 if ( $is_jetstr )
889 {
890 $clushost = "js-157-184.jetstream-cloud.org";
891 $lworkdir = "/N/us3_cluster/work/local";
892 }
893 $cmd = "ssh us3@$clushost 'ls -d $lworkdir' 2/dev/null";
894 exec( $cmd, $output, $stat );
895 $work_remote = $output[ 0 ];
896 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
897write_log( "$me: -LOCAL: remoteDir=$remoteDir" );
898
899 // Figure out local working directory
900 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
901 $pwd = chdir( "$work/$gfacID" );
902
903 $cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1";
904
905 exec( $cmd, $output, $stat );
906 if ( $stat != 0 )
907 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
908
909 $cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1";
910
911 exec( $cmd, $output, $stat );
912 if ( $stat != 0 )
913 {
914 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
915 sleep( 10 );
916 write_log( "$me: RETRY" );
917 exec( $cmd, $output, $stat );
918 if ( $stat != 0 )
919 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
920 }
921
922 $cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1";
923
924 exec( $cmd, $output, $stat );
925 if ( $stat != 0 )
926 {
927 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
928 sleep( 10 );
929 write_log( "$me: RETRY" );
930 exec( $cmd, $output, $stat );
931 if ( $stat != 0 )
932 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
933 }
934 }
935 else
936 {
937 $pwd = chdir( "$remoteDir" );
938write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
939 }
940
941
942 // Write the files to gfacDB
943
944 if ( file_exists( "stderr" ) )
945 { // Filter stderr to not have libnnls debug lines
946// exec( "mv stderr stderr+nnls", $output, $stat );
947// exec( "grep -vi nnls stderr+nnls >stderr", $output, $stat );
948 $stderr = file_get_contents( "stderr" );
949 }
950 else
951 $stderr = "";
952 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
953 $fn1_tarfile = "analysis-results.tar";
954 $fn2_tarfile = "output/" . $fn1_tarfile;
955 if ( file_exists( $fn1_tarfile ) )
956 $tarfile = file_get_contents( $fn1_tarfile );
957 else if ( file_exists( $fn2_tarfile ) )
958 $tarfile = file_get_contents( $fn2_tarfile );
959
960 $lense = strlen( $stderr );
961 if ( $lense > 1000000 )
962 { // Replace exceptionally large stderr with smaller version
963 exec( "mv stderr stderr-orig", $output, $stat );
964 exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
965 exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
966 exec( "cat stderr-h stderr-t >stderr", $output, $stat );
967 $stderr = file_get_contents( "stderr" );
968 }
969$lent = strlen( $tarfile );
970write_log( "$me: tarfile size: $lent");
971$lene = strlen( $stderr );
972write_log( "$me: stderr size: $lene (was $lense)");
973$leno = strlen( $stdout );
974write_log( "$me: stdout size: $leno");
975$estarf=mysql_real_escape_string($tarfile,$gfac_link);
976$lenf = strlen($estarf);
977write_log( "$me: es-tarfile size: $lenf");
978$esstdo=mysql_real_escape_string($stdout,$gfac_link);
979$leno = strlen($esstdo);
980write_log( "$me: es-stdout size: $leno");
981$esstde=mysql_real_escape_string($stderr,$gfac_link);
982$lene = strlen($esstde);
983write_log( "$me: es-stderr size: $lene");
984 $query = "UPDATE analysis SET " .
985 "stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
986 "stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
987 "tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
988
989 $result = mysql_query( $query, $gfac_link );
990
991 if ( ! $result )
992 {
993 write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
994 echo "Bad query\n";
995 return( -1 );
996 }
997}
998?>
Note: See TracBrowser for help on using the repository browser.