source: trunk/cleanup.php@ 40

Last change on this file since 40 was 40, checked in by gegorbet, 5 years ago

more time information in error,completion emails
File size: 33.0 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $org_domain;
20 global $dbhost;
21 global $user;
22 global $passwd;
23 global $db;
24 global $guser;
25 global $gpasswd;
26 global $gDB;
27 global $me;
28 global $work;
29 global $email_address;
30 global $queuestatus;
31 global $jobtype;
32 global $editXMLFilename;
33 global $submittime;
34 global $endtime;
35 global $status;
36 global $stderr;
37 global $stdout;
38 global $tarfile;
39 global $requestID;
40 global $submit_dir;
41 $me = 'cleanup_aira.php';
42
43 $requestID = $reqID;
44 $db = $us3_db;
45 write_log( "$me: debug db=$db; requestID=$requestID" );
46
47 $us3_link = mysqli_connect( $dbhost, $user, $passwd, $db );
48
49 if ( ! $us3_link )
50 {
51 write_log( "$me: could not connect: $dbhost, $user, $passwd, $db" );
52 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
53 return( -1 );
54 }
55
56 // First get basic info for email messages
57 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
58 "WHERE HPCAnalysisRequestID=$requestID";
59 $result = mysqli_query( $us3_link, $query );
60
61 if ( ! $result )
62 {
63 write_log( "$me: Bad query: $query" );
64 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
65 return( -1 );
66 }
67
68 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysqli_fetch_array( $result );
69
70 $query = "SELECT personID FROM people " .
71 "WHERE personGUID='$investigatorGUID'";
72 $result = mysqli_query( $us3_link, $query );
73
74 list( $personID ) = mysqli_fetch_array( $result );
75
76 $query = "SELECT clusterName, submitTime, queueStatus, analType " .
77 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
78 "WHERE h.HPCAnalysisRequestID=$requestID " .
79 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
80
81 $result = mysqli_query( $us3_link, $query );
82
83 if ( ! $result )
84 {
85 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
86 return( -1 );
87 }
88
89 if ( mysqli_num_rows( $result ) == 0 )
90 {
91 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
92 return( -1 );
93 }
94
95 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysqli_fetch_array( $result );
96
97 // Get the GFAC ID
98 $query = "SELECT HPCAnalysisResultID, gfacID, endTime FROM HPCAnalysisResult " .
99 "WHERE HPCAnalysisRequestID=$requestID";
100
101 $result = mysqli_query( $us3_link, $query );
102
103 if ( ! $result )
104 {
105 write_log( "$me: Bad query: $query" );
106 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
107 return( -1 );
108 }
109
110 list( $HPCAnalysisResultID, $gfacID, $endtime ) = mysqli_fetch_array( $result );
111
112 // Get data from global GFAC DB then insert it into US3 DB
113
114/*
115 $result = mysqli_select_db( $gfac_link, $gDB );
116
117 if ( ! $result )
118 {
119 write_log( "$me: Could not connect to DB $gDB" );
120 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
121 return( -1 );
122 }
123 */
124
125 $query = "SELECT status, cluster, id FROM analysis " .
126 "WHERE gfacID='$gfacID'";
127
128 $result = mysqli_query( $gfac_link, $query );
129 if ( ! $result )
130 {
131 write_log( "$me: Could not select GFAC status for $gfacID" );
132 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
133 return( -1 );
134 }
135
136 list( $status, $cluster, $id ) = mysqli_fetch_array( $result );
137
138 $is_us3iab = preg_match( "/us3iab/", $cluster );
139 $is_local = preg_match( "/-local/", $cluster );
140
141 if ( $is_us3iab || $is_local )
142 {
143 $clushost = $cluster;
144 $clushost = preg_replace( "/\-local/", "", $clushost );
145 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
146 }
147
148
149 $query = "SELECT id FROM analysis " .
150 "WHERE gfacID='$gfacID'";
151
152 $result = mysqli_query( $gfac_link, $query );
153
154 if ( ! $result )
155 {
156 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
157 mail_to_user( "fail", "Internal error " . mysqli_error( $gfac_link ) );
158 return( -1 );
159 }
160
161 list( $analysisID ) = mysqli_fetch_array( $result );
162
163 // Get the request guid (LIMS submit dir name)
164 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
165 "WHERE HPCAnalysisRequestID = $requestID ";
166 $result = mysqli_query( $us3_link, $query );
167
168 if ( ! $result )
169 {
170 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
171 }
172
173 list( $requestGUID ) = mysqli_fetch_array( $result );
174 $output_dir = "$submit_dir/$requestGUID";
175
176 // Get stderr,stdout,tarfile from work directory
177 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
178 chdir( "$output_dir" );
179//write_log( "$me: gfacID=$gfacID" );
180//write_log( "$me: submit_dir=$submit_dir" );
181//write_log( "$me: requestGUID=$requestGUID" );
182write_log( "$me: output_dir=$output_dir" );
183
184 $stderr = "";
185 $stdout = "";
186 $tarfile = "";
187 $fn_stderr = "Ultrascan.stderr";
188 $fn_stdout = "Ultrascan.stdout";
189 $fn_tarfile = "analysis-results.tar";
190 $secwait = 10;
191 $num_try = 0;
192write_log( "$me: fn_tarfile=$fn_tarfile" );
193 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
194 {
195 sleep( $secwait );
196 $num_try++;
197 $secwait *= 2;
198write_log( "$me: tar-exists: num_try=$num_try" );
199 }
200
201 $ofiles = scandir( $output_dir );
202 foreach ( $ofiles as $ofile )
203 {
204 if ( preg_match( "/^.*stderr$/", $ofile ) )
205 $fn_stderr = $ofile;
206 if ( preg_match( "/^.*stdout$/", $ofile ) )
207 $fn_stdout = $ofile;
208//write_log( "$me: ofile=$ofile" );
209 }
210write_log( "$me: fn_stderr=$fn_stderr" );
211write_log( "$me: fn_stdout=$fn_stdout" );
212if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
213else write_log( "$me: NOT FOUND: $fn_tarfile" );
214
215 $stderr = '';
216 $stdout = '';
217 $tarfile = '';
218 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
219 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
220 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
221write_log( "$me(0): length contents stderr,stdout,tarfile -- "
222 . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
223 // If stdout,stderr have no content, retry after delay
224 if ( strlen( $stdout ) == 0 || strlen( $stderr ) == 0 )
225 {
226 sleep( 20 );
227 if ( file_exists( $fn_stderr ) )
228 $stderr = file_get_contents( $fn_stderr );
229 if ( file_exists( $fn_stdout ) )
230 $stdout = file_get_contents( $fn_stdout );
231 }
232
233write_log( "$me: length contents stderr,stdout,tarfile -- "
234 . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
235
236 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
237 { // Filter "ipath_userinit" lines out of alamo stdout lines
238 $prefln = strlen( $stdout );
239 $output = array();
240 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
241 $stdout = implode( "\n", $output );
242 $posfln = strlen( $stdout );
243write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
244 }
245
246 // Save queue messages for post-mortem analysis
247 $query = "SELECT message, time FROM queue_messages " .
248 "WHERE analysisID = $analysisID " .
249 "ORDER BY time ";
250 $result = mysqli_query( $gfac_link, $query );
251
252 if ( ! $result )
253 {
254 // Just log it and continue
255 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
256 }
257
258 $now = date( 'Y-m-d H:i:s' );
259 $message_log = "US3 DB: $db\n" .
260 "RequestID: $requestID\n" .
261 "GFAC ID: $gfacID\n" .
262 "Processed: $now\n\n" .
263 "Queue Messages\n\n" ;
264 if ( mysqli_num_rows( $result ) > 0 )
265 {
266 while ( list( $message, $time ) = mysqli_fetch_array( $result ) )
267 $message_log .= "$time $message\n";
268 }
269
270 $query = "DELETE FROM queue_messages " .
271 "WHERE analysisID = $analysisID ";
272
273 $result = mysqli_query( $gfac_link, $query );
274
275 if ( ! $result )
276 {
277 // Just log it and continue
278 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
279 }
280
281 $query = "SELECT queue_msg FROM analysis " .
282 "WHERE gfacID='$gfacID' ";
283
284 $result = mysqli_query( $gfac_link, $query );
285 list( $queue_msg ) = mysqli_fetch_array( $result );
286
287 // But let's allow for investigation of other large stdout and/or stderr
288 if ( strlen( $stdout ) > 20480000 ||
289 strlen( $stderr ) > 20480000 )
290 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
291
292 $message_log .= "\n\n\nStdout Contents\n\n" .
293 $stdout .
294 "\n\n\nStderr Contents\n\n" .
295 $stderr .
296 "\n\n\nGFAC Status: $status\n" .
297 "GFAC message field: $queue_msg\n";
298
299 // Delete data from GFAC DB
300 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
301
302 $result = mysqli_query( $gfac_link, $query );
303
304 if ( ! $result )
305 {
306 // Just log it and continue
307 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
308 }
309
310
311 // Try to create it if necessary, and write the file
312 // Let's use FILE_APPEND, in case this is the second time around and the
313 // GFAC job status was INSERTed, rather than UPDATEd
314 if ( ! is_dir( $output_dir ) )
315 mkdir( $output_dir, 0775, true );
316 $message_filename = "$output_dir/$db-$requestID-messages.txt";
317 file_put_contents( $message_filename, $message_log, FILE_APPEND );
318 // mysqli_close( $gfac_link );
319
320 /////////
321 // Insert data into HPCAnalysis
322
323 $query = "UPDATE HPCAnalysisResult SET " .
324 "stderr='" . mysqli_real_escape_string( $us3_link, $stderr ) . "', " .
325 "stdout='" . mysqli_real_escape_string( $us3_link, $stdout ) . "', " .
326 "queueStatus='completed' " .
327 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
328
329 $result = mysqli_query( $us3_link, $query );
330
331 if ( ! $result )
332 {
333 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
334 mail_to_user( "fail", "Bad query:\n$query\n" . mysqli_error( $us3_link ) );
335 return( -1 );
336 }
337
338 // Delete data from GFAC DB
339 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
340
341 $result = mysqli_query( $gfac_link, $query );
342
343 if ( ! $result )
344 {
345 // Just log it and continue
346 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
347 }
348
349 // Expand the tar file
350
351 if ( strlen( $tarfile ) == 0 )
352 {
353 write_log( "$me: No tarfile" );
354 mail_to_user( "fail", "No results" );
355 return( -1 );
356 }
357
358 $tar_out = array();
359 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
360
361 // Insert the model files and noise files
362 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
363 $noiseIDs = array();
364 $modelGUIDs = array();
365 $mrecsIDs = array();
366 $fns_used = array();
367
368 foreach ( $files as $file )
369 {
370 $split = explode( ";", $file );
371
372 if ( count( $split ) > 1 )
373 {
374 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
375
376 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
377 list( $other, $variance ) = explode( "=", $variance );
378 list( $other, $meniscus ) = explode( "=", $meniscus );
379 }
380 else
381 $fn = $file;
382
383 if ( preg_match( "/mdl.tmp$/", $fn ) )
384 continue;
385
386 if ( in_array( $fn, $fns_used ) )
387 continue;
388
389 $fns_used[] = $fn;
390
391 if ( filesize( $fn ) < 100 )
392 {
393 write_log( "$me:fn is invalid $fn size filesize($fn)" );
394 mail_to_user( "fail", "Internal error\n$fn is invalid" );
395 return( -1 );
396 }
397
398 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
399 {
400 $xml = file_get_contents( $fn );
401 $statistics = parse_xml( $xml, 'statistics' );
402// $ntries = 0;
403//
404// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
405// { // job_statistics file not totally copied, so retry
406// sleep( 10 );
407// $xml = file_get_contents( $fn );
408// $statistics = parse_xml( $xml, 'statistics' );
409// $ntries++;
410//write_log( "$me:jobstats retry $ntries" );
411// }
412//write_log( "$me:cputime=$statistics['cputime']" );
413
414 $otherdata = parse_xml( $xml, 'id' );
415
416 $query = "UPDATE HPCAnalysisResult SET " .
417 "wallTime = {$statistics['walltime']}, " .
418 "CPUTime = {$statistics['cputime']}, " .
419 "CPUCount = {$statistics['cpucount']}, " .
420 "max_rss = {$statistics['maxmemory']}, " .
421 "startTime = '{$otherdata['starttime']}', " .
422 "endTime = '{$otherdata['endtime']}', " .
423 "mgroupcount = {$otherdata['groupcount']} " .
424 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
425 $result = mysqli_query( $us3_link, $query );
426
427 if ( ! $result )
428 {
429 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
430 }
431
432 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
433
434 $file_type = "job_stats";
435 $id = 1;
436 }
437
438 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
439 {
440 $xml = file_get_contents( $fn );
441 $noise_data = parse_xml( $xml, "noise" );
442 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
443 $desc = $noise_data[ 'description' ];
444 $modelGUID = $noise_data[ 'modelGUID' ];
445 $noiseGUID = $noise_data[ 'noiseGUID' ];
446 $editGUID = '00000000-0000-0000-0000-000000000000';
447 if ( isset( $model_data[ 'editGUID' ] ) )
448 $editGUID = $model_data[ 'editGUID' ];
449
450 $query = "INSERT INTO noise SET " .
451 "noiseGUID='$noiseGUID'," .
452 "modelGUID='$modelGUID'," .
453 "editedDataID=" .
454 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
455 "modelID=1, " .
456 "noiseType='$type'," .
457 "description='$desc'," .
458 "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
459
460 // Add later after all files are processed: editDataID, modelID
461
462 $result = mysqli_query( $us3_link, $query );
463
464 if ( ! $result )
465 {
466 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
467 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
468 return( -1 );
469 }
470
471 $id = mysqli_insert_id( $us3_link );
472 $file_type = "noise";
473 $noiseIDs[] = $id;
474
475 // Keep track of modelGUIDs for later, when we replace them
476 $modelGUIDs[ $id ] = $modelGUID;
477
478 }
479
480 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
481 {
482 $xml = file_get_contents( $fn );
483 $mrecs_data = parse_xml( $xml, "modelrecords" );
484 $desc = $mrecs_data[ 'description' ];
485 $editGUID = $mrecs_data[ 'editGUID' ];
486write_log( "$me: mrecs file editGUID=$editGUID" );
487 if ( strlen( $editGUID ) < 36 )
488 $editGUID = "12345678-0123-5678-0123-567890123456";
489 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
490 $modelGUID = $mrecs_data[ 'modelGUID' ];
491
492 $query = "INSERT INTO pcsa_modelrecs SET " .
493 "editedDataID=" .
494 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
495 "modelID=0, " .
496 "mrecsGUID='$mrecGUID'," .
497 "description='$desc'," .
498 "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
499
500 // Add later after all files are processed: editDataID, modelID
501
502 $result = mysqli_query( $us3_link, $query );
503
504 if ( ! $result )
505 {
506 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
507 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
508 return( -1 );
509 }
510
511 $id = mysqli_insert_id( $us3_link );
512 $file_type = "mrecs";
513 $mrecsIDs[] = $id;
514
515 // Keep track of modelGUIDs for later, when we replace them
516 $rmodlGUIDs[ $id ] = $modelGUID;
517//write_log( "$me: mrecs file inserted into DB : id=$id" );
518 }
519
520 else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
521 {
522 $xml = file_get_contents( $fn );
523 $model_data = parse_xml( $xml, "model" );
524 $description = $model_data[ 'description' ];
525 $modelGUID = $model_data[ 'modelGUID' ];
526 $editGUID = $model_data[ 'editGUID' ];
527
528 if ( $mc_iteration > 1 )
529 {
530 $miter = sprintf( "_mcN%03d", $mc_iteration );
531 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
532write_log( "$me: MODELUpd: O:description=$description" );
533 }
534
535 $query = "INSERT INTO model SET " .
536 "modelGUID='$modelGUID'," .
537 "editedDataID=" .
538 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
539 "description='$description'," .
540 "MCIteration='$mc_iteration'," .
541 "meniscus='$meniscus'," .
542 "variance='$variance'," .
543 "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
544
545 $result = mysqli_query( $us3_link, $query );
546
547 if ( ! $result )
548 {
549 write_log( "$me: Bad query:\n$query " . mysqli_error( $us3_link ) );
550 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
551 return( -1 );
552 }
553
554 $modelID = mysqli_insert_id( $us3_link );
555 $id = $modelID;
556 $file_type = "model";
557
558 $query = "INSERT INTO modelPerson SET " .
559 "modelID=$modelID, personID=$personID";
560 $result = mysqli_query( $us3_link, $query );
561 }
562
563 else // Undetermined type: skip result data update
564 continue;
565
566 $query = "INSERT INTO HPCAnalysisResultData SET " .
567 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
568 "HPCAnalysisResultType='$file_type', " .
569 "resultID=$id";
570
571 $result = mysqli_query( $us3_link, $query );
572
573 if ( ! $result )
574 {
575 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
576 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
577 return( -1 );
578 }
579 }
580
581 // Now fix up noise entries
582 // For noise files, there is, at most two: ti_noise and ri_noise
583 // In this case there will only be one modelID
584
585 foreach ( $noiseIDs as $noiseID )
586 {
587 $modelGUID = $modelGUIDs[ $noiseID ];
588 $query = "UPDATE noise SET " .
589 "editedDataID=" .
590 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
591 "modelID=" .
592 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
593 "WHERE noiseID=$noiseID";
594
595 $result = mysqli_query( $us3_link, $query );
596
597 if ( ! $result )
598 {
599 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
600 mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
601 return( -1 );
602 }
603 }
604
605 // Now possibly fix up mrecs entries
606
607 foreach ( $mrecsIDs as $mrecsID )
608 {
609 $modelGUID = $rmodlGUIDs[ $mrecsID ];
610 $query = "UPDATE pcsa_modelrecs SET " .
611 "modelID=" .
612 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
613 "WHERE mrecsID=$mrecsID";
614
615 $result = mysqli_query( $us3_link, $query );
616
617 if ( ! $result )
618 {
619 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
620 mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
621 return( -1 );
622 }
623write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
624 }
625//write_log( "$me: mrecs entries updated" );
626
627 // Copy results to LIMS submit directory (files there are deleted after 7 days)
628 global $submit_dir; // LIMS submit files dir
629
630 // Get the request guid (LIMS submit dir name)
631 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
632 "WHERE HPCAnalysisRequestID = $requestID ";
633 $result = mysqli_query( $us3_link, $query );
634
635 if ( ! $result )
636 {
637 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
638 }
639
640// list( $requestGUID ) = mysqli_fetch_array( $result );
641//
642// chdir( "$submit_dir/$requestGUID" );
643// $f = fopen( "analysis-results.tar", "w" );
644// fwrite( $f, $tarfile );
645// fclose( $f );
646
647 // Clean up
648// chdir ( $work );
649 // exec( "rm -rf $gfacID" );
650
651 mysqli_close( $us3_link );
652
653 /////////
654 // Send email
655
656 mail_to_user( "success", "" );
657}
658
659function mail_to_user( $type, $msg )
660{
661 // Note to me. Just changed subject line to include a modified $status instead
662 // of the $type variable passed. More informative than just "fail" or "success."
663 // See how it works for awhile and then consider removing $type parameter from
664 // function.
665 global $email_address;
666 global $submittime;
667 global $endtime;
668 global $queuestatus;
669 global $status;
670 global $cluster;
671 global $jobtype;
672 global $org_name;
673 global $admin_email;
674 global $db;
675 global $dbhost;
676 global $requestID;
677 global $gfacID;
678 global $editXMLFilename;
679 global $stdout;
680 global $org_domain;
681
682global $me;
683write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
684
685 // Get GFAC status and message
686 // function get_gfac_message() also sets global $status
687 $gfac_message = get_gfac_message( $gfacID );
688 if ( $gfac_message === false ) $gfac_message = "Job Finished";
689
690 // Create a status to put in the subject line
691 switch ( $status )
692 {
693 case "COMPLETE":
694 $subj_status = 'completed';
695 break;
696
697 case "CANCELLED":
698 case "CANCELED":
699 $subj_status = 'canceled';
700 break;
701
702 case "FAILED":
703 $subj_status = 'failed';
704 if ( preg_match( "/^US3-A/i", $gfacID ) )
705 { // For A/Thrift FAIL, get error message
706 $gfac_message = getExperimentErrors( $gfacID );
707//$gfac_message .= "Test ERROR MESSAGE";
708 }
709 break;
710
711 case "ERROR":
712 $subj_status = 'unknown error';
713 break;
714
715 default:
716 $subj_status = $status; // For now
717 break;
718
719 }
720
721 $queuestatus = $subj_status;
722 $limshost = $dbhost;
723 if ( $limshost == 'localhost' )
724 {
725 $limshost = gethostname();
726 if ( preg_match( "/scyld/", $limshost ) )
727 $limshost = 'alamo.uthscsa.edu';
728 else if ( preg_match( "/novalo/", $limshost ) )
729 $limshost = 'uslims3.aucsolutions.com';
730 else if ( ! preg_match( "/\./", $limshost ) )
731 $limshost = $limshost . $org_domain;
732 }
733
734 // Parse the editXMLFilename
735 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
736 explode( ".", $editXMLFilename );
737
738 $headers = "From: $org_name Admin<$admin_email>" . "\n";
739 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
740
741 // Set the reply address
742 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
743 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
744
745 // Try to avoid spam filters
746 $now = time();
747 $tnow = date( 'Y-m-d H:i:s' );
748 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
749 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
750 $headers .= "MIME-Version: 1.0" . "\n";
751 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
752
753 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
754 $message = "
755 Your UltraScan job is complete:
756
757 Submission Time : $submittime
758 Job End Time : $endtime
759 Mail Time : $tnow
760 LIMS Host : $limshost
761 Analysis ID : $gfacID
762 Request ID : $requestID ( $db )
763 RunID : $runID
764 EditID : $editID
765 Data Type : $dataType
766 Cell/Channel/Wl : $cell / $channel / $wl
767 Status : $queuestatus
768 Cluster : $cluster
769 Job Type : $jobtype
770 GFAC Status : $status
771 GFAC Message : $gfac_message
772 Stdout : $stdout
773 ";
774
775 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
776
777 // Handle the error case where an error occurs before fetching the
778 // user's email address
779 if ( $email_address == "" ) $email_address = $admin_email;
780
781 mail( $email_address, $subject, $message, $headers );
782}
783
784function parse_xml( $xml, $type )
785{
786 $parser = new XMLReader();
787 $parser->xml( $xml );
788
789 $results = array();
790
791 while ( $parser->read() )
792 {
793 if ( $parser->name == $type )
794 {
795 while ( $parser->moveToNextAttribute() )
796 {
797 $results[ $parser->name ] = $parser->value;
798 }
799
800 break;
801 }
802 }
803
804 $parser->close();
805 return $results;
806}
807
808// Function to get information about the current job GFAC
809function get_gfac_message( $gfacID )
810{
811 global $serviceURL;
812 global $me;
813
814 $hex = "[0-9a-fA-F]";
815 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
816 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
817 {
818 // Then it's not a GFAC job
819 return false;
820 }
821
822 $url = "$serviceURL/jobstatus/$gfacID";
823 try
824 {
825 $post = new HttpRequest( $url, HttpRequest::METH_GET );
826 $http = $post->send();
827 $xml = $post->getResponseBody();
828 }
829 catch ( HttpException $e )
830 {
831 write_log( "$me: Job status not available - $gfacID" );
832 return false;
833 }
834
835 // Parse the result
836 $gfac_message = parse_message( $xml );
837
838 return $gfac_message;
839}
840
841function parse_message( $xml )
842{
843 global $status;
844 $status = "";
845 $gfac_message = "";
846
847 $parser = new XMLReader();
848 $parser->xml( $xml );
849
850 $results = array();
851
852 while( $parser->read() )
853 {
854 $type = $parser->nodeType;
855
856 if ( $type == XMLReader::ELEMENT )
857 $name = $parser->name;
858
859 else if ( $type == XMLReader::TEXT )
860 {
861 if ( $name == "status" )
862 $status = $parser->value;
863 else
864 $gfac_message = $parser->value;
865 }
866 }
867
868 $parser->close();
869 return $gfac_message;
870}
871
872function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
873{
874 global $work;
875 global $work_remote;
876 global $me;
877 global $db;
878 global $dbhost;
879 global $status;
880 $is_us3iab = preg_match( "/us3iab/", $cluster );
881 $is_jetstr = preg_match( "/jetstream/", $cluster );
882 $limshost = $dbhost;
883 $stderr = '';
884 $stdout = '';
885 $tarfile = '';
886
887 if ( $limshost == 'localhost' )
888 { // If DB host is local host, get full LIMS host name
889 $limshost = gethostname();
890 if ( preg_match( "/scyld/", $limshost ) )
891 $limshost = 'alamo.uthscsa.edu';
892 else if ( preg_match( "/novalo/", $limshost ) )
893 $limshost = 'uslims3.aucsolutions.com';
894 else if ( ! preg_match( "/\./", $limshost ) )
895 $limshost = $limshost . $org_domain;
896 }
897
898 if ( preg_match( "/alamo/", $limshost ) &&
899 preg_match( "/alamo/", $cluster ) )
900 { // If both LIMS and cluster are alamo, set up local transfers
901 $is_us3iab = 1;
902 if ( ! preg_match( "/\/local/", $work_remote ) )
903 $work_remote = $work_remote . "/local";
904 }
905
906 // Figure out job's remote (or local) work directory
907 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
908//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
909
910 // Get stdout, stderr, output/analysis-results.tar
911 $output = array();
912
913 if ( $is_us3iab == 0 )
914 {
915 // For "-local", recompute remote work directory
916 $clushost = "$cluster.uthscsa.edu";
917 $lworkdir = "~us3/lims/work/local";
918 if ( $is_jetstr )
919 {
920 $clushost = "js-169-137.jetstream-cloud.org";
921 $lworkdir = "/N/us3_cluster/work/local";
922 }
923 $cmd = "ssh us3@$clushost 'ls -d $lworkdir' 2>/dev/null";
924 exec( $cmd, $output, $stat );
925 $work_remote = $output[ 0 ];
926 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
927write_log( "$me: -LOCAL: remoteDir=$remoteDir" );
928
929 // Figure out local working directory
930 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
931 $pwd = chdir( "$work/$gfacID" );
932
933 $cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1";
934
935 exec( $cmd, $output, $stat );
936 if ( $stat != 0 )
937 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
938
939 $cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1";
940
941 exec( $cmd, $output, $stat );
942 if ( $stat != 0 )
943 {
944 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
945 sleep( 10 );
946 write_log( "$me: RETRY" );
947 exec( $cmd, $output, $stat );
948 if ( $stat != 0 )
949 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
950 }
951
952 $cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1";
953
954 exec( $cmd, $output, $stat );
955 if ( $stat != 0 )
956 {
957 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
958 sleep( 10 );
959 write_log( "$me: RETRY" );
960 exec( $cmd, $output, $stat );
961 if ( $stat != 0 )
962 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
963 }
964 }
965 else
966 { // Is US3IAB or alamo-to-alamo, so just change to local work directory
967 $pwd = chdir( "$remoteDir" );
968write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
969 }
970
971
972 // Write the files to gfacDB
973
974 $secwait = 10;
975 $num_try = 0;
976 while ( ! file_exists( "stderr" ) && $num_try < 3 )
977 { // Do waits and retries to let stderr appear
978 sleep( $secwait );
979 $num_try++;
980 $secwait *= 2;
981write_log( "$me: not-exist-stderr: num_try=$num_try" );
982 }
983
984 $lense = 0;
985 if ( file_exists( "stderr" ) )
986 {
987 $lense = filesize( "stderr" );
988 if ( $lense > 1000000 )
989 { // Replace exceptionally large stderr with smaller version
990 exec( "mv stderr stderr-orig", $output, $stat );
991 exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
992 exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
993 exec( "cat stderr-h stderr-t >stderr", $output, $stat );
994 }
995 $stderr = file_get_contents( "stderr" );
996 }
997 else
998 {
999 $stderr = "";
1000 }
1001
1002 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
1003
1004 $fn1_tarfile = "analysis-results.tar";
1005 $fn2_tarfile = "output/" . $fn1_tarfile;
1006 if ( file_exists( $fn1_tarfile ) )
1007 $tarfile = file_get_contents( $fn1_tarfile );
1008 else if ( file_exists( $fn2_tarfile ) )
1009 $tarfile = file_get_contents( $fn2_tarfile );
1010
1011// $lense = strlen( $stderr );
1012// if ( $lense > 1000000 )
1013// { // Replace exceptionally large stderr with smaller version
1014// exec( "mv stderr stderr-orig", $output, $stat );
1015// exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
1016// exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
1017// exec( "cat stderr-h stderr-t >stderr", $output, $stat );
1018// $stderr = file_get_contents( "stderr" );
1019// }
1020$lene = strlen( $stderr );
1021write_log( "$me: stderr size: $lene (was $lense)");
1022$leno = strlen( $stdout );
1023write_log( "$me: stdout size: $leno");
1024$lent = strlen( $tarfile );
1025write_log( "$me: tarfile size: $lent");
1026 $esstde = mysqli_real_escape_string( $gfac_link, $stderr );
1027 $esstdo = mysqli_real_escape_string( $gfac_link, $stdout );
1028 $estarf = mysqli_real_escape_string( $gfac_link, $tarfile );
1029$lene = strlen($esstde);
1030write_log( "$me: es-stderr size: $lene");
1031$leno = strlen($esstdo);
1032write_log( "$me: es-stdout size: $leno");
1033$lenf = strlen($estarf);
1034write_log( "$me: es-tarfile size: $lenf");
1035 $query = "UPDATE analysis SET " .
1036 "stderr='" . $esstde . "'," .
1037 "stdout='" . $esstdo . "'," .
1038 "tarfile='" . $estarf . "'";
1039
1040 $result = mysqli_query( $gfac_link, $query );
1041
1042 if ( ! $result )
1043 {
1044 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
1045 echo "Bad query\n";
1046 return( -1 );
1047 }
1048}
1049?>
Note: See TracBrowser for help on using the repository browser.