source: trunk/cleanup.php@ 39

Last change on this file since 39 was 39, checked in by gegorbet, 6 years ago

analType and other minor enhancements
File size: 32.9 KB
Line 
1<?php
2/*
3 * cleanup.php
4 *
5 * functions relating to copying results and cleaning up the gfac DB
6 * where the job used an Airavata interface.
7 *
8 */
9
10$email_address = '';
11$queuestatus = '';
12$jobtype = '';
13$db = '';
14$editXMLFilename = '';
15$status = '';
16
17function aira_cleanup( $us3_db, $reqID, $gfac_link )
18{
19 global $org_domain;
20 global $dbhost;
21 global $user;
22 global $passwd;
23 global $db;
24 global $guser;
25 global $gpasswd;
26 global $gDB;
27 global $me;
28 global $work;
29 global $email_address;
30 global $queuestatus;
31 global $jobtype;
32 global $editXMLFilename;
33 global $submittime;
34 global $status;
35 global $stderr;
36 global $stdout;
37 global $tarfile;
38 global $requestID;
39 global $submit_dir;
40 $me = 'cleanup_aira.php';
41
42 $requestID = $reqID;
43 $db = $us3_db;
44 write_log( "$me: debug db=$db; requestID=$requestID" );
45
46 $us3_link = mysqli_connect( $dbhost, $user, $passwd, $db );
47
48 if ( ! $us3_link )
49 {
50 write_log( "$me: could not connect: $dbhost, $user, $passwd, $db" );
51 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
52 return( -1 );
53 }
54
55 // First get basic info for email messages
56 $query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
57 "WHERE HPCAnalysisRequestID=$requestID";
58 $result = mysqli_query( $us3_link, $query );
59
60 if ( ! $result )
61 {
62 write_log( "$me: Bad query: $query" );
63 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
64 return( -1 );
65 }
66
67 list( $email_address, $investigatorGUID, $editXMLFilename ) = mysqli_fetch_array( $result );
68
69 $query = "SELECT personID FROM people " .
70 "WHERE personGUID='$investigatorGUID'";
71 $result = mysqli_query( $us3_link, $query );
72
73 list( $personID ) = mysqli_fetch_array( $result );
74
75 $query = "SELECT clusterName, submitTime, queueStatus, analType " .
76 "FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
77 "WHERE h.HPCAnalysisRequestID=$requestID " .
78 "AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
79
80 $result = mysqli_query( $us3_link, $query );
81
82 if ( ! $result )
83 {
84 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
85 return( -1 );
86 }
87
88 if ( mysqli_num_rows( $result ) == 0 )
89 {
90 write_log( "$me: US3 Table error - No records for requestID: $requestID" );
91 return( -1 );
92 }
93
94 list( $cluster, $submittime, $queuestatus, $jobtype ) = mysqli_fetch_array( $result );
95
96 // Get the GFAC ID
97 $query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
98 "WHERE HPCAnalysisRequestID=$requestID";
99
100 $result = mysqli_query( $us3_link, $query );
101
102 if ( ! $result )
103 {
104 write_log( "$me: Bad query: $query" );
105 mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
106 return( -1 );
107 }
108
109 list( $HPCAnalysisResultID, $gfacID ) = mysqli_fetch_array( $result );
110
111 // Get data from global GFAC DB then insert it into US3 DB
112
113/*
114 $result = mysqli_select_db( $gfac_link, $gDB );
115
116 if ( ! $result )
117 {
118 write_log( "$me: Could not connect to DB $gDB" );
119 mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
120 return( -1 );
121 }
122 */
123
124 $query = "SELECT status, cluster, id FROM analysis " .
125 "WHERE gfacID='$gfacID'";
126
127 $result = mysqli_query( $gfac_link, $query );
128 if ( ! $result )
129 {
130 write_log( "$me: Could not select GFAC status for $gfacID" );
131 mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
132 return( -1 );
133 }
134
135 list( $status, $cluster, $id ) = mysqli_fetch_array( $result );
136
137 $is_us3iab = preg_match( "/us3iab/", $cluster );
138 $is_local = preg_match( "/-local/", $cluster );
139
140 if ( $is_us3iab || $is_local )
141 {
142 $clushost = $cluster;
143 $clushost = preg_replace( "/\-local/", "", $clushost );
144 get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
145 }
146
147
148 $query = "SELECT id FROM analysis " .
149 "WHERE gfacID='$gfacID'";
150
151 $result = mysqli_query( $gfac_link, $query );
152
153 if ( ! $result )
154 {
155 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
156 mail_to_user( "fail", "Internal error " . mysqli_error( $gfac_link ) );
157 return( -1 );
158 }
159
160 list( $analysisID ) = mysqli_fetch_array( $result );
161
162 // Get the request guid (LIMS submit dir name)
163 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
164 "WHERE HPCAnalysisRequestID = $requestID ";
165 $result = mysqli_query( $us3_link, $query );
166
167 if ( ! $result )
168 {
169 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
170 }
171
172 list( $requestGUID ) = mysqli_fetch_array( $result );
173 $output_dir = "$submit_dir/$requestGUID";
174
175 // Get stderr,stdout,tarfile from work directory
176 if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
177 chdir( "$output_dir" );
178//write_log( "$me: gfacID=$gfacID" );
179//write_log( "$me: submit_dir=$submit_dir" );
180//write_log( "$me: requestGUID=$requestGUID" );
181write_log( "$me: output_dir=$output_dir" );
182
183 $stderr = "";
184 $stdout = "";
185 $tarfile = "";
186 $fn_stderr = "Ultrascan.stderr";
187 $fn_stdout = "Ultrascan.stdout";
188 $fn_tarfile = "analysis-results.tar";
189 $secwait = 10;
190 $num_try = 0;
191write_log( "$me: fn_tarfile=$fn_tarfile" );
192 while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
193 {
194 sleep( $secwait );
195 $num_try++;
196 $secwait *= 2;
197write_log( "$me: tar-exists: num_try=$num_try" );
198 }
199
200 $ofiles = scandir( $output_dir );
201 foreach ( $ofiles as $ofile )
202 {
203 if ( preg_match( "/^.*stderr$/", $ofile ) )
204 $fn_stderr = $ofile;
205 if ( preg_match( "/^.*stdout$/", $ofile ) )
206 $fn_stdout = $ofile;
207//write_log( "$me: ofile=$ofile" );
208 }
209write_log( "$me: fn_stderr=$fn_stderr" );
210write_log( "$me: fn_stdout=$fn_stdout" );
211if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
212else write_log( "$me: NOT FOUND: $fn_tarfile" );
213
214 $stderr = '';
215 $stdout = '';
216 $tarfile = '';
217 if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
218 if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
219 if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
220write_log( "$me(0): length contents stderr,stdout,tarfile -- "
221 . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
222 // If stdout,stderr have no content, retry after delay
223 if ( strlen( $stdout ) == 0 || strlen( $stderr ) == 0 )
224 {
225 sleep( 20 );
226 if ( file_exists( $fn_stderr ) )
227 $stderr = file_get_contents( $fn_stderr );
228 if ( file_exists( $fn_stdout ) )
229 $stdout = file_get_contents( $fn_stdout );
230 }
231
232write_log( "$me: length contents stderr,stdout,tarfile -- "
233 . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
234
235 if ( $cluster == 'alamo' || $cluster == 'alamo-local' )
236 { // Filter "ipath_userinit" lines out of alamo stdout lines
237 $prefln = strlen( $stdout );
238 $output = array();
239 exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
240 $stdout = implode( "\n", $output );
241 $posfln = strlen( $stdout );
242write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
243 }
244
245 // Save queue messages for post-mortem analysis
246 $query = "SELECT message, time FROM queue_messages " .
247 "WHERE analysisID = $analysisID " .
248 "ORDER BY time ";
249 $result = mysqli_query( $gfac_link, $query );
250
251 if ( ! $result )
252 {
253 // Just log it and continue
254 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
255 }
256
257 $now = date( 'Y-m-d H:i:s' );
258 $message_log = "US3 DB: $db\n" .
259 "RequestID: $requestID\n" .
260 "GFAC ID: $gfacID\n" .
261 "Processed: $now\n\n" .
262 "Queue Messages\n\n" ;
263 if ( mysqli_num_rows( $result ) > 0 )
264 {
265 while ( list( $message, $time ) = mysqli_fetch_array( $result ) )
266 $message_log .= "$time $message\n";
267 }
268
269 $query = "DELETE FROM queue_messages " .
270 "WHERE analysisID = $analysisID ";
271
272 $result = mysqli_query( $gfac_link, $query );
273
274 if ( ! $result )
275 {
276 // Just log it and continue
277 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
278 }
279
280 $query = "SELECT queue_msg FROM analysis " .
281 "WHERE gfacID='$gfacID' ";
282
283 $result = mysqli_query( $gfac_link, $query );
284 list( $queue_msg ) = mysqli_fetch_array( $result );
285
286 // But let's allow for investigation of other large stdout and/or stderr
287 if ( strlen( $stdout ) > 20480000 ||
288 strlen( $stderr ) > 20480000 )
289 write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
290
291 $message_log .= "\n\n\nStdout Contents\n\n" .
292 $stdout .
293 "\n\n\nStderr Contents\n\n" .
294 $stderr .
295 "\n\n\nGFAC Status: $status\n" .
296 "GFAC message field: $queue_msg\n";
297
298 // Delete data from GFAC DB
299 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
300
301 $result = mysqli_query( $gfac_link, $query );
302
303 if ( ! $result )
304 {
305 // Just log it and continue
306 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
307 }
308
309
310 // Try to create it if necessary, and write the file
311 // Let's use FILE_APPEND, in case this is the second time around and the
312 // GFAC job status was INSERTed, rather than UPDATEd
313 if ( ! is_dir( $output_dir ) )
314 mkdir( $output_dir, 0775, true );
315 $message_filename = "$output_dir/$db-$requestID-messages.txt";
316 file_put_contents( $message_filename, $message_log, FILE_APPEND );
317 // mysqli_close( $gfac_link );
318
319 /////////
320 // Insert data into HPCAnalysis
321
322 $query = "UPDATE HPCAnalysisResult SET " .
323 "stderr='" . mysqli_real_escape_string( $us3_link, $stderr ) . "', " .
324 "stdout='" . mysqli_real_escape_string( $us3_link, $stdout ) . "', " .
325 "queueStatus='completed' " .
326 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
327
328 $result = mysqli_query( $us3_link, $query );
329
330 if ( ! $result )
331 {
332 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
333 mail_to_user( "fail", "Bad query:\n$query\n" . mysqli_error( $us3_link ) );
334 return( -1 );
335 }
336
337 // Delete data from GFAC DB
338 $query = "DELETE from analysis WHERE gfacID='$gfacID'";
339
340 $result = mysqli_query( $gfac_link, $query );
341
342 if ( ! $result )
343 {
344 // Just log it and continue
345 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
346 }
347
348 // Expand the tar file
349
350 if ( strlen( $tarfile ) == 0 )
351 {
352 write_log( "$me: No tarfile" );
353 mail_to_user( "fail", "No results" );
354 return( -1 );
355 }
356
357 $tar_out = array();
358 exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
359
360 // Insert the model files and noise files
361 $files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
362 $noiseIDs = array();
363 $modelGUIDs = array();
364 $mrecsIDs = array();
365 $fns_used = array();
366
367 foreach ( $files as $file )
368 {
369 $split = explode( ";", $file );
370
371 if ( count( $split ) > 1 )
372 {
373 list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
374
375 list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
376 list( $other, $variance ) = explode( "=", $variance );
377 list( $other, $meniscus ) = explode( "=", $meniscus );
378 }
379 else
380 $fn = $file;
381
382 if ( preg_match( "/mdl.tmp$/", $fn ) )
383 continue;
384
385 if ( in_array( $fn, $fns_used ) )
386 continue;
387
388 $fns_used[] = $fn;
389
390 if ( filesize( $fn ) < 100 )
391 {
392 write_log( "$me:fn is invalid $fn size filesize($fn)" );
393 mail_to_user( "fail", "Internal error\n$fn is invalid" );
394 return( -1 );
395 }
396
397 if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
398 {
399 $xml = file_get_contents( $fn );
400 $statistics = parse_xml( $xml, 'statistics' );
401// $ntries = 0;
402//
403// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
404// { // job_statistics file not totally copied, so retry
405// sleep( 10 );
406// $xml = file_get_contents( $fn );
407// $statistics = parse_xml( $xml, 'statistics' );
408// $ntries++;
409//write_log( "$me:jobstats retry $ntries" );
410// }
411//write_log( "$me:cputime=$statistics['cputime']" );
412
413 $otherdata = parse_xml( $xml, 'id' );
414
415 $query = "UPDATE HPCAnalysisResult SET " .
416 "wallTime = {$statistics['walltime']}, " .
417 "CPUTime = {$statistics['cputime']}, " .
418 "CPUCount = {$statistics['cpucount']}, " .
419 "max_rss = {$statistics['maxmemory']}, " .
420 "startTime = '{$otherdata['starttime']}', " .
421 "endTime = '{$otherdata['endtime']}', " .
422 "mgroupcount = {$otherdata['groupcount']} " .
423 "WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
424 $result = mysqli_query( $us3_link, $query );
425
426 if ( ! $result )
427 {
428 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
429 }
430
431 file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
432
433 $file_type = "job_stats";
434 $id = 1;
435 }
436
437 else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
438 {
439 $xml = file_get_contents( $fn );
440 $noise_data = parse_xml( $xml, "noise" );
441 $type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
442 $desc = $noise_data[ 'description' ];
443 $modelGUID = $noise_data[ 'modelGUID' ];
444 $noiseGUID = $noise_data[ 'noiseGUID' ];
445 $editGUID = '00000000-0000-0000-0000-000000000000';
446 if ( isset( $model_data[ 'editGUID' ] ) )
447 $editGUID = $model_data[ 'editGUID' ];
448
449 $query = "INSERT INTO noise SET " .
450 "noiseGUID='$noiseGUID'," .
451 "modelGUID='$modelGUID'," .
452 "editedDataID=" .
453 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
454 "modelID=1, " .
455 "noiseType='$type'," .
456 "description='$desc'," .
457 "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
458
459 // Add later after all files are processed: editDataID, modelID
460
461 $result = mysqli_query( $us3_link, $query );
462
463 if ( ! $result )
464 {
465 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
466 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
467 return( -1 );
468 }
469
470 $id = mysqli_insert_id( $us3_link );
471 $file_type = "noise";
472 $noiseIDs[] = $id;
473
474 // Keep track of modelGUIDs for later, when we replace them
475 $modelGUIDs[ $id ] = $modelGUID;
476
477 }
478
479 else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
480 {
481 $xml = file_get_contents( $fn );
482 $mrecs_data = parse_xml( $xml, "modelrecords" );
483 $desc = $mrecs_data[ 'description' ];
484 $editGUID = $mrecs_data[ 'editGUID' ];
485write_log( "$me: mrecs file editGUID=$editGUID" );
486 if ( strlen( $editGUID ) < 36 )
487 $editGUID = "12345678-0123-5678-0123-567890123456";
488 $mrecGUID = $mrecs_data[ 'mrecGUID' ];
489 $modelGUID = $mrecs_data[ 'modelGUID' ];
490
491 $query = "INSERT INTO pcsa_modelrecs SET " .
492 "editedDataID=" .
493 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
494 "modelID=0, " .
495 "mrecsGUID='$mrecGUID'," .
496 "description='$desc'," .
497 "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
498
499 // Add later after all files are processed: editDataID, modelID
500
501 $result = mysqli_query( $us3_link, $query );
502
503 if ( ! $result )
504 {
505 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
506 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
507 return( -1 );
508 }
509
510 $id = mysqli_insert_id( $us3_link );
511 $file_type = "mrecs";
512 $mrecsIDs[] = $id;
513
514 // Keep track of modelGUIDs for later, when we replace them
515 $rmodlGUIDs[ $id ] = $modelGUID;
516//write_log( "$me: mrecs file inserted into DB : id=$id" );
517 }
518
519 else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
520 {
521 $xml = file_get_contents( $fn );
522 $model_data = parse_xml( $xml, "model" );
523 $description = $model_data[ 'description' ];
524 $modelGUID = $model_data[ 'modelGUID' ];
525 $editGUID = $model_data[ 'editGUID' ];
526
527 if ( $mc_iteration > 1 )
528 {
529 $miter = sprintf( "_mcN%03d", $mc_iteration );
530 $description = preg_replace( "/_mc[0-9]+/", $miter, $description );
531write_log( "$me: MODELUpd: O:description=$description" );
532 }
533
534 $query = "INSERT INTO model SET " .
535 "modelGUID='$modelGUID'," .
536 "editedDataID=" .
537 "(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
538 "description='$description'," .
539 "MCIteration='$mc_iteration'," .
540 "meniscus='$meniscus'," .
541 "variance='$variance'," .
542 "xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
543
544 $result = mysqli_query( $us3_link, $query );
545
546 if ( ! $result )
547 {
548 write_log( "$me: Bad query:\n$query " . mysqli_error( $us3_link ) );
549 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
550 return( -1 );
551 }
552
553 $modelID = mysqli_insert_id( $us3_link );
554 $id = $modelID;
555 $file_type = "model";
556
557 $query = "INSERT INTO modelPerson SET " .
558 "modelID=$modelID, personID=$personID";
559 $result = mysqli_query( $us3_link, $query );
560 }
561
562 else // Undetermined type: skip result data update
563 continue;
564
565 $query = "INSERT INTO HPCAnalysisResultData SET " .
566 "HPCAnalysisResultID='$HPCAnalysisResultID', " .
567 "HPCAnalysisResultType='$file_type', " .
568 "resultID=$id";
569
570 $result = mysqli_query( $us3_link, $query );
571
572 if ( ! $result )
573 {
574 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
575 mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
576 return( -1 );
577 }
578 }
579
580 // Now fix up noise entries
581 // For noise files, there is, at most two: ti_noise and ri_noise
582 // In this case there will only be one modelID
583
584 foreach ( $noiseIDs as $noiseID )
585 {
586 $modelGUID = $modelGUIDs[ $noiseID ];
587 $query = "UPDATE noise SET " .
588 "editedDataID=" .
589 "(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
590 "modelID=" .
591 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
592 "WHERE noiseID=$noiseID";
593
594 $result = mysqli_query( $us3_link, $query );
595
596 if ( ! $result )
597 {
598 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
599 mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
600 return( -1 );
601 }
602 }
603
604 // Now possibly fix up mrecs entries
605
606 foreach ( $mrecsIDs as $mrecsID )
607 {
608 $modelGUID = $rmodlGUIDs[ $mrecsID ];
609 $query = "UPDATE pcsa_modelrecs SET " .
610 "modelID=" .
611 "(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
612 "WHERE mrecsID=$mrecsID";
613
614 $result = mysqli_query( $us3_link, $query );
615
616 if ( ! $result )
617 {
618 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
619 mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
620 return( -1 );
621 }
622write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
623 }
624//write_log( "$me: mrecs entries updated" );
625
626 // Copy results to LIMS submit directory (files there are deleted after 7 days)
627 global $submit_dir; // LIMS submit files dir
628
629 // Get the request guid (LIMS submit dir name)
630 $query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
631 "WHERE HPCAnalysisRequestID = $requestID ";
632 $result = mysqli_query( $us3_link, $query );
633
634 if ( ! $result )
635 {
636 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
637 }
638
639// list( $requestGUID ) = mysqli_fetch_array( $result );
640//
641// chdir( "$submit_dir/$requestGUID" );
642// $f = fopen( "analysis-results.tar", "w" );
643// fwrite( $f, $tarfile );
644// fclose( $f );
645
646 // Clean up
647// chdir ( $work );
648 // exec( "rm -rf $gfacID" );
649
650 mysqli_close( $us3_link );
651
652 /////////
653 // Send email
654
655 mail_to_user( "success", "" );
656}
657
658function mail_to_user( $type, $msg )
659{
660 // Note to me. Just changed subject line to include a modified $status instead
661 // of the $type variable passed. More informative than just "fail" or "success."
662 // See how it works for awhile and then consider removing $type parameter from
663 // function.
664 global $email_address;
665 global $submittime;
666 global $queuestatus;
667 global $status;
668 global $cluster;
669 global $jobtype;
670 global $org_name;
671 global $admin_email;
672 global $db;
673 global $dbhost;
674 global $requestID;
675 global $gfacID;
676 global $editXMLFilename;
677 global $stdout;
678 global $org_domain;
679
680global $me;
681write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
682
683 // Get GFAC status and message
684 // function get_gfac_message() also sets global $status
685 $gfac_message = get_gfac_message( $gfacID );
686 if ( $gfac_message === false ) $gfac_message = "Job Finished";
687
688 // Create a status to put in the subject line
689 switch ( $status )
690 {
691 case "COMPLETE":
692 $subj_status = 'completed';
693 break;
694
695 case "CANCELLED":
696 case "CANCELED":
697 $subj_status = 'canceled';
698 break;
699
700 case "FAILED":
701 $subj_status = 'failed';
702 if ( preg_match( "/^US3-A/i", $gfacID ) )
703 { // For A/Thrift FAIL, get error message
704 $gfac_message = getExperimentErrors( $gfacID );
705//$gfac_message .= "Test ERROR MESSAGE";
706 }
707 break;
708
709 case "ERROR":
710 $subj_status = 'unknown error';
711 break;
712
713 default:
714 $subj_status = $status; // For now
715 break;
716
717 }
718
719 $queuestatus = $subj_status;
720 $limshost = $dbhost;
721 if ( $limshost == 'localhost' )
722 {
723 $limshost = gethostname();
724 if ( preg_match( "/scyld/", $limshost ) )
725 $limshost = 'alamo.uthscsa.edu';
726 else if ( preg_match( "/novalo/", $limshost ) )
727 $limshost = 'uslims3.aucsolutions.com';
728 else if ( ! preg_match( "/\./", $limshost ) )
729 $limshost = $limshost . $org_domain;
730 }
731
732 // Parse the editXMLFilename
733 list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
734 explode( ".", $editXMLFilename );
735
736 $headers = "From: $org_name Admin<$admin_email>" . "\n";
737 $headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
738
739 // Set the reply address
740 $headers .= "Reply-To: $org_name<$admin_email>" . "\n";
741 $headers .= "Return-Path: $org_name<$admin_email>" . "\n";
742
743 // Try to avoid spam filters
744 $now = time();
745 $headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
746 $headers .= "X-Mailer: PHP v" . phpversion() . "\n";
747 $headers .= "MIME-Version: 1.0" . "\n";
748 $headers .= "Content-Transfer-Encoding: 8bit" . "\n";
749
750 $subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
751 $message = "
752 Your UltraScan job is complete:
753
754 Submission Time : $submittime
755 LIMS Host : $limshost
756 Analysis ID : $gfacID
757 Request ID : $requestID ( $db )
758 RunID : $runID
759 EditID : $editID
760 Data Type : $dataType
761 Cell/Channel/Wl : $cell / $channel / $wl
762 Status : $queuestatus
763 Cluster : $cluster
764 Job Type : $jobtype
765 GFAC Status : $status
766 GFAC Message : $gfac_message
767 Stdout : $stdout
768 ";
769
770 if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
771
772 // Handle the error case where an error occurs before fetching the
773 // user's email address
774 if ( $email_address == "" ) $email_address = $admin_email;
775
776 mail( $email_address, $subject, $message, $headers );
777}
778
779function parse_xml( $xml, $type )
780{
781 $parser = new XMLReader();
782 $parser->xml( $xml );
783
784 $results = array();
785
786 while ( $parser->read() )
787 {
788 if ( $parser->name == $type )
789 {
790 while ( $parser->moveToNextAttribute() )
791 {
792 $results[ $parser->name ] = $parser->value;
793 }
794
795 break;
796 }
797 }
798
799 $parser->close();
800 return $results;
801}
802
803// Function to get information about the current job GFAC
804function get_gfac_message( $gfacID )
805{
806 global $serviceURL;
807 global $me;
808
809 $hex = "[0-9a-fA-F]";
810 if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
811 ! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
812 {
813 // Then it's not a GFAC job
814 return false;
815 }
816
817 $url = "$serviceURL/jobstatus/$gfacID";
818 try
819 {
820 $post = new HttpRequest( $url, HttpRequest::METH_GET );
821 $http = $post->send();
822 $xml = $post->getResponseBody();
823 }
824 catch ( HttpException $e )
825 {
826 write_log( "$me: Job status not available - $gfacID" );
827 return false;
828 }
829
830 // Parse the result
831 $gfac_message = parse_message( $xml );
832
833 return $gfac_message;
834}
835
836function parse_message( $xml )
837{
838 global $status;
839 $status = "";
840 $gfac_message = "";
841
842 $parser = new XMLReader();
843 $parser->xml( $xml );
844
845 $results = array();
846
847 while( $parser->read() )
848 {
849 $type = $parser->nodeType;
850
851 if ( $type == XMLReader::ELEMENT )
852 $name = $parser->name;
853
854 else if ( $type == XMLReader::TEXT )
855 {
856 if ( $name == "status" )
857 $status = $parser->value;
858 else
859 $gfac_message = $parser->value;
860 }
861 }
862
863 $parser->close();
864 return $gfac_message;
865}
866
867function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
868{
869 global $work;
870 global $work_remote;
871 global $me;
872 global $db;
873 global $dbhost;
874 global $status;
875 $is_us3iab = preg_match( "/us3iab/", $cluster );
876 $is_jetstr = preg_match( "/jetstream/", $cluster );
877 $limshost = $dbhost;
878 $stderr = '';
879 $stdout = '';
880 $tarfile = '';
881
882 if ( $limshost == 'localhost' )
883 { // If DB host is local host, get full LIMS host name
884 $limshost = gethostname();
885 if ( preg_match( "/scyld/", $limshost ) )
886 $limshost = 'alamo.uthscsa.edu';
887 else if ( preg_match( "/novalo/", $limshost ) )
888 $limshost = 'uslims3.aucsolutions.com';
889 else if ( ! preg_match( "/\./", $limshost ) )
890 $limshost = $limshost . $org_domain;
891 }
892
893 if ( preg_match( "/alamo/", $limshost ) &&
894 preg_match( "/alamo/", $cluster ) )
895 { // If both LIMS and cluster are alamo, set up local transfers
896 $is_us3iab = 1;
897 if ( ! preg_match( "/\/local/", $work_remote ) )
898 $work_remote = $work_remote . "/local";
899 }
900
901 // Figure out job's remote (or local) work directory
902 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
903//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
904
905 // Get stdout, stderr, output/analysis-results.tar
906 $output = array();
907
908 if ( $is_us3iab == 0 )
909 {
910 // For "-local", recompute remote work directory
911 $clushost = "$cluster.uthscsa.edu";
912 $lworkdir = "~us3/lims/work/local";
913 if ( $is_jetstr )
914 {
915 $clushost = "js-169-137.jetstream-cloud.org";
916 $lworkdir = "/N/us3_cluster/work/local";
917 }
918 $cmd = "ssh us3@$clushost 'ls -d $lworkdir' 2>/dev/null";
919 exec( $cmd, $output, $stat );
920 $work_remote = $output[ 0 ];
921 $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
922write_log( "$me: -LOCAL: remoteDir=$remoteDir" );
923
924 // Figure out local working directory
925 if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
926 $pwd = chdir( "$work/$gfacID" );
927
928 $cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1";
929
930 exec( $cmd, $output, $stat );
931 if ( $stat != 0 )
932 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
933
934 $cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1";
935
936 exec( $cmd, $output, $stat );
937 if ( $stat != 0 )
938 {
939 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
940 sleep( 10 );
941 write_log( "$me: RETRY" );
942 exec( $cmd, $output, $stat );
943 if ( $stat != 0 )
944 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
945 }
946
947 $cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1";
948
949 exec( $cmd, $output, $stat );
950 if ( $stat != 0 )
951 {
952 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
953 sleep( 10 );
954 write_log( "$me: RETRY" );
955 exec( $cmd, $output, $stat );
956 if ( $stat != 0 )
957 write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
958 }
959 }
960 else
961 { // Is US3IAB or alamo-to-alamo, so just change to local work directory
962 $pwd = chdir( "$remoteDir" );
963write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
964 }
965
966
967 // Write the files to gfacDB
968
969 $secwait = 10;
970 $num_try = 0;
971 while ( ! file_exists( "stderr" ) && $num_try < 3 )
972 { // Do waits and retries to let stderr appear
973 sleep( $secwait );
974 $num_try++;
975 $secwait *= 2;
976write_log( "$me: not-exist-stderr: num_try=$num_try" );
977 }
978
979 $lense = 0;
980 if ( file_exists( "stderr" ) )
981 {
982 $lense = filesize( "stderr" );
983 if ( $lense > 1000000 )
984 { // Replace exceptionally large stderr with smaller version
985 exec( "mv stderr stderr-orig", $output, $stat );
986 exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
987 exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
988 exec( "cat stderr-h stderr-t >stderr", $output, $stat );
989 }
990 $stderr = file_get_contents( "stderr" );
991 }
992 else
993 {
994 $stderr = "";
995 }
996
997 if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
998
999 $fn1_tarfile = "analysis-results.tar";
1000 $fn2_tarfile = "output/" . $fn1_tarfile;
1001 if ( file_exists( $fn1_tarfile ) )
1002 $tarfile = file_get_contents( $fn1_tarfile );
1003 else if ( file_exists( $fn2_tarfile ) )
1004 $tarfile = file_get_contents( $fn2_tarfile );
1005
1006// $lense = strlen( $stderr );
1007// if ( $lense > 1000000 )
1008// { // Replace exceptionally large stderr with smaller version
1009// exec( "mv stderr stderr-orig", $output, $stat );
1010// exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
1011// exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
1012// exec( "cat stderr-h stderr-t >stderr", $output, $stat );
1013// $stderr = file_get_contents( "stderr" );
1014// }
1015$lene = strlen( $stderr );
1016write_log( "$me: stderr size: $lene (was $lense)");
1017$leno = strlen( $stdout );
1018write_log( "$me: stdout size: $leno");
1019$lent = strlen( $tarfile );
1020write_log( "$me: tarfile size: $lent");
1021 $esstde = mysqli_real_escape_string( $gfac_link, $stderr );
1022 $esstdo = mysqli_real_escape_string( $gfac_link, $stdout );
1023 $estarf = mysqli_real_escape_string( $gfac_link, $tarfile );
1024$lene = strlen($esstde);
1025write_log( "$me: es-stderr size: $lene");
1026$leno = strlen($esstdo);
1027write_log( "$me: es-stdout size: $leno");
1028$lenf = strlen($estarf);
1029write_log( "$me: es-tarfile size: $lenf");
1030 $query = "UPDATE analysis SET " .
1031 "stderr='" . $esstde . "'," .
1032 "stdout='" . $esstdo . "'," .
1033 "tarfile='" . $estarf . "'";
1034
1035 $result = mysqli_query( $gfac_link, $query );
1036
1037 if ( ! $result )
1038 {
1039 write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
1040 echo "Bad query\n";
1041 return( -1 );
1042 }
1043}
1044?>
Note: See TracBrowser for help on using the repository browser.