Context Navigation

source: trunk/cleanup.php@ 39

Last change on this file since 39 was 39, checked in by gegorbet, 6 years ago
analType and other minor enhancements
File size: 32.9 KB

Rev	Line
[1]	1	<?php
	2	/*
	3	* cleanup.php
	4	*
	5	* functions relating to copying results and cleaning up the gfac DB
[10]	6	* where the job used an Airavata interface.
[1]	7	*
	8	*/
	9
	10	$email_address = '';
	11	$queuestatus = '';
	12	$jobtype = '';
	13	$db = '';
	14	$editXMLFilename = '';
[4]	15	$status = '';
[1]	16
[10]	17	function aira_cleanup( $us3_db, $reqID, $gfac_link )
[1]	18	{
[25]	19	global $org_domain;
[1]	20	global $dbhost;
	21	global $user;
	22	global $passwd;
	23	global $db;
	24	global $guser;
	25	global $gpasswd;
	26	global $gDB;
	27	global $me;
	28	global $work;
	29	global $email_address;
	30	global $queuestatus;
	31	global $jobtype;
	32	global $editXMLFilename;
	33	global $submittime;
[4]	34	global $status;
[10]	35	global $stderr;
[5]	36	global $stdout;
[10]	37	global $tarfile;
[6]	38	global $requestID;
[10]	39	global $submit_dir;
	40	$me = 'cleanup_aira.php';
[1]	41
[6]	42	$requestID = $reqID;
[1]	43	$db = $us3_db;
	44	write_log( "$me: debug db=$db; requestID=$requestID" );
	45
[35]	46	$us3_link = mysqli_connect( $dbhost, $user, $passwd, $db );
[1]	47
	48	if ( ! $us3_link )
	49	{
[35]	50	write_log( "$me: could not connect: $dbhost, $user, $passwd, $db" );
[1]	51	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
	52	return( -1 );
	53	}
	54
	55	// First get basic info for email messages
	56	$query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
	57	"WHERE HPCAnalysisRequestID=$requestID";
[35]	58	$result = mysqli_query( $us3_link, $query );
[1]	59
	60	if ( ! $result )
	61	{
	62	write_log( "$me: Bad query: $query" );
[35]	63	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
[1]	64	return( -1 );
	65	}
	66
[35]	67	list( $email_address, $investigatorGUID, $editXMLFilename ) = mysqli_fetch_array( $result );
[1]	68
	69	$query = "SELECT personID FROM people " .
	70	"WHERE personGUID='$investigatorGUID'";
[35]	71	$result = mysqli_query( $us3_link, $query );
[1]	72
[35]	73	list( $personID ) = mysqli_fetch_array( $result );
[1]	74
[39]	75	$query = "SELECT clusterName, submitTime, queueStatus, analType " .
[1]	76	"FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
	77	"WHERE h.HPCAnalysisRequestID=$requestID " .
	78	"AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
	79
[35]	80	$result = mysqli_query( $us3_link, $query );
[1]	81
	82	if ( ! $result )
	83	{
[35]	84	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	85	return( -1 );
	86	}
	87
[35]	88	if ( mysqli_num_rows( $result ) == 0 )
[1]	89	{
	90	write_log( "$me: US3 Table error - No records for requestID: $requestID" );
	91	return( -1 );
	92	}
	93
[35]	94	list( $cluster, $submittime, $queuestatus, $jobtype ) = mysqli_fetch_array( $result );
[1]	95
	96	// Get the GFAC ID
	97	$query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
	98	"WHERE HPCAnalysisRequestID=$requestID";
	99
[35]	100	$result = mysqli_query( $us3_link, $query );
[1]	101
	102	if ( ! $result )
	103	{
	104	write_log( "$me: Bad query: $query" );
[35]	105	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
[1]	106	return( -1 );
	107	}
	108
[35]	109	list( $HPCAnalysisResultID, $gfacID ) = mysqli_fetch_array( $result );
[1]	110
[10]	111	// Get data from global GFAC DB then insert it into US3 DB
[1]	112
[35]	113	/*
	114	$result = mysqli_select_db( $gfac_link, $gDB );
[1]	115
	116	if ( ! $result )
	117	{
	118	write_log( "$me: Could not connect to DB $gDB" );
	119	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
	120	return( -1 );
	121	}
[35]	122	*/
[1]	123
	124	$query = "SELECT status, cluster, id FROM analysis " .
	125	"WHERE gfacID='$gfacID'";
	126
[35]	127	$result = mysqli_query( $gfac_link, $query );
[1]	128	if ( ! $result )
	129	{
	130	write_log( "$me: Could not select GFAC status for $gfacID" );
	131	mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
	132	return( -1 );
	133	}
	134
[35]	135	list( $status, $cluster, $id ) = mysqli_fetch_array( $result );
[1]	136
[25]	137	$is_us3iab = preg_match( "/us3iab/", $cluster );
	138	$is_local = preg_match( "/-local/", $cluster );
	139
	140	if ( $is_us3iab \|\| $is_local )
[1]	141	{
	142	$clushost = $cluster;
	143	$clushost = preg_replace( "/\-local/", "", $clushost );
	144	get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
	145	}
	146
[10]	147
	148	$query = "SELECT id FROM analysis " .
[1]	149	"WHERE gfacID='$gfacID'";
	150
[35]	151	$result = mysqli_query( $gfac_link, $query );
[1]	152
	153	if ( ! $result )
	154	{
[35]	155	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
	156	mail_to_user( "fail", "Internal error " . mysqli_error( $gfac_link ) );
[1]	157	return( -1 );
	158	}
	159
[35]	160	list( $analysisID ) = mysqli_fetch_array( $result );
[1]	161
[10]	162	// Get the request guid (LIMS submit dir name)
	163	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	164	"WHERE HPCAnalysisRequestID = $requestID ";
[35]	165	$result = mysqli_query( $us3_link, $query );
[10]	166
	167	if ( ! $result )
	168	{
[35]	169	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[10]	170	}
	171
[35]	172	list( $requestGUID ) = mysqli_fetch_array( $result );
[10]	173	$output_dir = "$submit_dir/$requestGUID";
	174
	175	// Get stderr,stdout,tarfile from work directory
	176	if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
	177	chdir( "$output_dir" );
	178	//write_log( "$me: gfacID=$gfacID" );
	179	//write_log( "$me: submit_dir=$submit_dir" );
	180	//write_log( "$me: requestGUID=$requestGUID" );
	181	write_log( "$me: output_dir=$output_dir" );
	182
	183	$stderr = "";
	184	$stdout = "";
	185	$tarfile = "";
	186	$fn_stderr = "Ultrascan.stderr";
	187	$fn_stdout = "Ultrascan.stdout";
	188	$fn_tarfile = "analysis-results.tar";
[27]	189	$secwait = 10;
[10]	190	$num_try = 0;
[26]	191	write_log( "$me: fn_tarfile=$fn_tarfile" );
[28]	192	while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
[10]	193	{
[27]	194	sleep( $secwait );
[10]	195	$num_try++;
[27]	196	$secwait *= 2;
[25]	197	write_log( "$me: tar-exists: num_try=$num_try" );
[10]	198	}
	199
	200	$ofiles = scandir( $output_dir );
	201	foreach ( $ofiles as $ofile )
	202	{
[27]	203	if ( preg_match( "/^.*stderr$/", $ofile ) )
[10]	204	$fn_stderr = $ofile;
[27]	205	if ( preg_match( "/^.*stdout$/", $ofile ) )
[10]	206	$fn_stdout = $ofile;
	207	//write_log( "$me: ofile=$ofile" );
	208	}
	209	write_log( "$me: fn_stderr=$fn_stderr" );
	210	write_log( "$me: fn_stdout=$fn_stdout" );
	211	if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
	212	else write_log( "$me: NOT FOUND: $fn_tarfile" );
	213
[26]	214	$stderr = '';
	215	$stdout = '';
	216	$tarfile = '';
[10]	217	if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
	218	if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
	219	if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
[31]	220	write_log( "$me(0): length contents stderr,stdout,tarfile -- "
	221	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
[26]	222	// If stdout,stderr have no content, retry after delay
	223	if ( strlen( $stdout ) == 0 \|\| strlen( $stderr ) == 0 )
	224	{
	225	sleep( 20 );
	226	if ( file_exists( $fn_stderr ) )
	227	$stderr = file_get_contents( $fn_stderr );
	228	if ( file_exists( $fn_stdout ) )
	229	$stdout = file_get_contents( $fn_stdout );
	230	}
[10]	231
[26]	232	write_log( "$me: length contents stderr,stdout,tarfile -- "
	233	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
	234
[10]	235	if ( $cluster == 'alamo' \|\| $cluster == 'alamo-local' )
	236	{ // Filter "ipath_userinit" lines out of alamo stdout lines
	237	$prefln = strlen( $stdout );
	238	$output = array();
	239	exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
	240	$stdout = implode( "\n", $output );
	241	$posfln = strlen( $stdout );
	242	write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
	243	}
	244
[1]	245	// Save queue messages for post-mortem analysis
	246	$query = "SELECT message, time FROM queue_messages " .
	247	"WHERE analysisID = $analysisID " .
	248	"ORDER BY time ";
[35]	249	$result = mysqli_query( $gfac_link, $query );
[1]	250
	251	if ( ! $result )
	252	{
	253	// Just log it and continue
[35]	254	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	255	}
	256
	257	$now = date( 'Y-m-d H:i:s' );
	258	$message_log = "US3 DB: $db\n" .
	259	"RequestID: $requestID\n" .
	260	"GFAC ID: $gfacID\n" .
	261	"Processed: $now\n\n" .
	262	"Queue Messages\n\n" ;
[35]	263	if ( mysqli_num_rows( $result ) > 0 )
[1]	264	{
[35]	265	while ( list( $message, $time ) = mysqli_fetch_array( $result ) )
[1]	266	$message_log .= "$time $message\n";
	267	}
	268
	269	$query = "DELETE FROM queue_messages " .
	270	"WHERE analysisID = $analysisID ";
	271
[35]	272	$result = mysqli_query( $gfac_link, $query );
[1]	273
	274	if ( ! $result )
	275	{
	276	// Just log it and continue
[35]	277	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	278	}
	279
[10]	280	$query = "SELECT queue_msg FROM analysis " .
	281	"WHERE gfacID='$gfacID' ";
	282
[35]	283	$result = mysqli_query( $gfac_link, $query );
	284	list( $queue_msg ) = mysqli_fetch_array( $result );
[1]	285
	286	// But let's allow for investigation of other large stdout and/or stderr
	287	if ( strlen( $stdout ) > 20480000 \|\|
	288	strlen( $stderr ) > 20480000 )
	289	write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
	290
	291	$message_log .= "\n\n\nStdout Contents\n\n" .
	292	$stdout .
	293	"\n\n\nStderr Contents\n\n" .
	294	$stderr .
	295	"\n\n\nGFAC Status: $status\n" .
	296	"GFAC message field: $queue_msg\n";
	297
	298	// Delete data from GFAC DB
	299	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
	300
[35]	301	$result = mysqli_query( $gfac_link, $query );
[1]	302
	303	if ( ! $result )
	304	{
	305	// Just log it and continue
[35]	306	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	307	}
	308
	309
	310	// Try to create it if necessary, and write the file
	311	// Let's use FILE_APPEND, in case this is the second time around and the
	312	// GFAC job status was INSERTed, rather than UPDATEd
	313	if ( ! is_dir( $output_dir ) )
	314	mkdir( $output_dir, 0775, true );
	315	$message_filename = "$output_dir/$db-$requestID-messages.txt";
	316	file_put_contents( $message_filename, $message_log, FILE_APPEND );
[35]	317	// mysqli_close( $gfac_link );
[1]	318
	319	/////////
	320	// Insert data into HPCAnalysis
	321
	322	$query = "UPDATE HPCAnalysisResult SET " .
[35]	323	"stderr='" . mysqli_real_escape_string( $us3_link, $stderr ) . "', " .
	324	"stdout='" . mysqli_real_escape_string( $us3_link, $stdout ) . "', " .
[10]	325	"queueStatus='completed' " .
[1]	326	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	327
[35]	328	$result = mysqli_query( $us3_link, $query );
[1]	329
	330	if ( ! $result )
	331	{
[35]	332	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	333	mail_to_user( "fail", "Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	334	return( -1 );
	335	}
	336
[10]	337	// Delete data from GFAC DB
	338	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
[1]	339
[35]	340	$result = mysqli_query( $gfac_link, $query );
[10]	341
	342	if ( ! $result )
	343	{
	344	// Just log it and continue
[35]	345	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[10]	346	}
	347
	348	// Expand the tar file
	349
[1]	350	if ( strlen( $tarfile ) == 0 )
	351	{
	352	write_log( "$me: No tarfile" );
	353	mail_to_user( "fail", "No results" );
	354	return( -1 );
	355	}
	356
	357	$tar_out = array();
[10]	358	exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
[1]	359
	360	// Insert the model files and noise files
	361	$files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
	362	$noiseIDs = array();
	363	$modelGUIDs = array();
[17]	364	$mrecsIDs = array();
	365	$fns_used = array();
[1]	366
	367	foreach ( $files as $file )
	368	{
	369	$split = explode( ";", $file );
	370
	371	if ( count( $split ) > 1 )
	372	{
	373	list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
	374
	375	list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
	376	list( $other, $variance ) = explode( "=", $variance );
	377	list( $other, $meniscus ) = explode( "=", $meniscus );
	378	}
	379	else
	380	$fn = $file;
	381
[10]	382	if ( preg_match( "/mdl.tmp$/", $fn ) )
	383	continue;
	384
[17]	385	if ( in_array( $fn, $fns_used ) )
	386	continue;
	387
	388	$fns_used[] = $fn;
	389
[1]	390	if ( filesize( $fn ) < 100 )
	391	{
[10]	392	write_log( "$me:fn is invalid $fn size filesize($fn)" );
[1]	393	mail_to_user( "fail", "Internal error\n$fn is invalid" );
	394	return( -1 );
	395	}
	396
	397	if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
	398	{
	399	$xml = file_get_contents( $fn );
	400	$statistics = parse_xml( $xml, 'statistics' );
[10]	401	// $ntries = 0;
	402	//
	403	// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
	404	// { // job_statistics file not totally copied, so retry
	405	// sleep( 10 );
	406	// $xml = file_get_contents( $fn );
	407	// $statistics = parse_xml( $xml, 'statistics' );
	408	// $ntries++;
	409	//write_log( "$me:jobstats retry $ntries" );
	410	// }
	411	//write_log( "$me:cputime=$statistics['cputime']" );
	412
[1]	413	$otherdata = parse_xml( $xml, 'id' );
	414
	415	$query = "UPDATE HPCAnalysisResult SET " .
	416	"wallTime = {$statistics['walltime']}, " .
	417	"CPUTime = {$statistics['cputime']}, " .
	418	"CPUCount = {$statistics['cpucount']}, " .
	419	"max_rss = {$statistics['maxmemory']}, " .
	420	"startTime = '{$otherdata['starttime']}', " .
	421	"endTime = '{$otherdata['endtime']}', " .
	422	"mgroupcount = {$otherdata['groupcount']} " .
	423	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
[35]	424	$result = mysqli_query( $us3_link, $query );
[1]	425
	426	if ( ! $result )
	427	{
[35]	428	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	429	}
	430
	431	file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
	432
[24]	433	$file_type = "job_stats";
	434	$id = 1;
[1]	435	}
	436
	437	else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
	438	{
	439	$xml = file_get_contents( $fn );
	440	$noise_data = parse_xml( $xml, "noise" );
	441	$type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
	442	$desc = $noise_data[ 'description' ];
	443	$modelGUID = $noise_data[ 'modelGUID' ];
	444	$noiseGUID = $noise_data[ 'noiseGUID' ];
[35]	445	$editGUID = '00000000-0000-0000-0000-000000000000';
	446	if ( isset( $model_data[ 'editGUID' ] ) )
	447	$editGUID = $model_data[ 'editGUID' ];
[1]	448
	449	$query = "INSERT INTO noise SET " .
	450	"noiseGUID='$noiseGUID'," .
	451	"modelGUID='$modelGUID'," .
[35]	452	"editedDataID=" .
	453	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
[1]	454	"modelID=1, " .
	455	"noiseType='$type'," .
	456	"description='$desc'," .
[35]	457	"xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
[1]	458
	459	// Add later after all files are processed: editDataID, modelID
	460
[35]	461	$result = mysqli_query( $us3_link, $query );
[1]	462
	463	if ( ! $result )
	464	{
[35]	465	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	466	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[1]	467	return( -1 );
	468	}
	469
[35]	470	$id = mysqli_insert_id( $us3_link );
[1]	471	$file_type = "noise";
	472	$noiseIDs[] = $id;
	473
	474	// Keep track of modelGUIDs for later, when we replace them
	475	$modelGUIDs[ $id ] = $modelGUID;
	476
	477	}
[10]	478
	479	else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
[1]	480	{
	481	$xml = file_get_contents( $fn );
[10]	482	$mrecs_data = parse_xml( $xml, "modelrecords" );
	483	$desc = $mrecs_data[ 'description' ];
	484	$editGUID = $mrecs_data[ 'editGUID' ];
	485	write_log( "$me: mrecs file editGUID=$editGUID" );
	486	if ( strlen( $editGUID ) < 36 )
	487	$editGUID = "12345678-0123-5678-0123-567890123456";
	488	$mrecGUID = $mrecs_data[ 'mrecGUID' ];
	489	$modelGUID = $mrecs_data[ 'modelGUID' ];
	490
	491	$query = "INSERT INTO pcsa_modelrecs SET " .
	492	"editedDataID=" .
	493	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	494	"modelID=0, " .
	495	"mrecsGUID='$mrecGUID'," .
	496	"description='$desc'," .
[35]	497	"xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
[10]	498
	499	// Add later after all files are processed: editDataID, modelID
	500
[35]	501	$result = mysqli_query( $us3_link, $query );
[10]	502
	503	if ( ! $result )
	504	{
[35]	505	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	506	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[10]	507	return( -1 );
	508	}
	509
[35]	510	$id = mysqli_insert_id( $us3_link );
[10]	511	$file_type = "mrecs";
	512	$mrecsIDs[] = $id;
	513
	514	// Keep track of modelGUIDs for later, when we replace them
	515	$rmodlGUIDs[ $id ] = $modelGUID;
	516	//write_log( "$me: mrecs file inserted into DB : id=$id" );
	517	}
	518
[23]	519	else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
[10]	520	{
	521	$xml = file_get_contents( $fn );
[1]	522	$model_data = parse_xml( $xml, "model" );
	523	$description = $model_data[ 'description' ];
	524	$modelGUID = $model_data[ 'modelGUID' ];
	525	$editGUID = $model_data[ 'editGUID' ];
	526
[10]	527	if ( $mc_iteration > 1 )
	528	{
	529	$miter = sprintf( "_mcN%03d", $mc_iteration );
	530	$description = preg_replace( "/_mc[0-9]+/", $miter, $description );
	531	write_log( "$me: MODELUpd: O:description=$description" );
	532	}
	533
[1]	534	$query = "INSERT INTO model SET " .
	535	"modelGUID='$modelGUID'," .
	536	"editedDataID=" .
	537	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	538	"description='$description'," .
	539	"MCIteration='$mc_iteration'," .
	540	"meniscus='$meniscus'," .
	541	"variance='$variance'," .
[35]	542	"xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
[1]	543
[35]	544	$result = mysqli_query( $us3_link, $query );
[1]	545
	546	if ( ! $result )
	547	{
[35]	548	write_log( "$me: Bad query:\n$query " . mysqli_error( $us3_link ) );
	549	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[1]	550	return( -1 );
	551	}
	552
[35]	553	$modelID = mysqli_insert_id( $us3_link );
[1]	554	$id = $modelID;
	555	$file_type = "model";
	556
	557	$query = "INSERT INTO modelPerson SET " .
	558	"modelID=$modelID, personID=$personID";
[35]	559	$result = mysqli_query( $us3_link, $query );
[1]	560	}
	561
[24]	562	else // Undetermined type: skip result data update
	563	continue;
	564
[1]	565	$query = "INSERT INTO HPCAnalysisResultData SET " .
	566	"HPCAnalysisResultID='$HPCAnalysisResultID', " .
	567	"HPCAnalysisResultType='$file_type', " .
	568	"resultID=$id";
	569
[35]	570	$result = mysqli_query( $us3_link, $query );
[1]	571
	572	if ( ! $result )
	573	{
[35]	574	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	575	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[1]	576	return( -1 );
	577	}
	578	}
	579
	580	// Now fix up noise entries
	581	// For noise files, there is, at most two: ti_noise and ri_noise
	582	// In this case there will only be one modelID
	583
	584	foreach ( $noiseIDs as $noiseID )
	585	{
	586	$modelGUID = $modelGUIDs[ $noiseID ];
	587	$query = "UPDATE noise SET " .
	588	"editedDataID=" .
[6]	589	"(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
[1]	590	"modelID=" .
	591	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	592	"WHERE noiseID=$noiseID";
	593
[35]	594	$result = mysqli_query( $us3_link, $query );
[1]	595
	596	if ( ! $result )
	597	{
[35]	598	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	599	mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
[1]	600	return( -1 );
	601	}
	602	}
	603
[13]	604	// Now possibly fix up mrecs entries
	605
	606	foreach ( $mrecsIDs as $mrecsID )
	607	{
	608	$modelGUID = $rmodlGUIDs[ $mrecsID ];
	609	$query = "UPDATE pcsa_modelrecs SET " .
	610	"modelID=" .
	611	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	612	"WHERE mrecsID=$mrecsID";
	613
[35]	614	$result = mysqli_query( $us3_link, $query );
[13]	615
	616	if ( ! $result )
	617	{
[35]	618	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	619	mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
[13]	620	return( -1 );
	621	}
	622	write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
	623	}
	624	//write_log( "$me: mrecs entries updated" );
	625
[1]	626	// Copy results to LIMS submit directory (files there are deleted after 7 days)
	627	global $submit_dir; // LIMS submit files dir
	628
	629	// Get the request guid (LIMS submit dir name)
	630	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	631	"WHERE HPCAnalysisRequestID = $requestID ";
[35]	632	$result = mysqli_query( $us3_link, $query );
[1]	633
	634	if ( ! $result )
	635	{
[35]	636	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	637	}
	638
[35]	639	// list( $requestGUID ) = mysqli_fetch_array( $result );
[10]	640	//
	641	// chdir( "$submit_dir/$requestGUID" );
	642	// $f = fopen( "analysis-results.tar", "w" );
	643	// fwrite( $f, $tarfile );
	644	// fclose( $f );
[1]	645
	646	// Clean up
[10]	647	// chdir ( $work );
[1]	648	// exec( "rm -rf $gfacID" );
	649
[35]	650	mysqli_close( $us3_link );
[1]	651
	652	/////////
	653	// Send email
	654
	655	mail_to_user( "success", "" );
	656	}
	657
	658	function mail_to_user( $type, $msg )
	659	{
	660	// Note to me. Just changed subject line to include a modified $status instead
	661	// of the $type variable passed. More informative than just "fail" or "success."
	662	// See how it works for awhile and then consider removing $type parameter from
	663	// function.
	664	global $email_address;
	665	global $submittime;
	666	global $queuestatus;
	667	global $status;
	668	global $cluster;
	669	global $jobtype;
	670	global $org_name;
	671	global $admin_email;
[6]	672	global $db;
[1]	673	global $dbhost;
	674	global $requestID;
	675	global $gfacID;
	676	global $editXMLFilename;
[5]	677	global $stdout;
[25]	678	global $org_domain;
[1]	679
	680	global $me;
	681	write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
	682
	683	// Get GFAC status and message
[4]	684	// function get_gfac_message() also sets global $status
	685	$gfac_message = get_gfac_message( $gfacID );
[10]	686	if ( $gfac_message === false ) $gfac_message = "Job Finished";
[1]	687
	688	// Create a status to put in the subject line
	689	switch ( $status )
	690	{
	691	case "COMPLETE":
	692	$subj_status = 'completed';
	693	break;
	694
	695	case "CANCELLED":
	696	case "CANCELED":
	697	$subj_status = 'canceled';
	698	break;
	699
	700	case "FAILED":
	701	$subj_status = 'failed';
[10]	702	if ( preg_match( "/^US3-A/i", $gfacID ) )
	703	{ // For A/Thrift FAIL, get error message
	704	$gfac_message = getExperimentErrors( $gfacID );
	705	//$gfac_message .= "Test ERROR MESSAGE";
	706	}
[1]	707	break;
	708
	709	case "ERROR":
	710	$subj_status = 'unknown error';
	711	break;
	712
	713	default:
	714	$subj_status = $status; // For now
	715	break;
	716
	717	}
	718
[10]	719	$queuestatus = $subj_status;
	720	$limshost = $dbhost;
	721	if ( $limshost == 'localhost' )
[25]	722	{
[10]	723	$limshost = gethostname();
[35]	724	if ( preg_match( "/scyld/", $limshost ) )
	725	$limshost = 'alamo.uthscsa.edu';
[37]	726	else if ( preg_match( "/novalo/", $limshost ) )
	727	$limshost = 'uslims3.aucsolutions.com';
[35]	728	else if ( ! preg_match( "/\./", $limshost ) )
[25]	729	$limshost = $limshost . $org_domain;
	730	}
[10]	731
[1]	732	// Parse the editXMLFilename
	733	list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
	734	explode( ".", $editXMLFilename );
	735
	736	$headers = "From: $org_name Admin<$admin_email>" . "\n";
	737	$headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
	738
	739	// Set the reply address
	740	$headers .= "Reply-To: $org_name<$admin_email>" . "\n";
	741	$headers .= "Return-Path: $org_name<$admin_email>" . "\n";
	742
	743	// Try to avoid spam filters
	744	$now = time();
	745	$headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
	746	$headers .= "X-Mailer: PHP v" . phpversion() . "\n";
	747	$headers .= "MIME-Version: 1.0" . "\n";
	748	$headers .= "Content-Transfer-Encoding: 8bit" . "\n";
	749
	750	$subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
	751	$message = "
	752	Your UltraScan job is complete:
	753
	754	Submission Time : $submittime
[10]	755	LIMS Host : $limshost
[1]	756	Analysis ID : $gfacID
[6]	757	Request ID : $requestID ( $db )
[1]	758	RunID : $runID
	759	EditID : $editID
	760	Data Type : $dataType
	761	Cell/Channel/Wl : $cell / $channel / $wl
	762	Status : $queuestatus
	763	Cluster : $cluster
	764	Job Type : $jobtype
	765	GFAC Status : $status
[4]	766	GFAC Message : $gfac_message
[5]	767	Stdout : $stdout
[1]	768	";
	769
[4]	770	if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
[1]	771
	772	// Handle the error case where an error occurs before fetching the
	773	// user's email address
	774	if ( $email_address == "" ) $email_address = $admin_email;
	775
	776	mail( $email_address, $subject, $message, $headers );
	777	}
	778
	779	function parse_xml( $xml, $type )
	780	{
	781	$parser = new XMLReader();
	782	$parser->xml( $xml );
	783
	784	$results = array();
	785
	786	while ( $parser->read() )
	787	{
	788	if ( $parser->name == $type )
	789	{
	790	while ( $parser->moveToNextAttribute() )
	791	{
	792	$results[ $parser->name ] = $parser->value;
	793	}
	794
	795	break;
	796	}
	797	}
	798
	799	$parser->close();
	800	return $results;
	801	}
	802
	803	// Function to get information about the current job GFAC
	804	function get_gfac_message( $gfacID )
	805	{
	806	global $serviceURL;
	807	global $me;
	808
	809	$hex = "[0-9a-fA-F]";
	810	if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
	811	! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
	812	{
	813	// Then it's not a GFAC job
	814	return false;
	815	}
	816
	817	$url = "$serviceURL/jobstatus/$gfacID";
	818	try
	819	{
	820	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	821	$http = $post->send();
	822	$xml = $post->getResponseBody();
	823	}
	824	catch ( HttpException $e )
	825	{
	826	write_log( "$me: Job status not available - $gfacID" );
	827	return false;
	828	}
	829
	830	// Parse the result
	831	$gfac_message = parse_message( $xml );
	832
	833	return $gfac_message;
	834	}
	835
	836	function parse_message( $xml )
	837	{
[4]	838	global $status;
[1]	839	$status = "";
	840	$gfac_message = "";
	841
	842	$parser = new XMLReader();
	843	$parser->xml( $xml );
	844
	845	$results = array();
	846
	847	while( $parser->read() )
	848	{
	849	$type = $parser->nodeType;
	850
	851	if ( $type == XMLReader::ELEMENT )
	852	$name = $parser->name;
	853
	854	else if ( $type == XMLReader::TEXT )
	855	{
	856	if ( $name == "status" )
	857	$status = $parser->value;
	858	else
	859	$gfac_message = $parser->value;
	860	}
	861	}
	862
	863	$parser->close();
	864	return $gfac_message;
	865	}
	866
	867	function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
	868	{
	869	global $work;
	870	global $work_remote;
	871	global $me;
	872	global $db;
[35]	873	global $dbhost;
[1]	874	global $status;
[25]	875	$is_us3iab = preg_match( "/us3iab/", $cluster );
[31]	876	$is_jetstr = preg_match( "/jetstream/", $cluster );
[35]	877	$limshost = $dbhost;
	878	$stderr = '';
	879	$stdout = '';
	880	$tarfile = '';
[1]	881
[35]	882	if ( $limshost == 'localhost' )
	883	{ // If DB host is local host, get full LIMS host name
	884	$limshost = gethostname();
	885	if ( preg_match( "/scyld/", $limshost ) )
	886	$limshost = 'alamo.uthscsa.edu';
[37]	887	else if ( preg_match( "/novalo/", $limshost ) )
	888	$limshost = 'uslims3.aucsolutions.com';
[35]	889	else if ( ! preg_match( "/\./", $limshost ) )
	890	$limshost = $limshost . $org_domain;
	891	}
	892
	893	if ( preg_match( "/alamo/", $limshost ) &&
	894	preg_match( "/alamo/", $cluster ) )
	895	{ // If both LIMS and cluster are alamo, set up local transfers
	896	$is_us3iab = 1;
	897	if ( ! preg_match( "/\/local/", $work_remote ) )
	898	$work_remote = $work_remote . "/local";
	899	}
	900
	901	// Figure out job's remote (or local) work directory
[1]	902	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
[28]	903	//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
[1]	904
	905	// Get stdout, stderr, output/analysis-results.tar
	906	$output = array();
	907
[25]	908	if ( $is_us3iab == 0 )
	909	{
[28]	910	// For "-local", recompute remote work directory
[31]	911	$clushost = "$cluster.uthscsa.edu";
	912	$lworkdir = "~us3/lims/work/local";
	913	if ( $is_jetstr )
	914	{
[32]	915	$clushost = "js-169-137.jetstream-cloud.org";
[31]	916	$lworkdir = "/N/us3_cluster/work/local";
	917	}
[35]	918	$cmd = "ssh us3@$clushost 'ls -d $lworkdir' 2>/dev/null";
[28]	919	exec( $cmd, $output, $stat );
	920	$work_remote = $output[ 0 ];
	921	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
	922	write_log( "$me: -LOCAL: remoteDir=$remoteDir" );
	923
[25]	924	// Figure out local working directory
	925	if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
	926	$pwd = chdir( "$work/$gfacID" );
[1]	927
[31]	928	$cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1";
[25]	929
[1]	930	exec( $cmd, $output, $stat );
[25]	931	if ( $stat != 0 )
[1]	932	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	933
[31]	934	$cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1";
[25]	935
[1]	936	exec( $cmd, $output, $stat );
[25]	937	if ( $stat != 0 )
	938	{
[1]	939	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	940	sleep( 10 );
	941	write_log( "$me: RETRY" );
	942	exec( $cmd, $output, $stat );
	943	if ( $stat != 0 )
	944	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	945	}
	946
[31]	947	$cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1";
[25]	948
	949	exec( $cmd, $output, $stat );
	950	if ( $stat != 0 )
	951	{
	952	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	953	sleep( 10 );
	954	write_log( "$me: RETRY" );
	955	exec( $cmd, $output, $stat );
	956	if ( $stat != 0 )
	957	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	958	}
[1]	959	}
[25]	960	else
[35]	961	{ // Is US3IAB or alamo-to-alamo, so just change to local work directory
[25]	962	$pwd = chdir( "$remoteDir" );
	963	write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
	964	}
[1]	965
[25]	966
[1]	967	// Write the files to gfacDB
	968
[35]	969	$secwait = 10;
	970	$num_try = 0;
	971	while ( ! file_exists( "stderr" ) && $num_try < 3 )
	972	{ // Do waits and retries to let stderr appear
	973	sleep( $secwait );
	974	$num_try++;
	975	$secwait *= 2;
	976	write_log( "$me: not-exist-stderr: num_try=$num_try" );
	977	}
	978
	979	$lense = 0;
[25]	980	if ( file_exists( "stderr" ) )
[35]	981	{
	982	$lense = filesize( "stderr" );
	983	if ( $lense > 1000000 )
	984	{ // Replace exceptionally large stderr with smaller version
	985	exec( "mv stderr stderr-orig", $output, $stat );
	986	exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
	987	exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
	988	exec( "cat stderr-h stderr-t >stderr", $output, $stat );
	989	}
[25]	990	$stderr = file_get_contents( "stderr" );
	991	}
	992	else
[35]	993	{
[25]	994	$stderr = "";
[35]	995	}
	996
[1]	997	if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
[35]	998
[25]	999	$fn1_tarfile = "analysis-results.tar";
	1000	$fn2_tarfile = "output/" . $fn1_tarfile;
	1001	if ( file_exists( $fn1_tarfile ) )
	1002	$tarfile = file_get_contents( $fn1_tarfile );
	1003	else if ( file_exists( $fn2_tarfile ) )
	1004	$tarfile = file_get_contents( $fn2_tarfile );
[1]	1005
[35]	1006	// $lense = strlen( $stderr );
	1007	// if ( $lense > 1000000 )
	1008	// { // Replace exceptionally large stderr with smaller version
	1009	// exec( "mv stderr stderr-orig", $output, $stat );
	1010	// exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
	1011	// exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
	1012	// exec( "cat stderr-h stderr-t >stderr", $output, $stat );
	1013	// $stderr = file_get_contents( "stderr" );
	1014	// }
[25]	1015	$lene = strlen( $stderr );
	1016	write_log( "$me: stderr size: $lene (was $lense)");
	1017	$leno = strlen( $stdout );
	1018	write_log( "$me: stdout size: $leno");
[35]	1019	$lent = strlen( $tarfile );
	1020	write_log( "$me: tarfile size: $lent");
	1021	$esstde = mysqli_real_escape_string( $gfac_link, $stderr );
	1022	$esstdo = mysqli_real_escape_string( $gfac_link, $stdout );
	1023	$estarf = mysqli_real_escape_string( $gfac_link, $tarfile );
	1024	$lene = strlen($esstde);
	1025	write_log( "$me: es-stderr size: $lene");
	1026	$leno = strlen($esstdo);
	1027	write_log( "$me: es-stdout size: $leno");
[25]	1028	$lenf = strlen($estarf);
[35]	1029	write_log( "$me: es-tarfile size: $lenf");
[1]	1030	$query = "UPDATE analysis SET " .
[35]	1031	"stderr='" . $esstde . "'," .
	1032	"stdout='" . $esstdo . "'," .
	1033	"tarfile='" . $estarf . "'";
[1]	1034
[35]	1035	$result = mysqli_query( $gfac_link, $query );
[1]	1036
	1037	if ( ! $result )
	1038	{
[35]	1039	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	1040	echo "Bad query\n";
	1041	return( -1 );
	1042	}
	1043	}
	1044	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: