Context Navigation

source: trunk/cleanup.php@ 31

Last change on this file since 31 was 31, checked in by gegorbet, 7 years ago
global-fit and jetstream,stampede2 mods
File size: 31.2 KB

Rev	Line
[1]	1	<?php
	2	/*
	3	* cleanup.php
	4	*
	5	* functions relating to copying results and cleaning up the gfac DB
[10]	6	* where the job used an Airavata interface.
[1]	7	*
	8	*/
	9
	10	$email_address = '';
	11	$queuestatus = '';
	12	$jobtype = '';
	13	$db = '';
	14	$editXMLFilename = '';
[4]	15	$status = '';
[1]	16
[10]	17	function aira_cleanup( $us3_db, $reqID, $gfac_link )
[1]	18	{
[25]	19	global $org_domain;
[1]	20	global $dbhost;
	21	global $user;
	22	global $passwd;
	23	global $db;
	24	global $guser;
	25	global $gpasswd;
	26	global $gDB;
	27	global $me;
	28	global $work;
	29	global $email_address;
	30	global $queuestatus;
	31	global $jobtype;
	32	global $editXMLFilename;
	33	global $submittime;
[4]	34	global $status;
[10]	35	global $stderr;
[5]	36	global $stdout;
[10]	37	global $tarfile;
[6]	38	global $requestID;
[10]	39	global $submit_dir;
	40	$me = 'cleanup_aira.php';
[1]	41
[6]	42	$requestID = $reqID;
[1]	43	$db = $us3_db;
	44	write_log( "$me: debug db=$db; requestID=$requestID" );
	45
	46	$us3_link = mysql_connect( $dbhost, $user, $passwd );
	47
	48	if ( ! $us3_link )
	49	{
	50	write_log( "$me: could not connect: $dbhost, $user, $passwd" );
	51	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
	52	return( -1 );
	53	}
	54
	55	$result = mysql_select_db( $db, $us3_link );
	56
	57	if ( ! $result )
	58	{
	59	write_log( "$me: could not select DB $db" );
	60	mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
	61	return( -1 );
	62	}
	63
	64	// First get basic info for email messages
	65	$query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
	66	"WHERE HPCAnalysisRequestID=$requestID";
	67	$result = mysql_query( $query, $us3_link );
	68
	69	if ( ! $result )
	70	{
	71	write_log( "$me: Bad query: $query" );
	72	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
	73	return( -1 );
	74	}
	75
	76	list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
	77
	78	$query = "SELECT personID FROM people " .
	79	"WHERE personGUID='$investigatorGUID'";
	80	$result = mysql_query( $query, $us3_link );
	81
	82	list( $personID ) = mysql_fetch_array( $result );
	83
	84	$query = "SELECT clusterName, submitTime, queueStatus, method " .
	85	"FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
	86	"WHERE h.HPCAnalysisRequestID=$requestID " .
	87	"AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
	88
	89	$result = mysql_query( $query, $us3_link );
	90
	91	if ( ! $result )
	92	{
	93	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	94	return( -1 );
	95	}
	96
	97	if ( mysql_num_rows( $result ) == 0 )
	98	{
	99	write_log( "$me: US3 Table error - No records for requestID: $requestID" );
	100	return( -1 );
	101	}
	102
	103	list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
	104
	105	// Get the GFAC ID
	106	$query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
	107	"WHERE HPCAnalysisRequestID=$requestID";
	108
	109	$result = mysql_query( $query, $us3_link );
	110
	111	if ( ! $result )
	112	{
	113	write_log( "$me: Bad query: $query" );
	114	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
	115	return( -1 );
	116	}
	117
	118	list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
	119
[10]	120	// Get data from global GFAC DB then insert it into US3 DB
[1]	121
	122	$result = mysql_select_db( $gDB, $gfac_link );
	123
	124	if ( ! $result )
	125	{
	126	write_log( "$me: Could not connect to DB $gDB" );
	127	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
	128	return( -1 );
	129	}
	130
	131	$query = "SELECT status, cluster, id FROM analysis " .
	132	"WHERE gfacID='$gfacID'";
	133
	134	$result = mysql_query( $query, $gfac_link );
	135	if ( ! $result )
	136	{
	137	write_log( "$me: Could not select GFAC status for $gfacID" );
	138	mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
	139	return( -1 );
	140	}
	141
	142	list( $status, $cluster, $id ) = mysql_fetch_array( $result );
	143
[25]	144	$is_us3iab = preg_match( "/us3iab/", $cluster );
	145	$is_local = preg_match( "/-local/", $cluster );
	146
	147	if ( $is_us3iab \|\| $is_local )
[1]	148	{
	149	$clushost = $cluster;
	150	$clushost = preg_replace( "/\-local/", "", $clushost );
	151	get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
	152	}
	153
[10]	154
	155	$query = "SELECT id FROM analysis " .
[1]	156	"WHERE gfacID='$gfacID'";
	157
	158	$result = mysql_query( $query, $gfac_link );
	159
	160	if ( ! $result )
	161	{
	162	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	163	mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
	164	return( -1 );
	165	}
	166
[10]	167	list( $analysisID ) = mysql_fetch_array( $result );
[1]	168
[10]	169	// Get the request guid (LIMS submit dir name)
	170	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	171	"WHERE HPCAnalysisRequestID = $requestID ";
	172	$result = mysql_query( $query, $us3_link );
	173
	174	if ( ! $result )
	175	{
	176	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	177	}
	178
	179	list( $requestGUID ) = mysql_fetch_array( $result );
	180	$output_dir = "$submit_dir/$requestGUID";
	181
	182	// Get stderr,stdout,tarfile from work directory
	183	if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
	184	chdir( "$output_dir" );
	185	//write_log( "$me: gfacID=$gfacID" );
	186	//write_log( "$me: submit_dir=$submit_dir" );
	187	//write_log( "$me: requestGUID=$requestGUID" );
	188	write_log( "$me: output_dir=$output_dir" );
	189
	190	$stderr = "";
	191	$stdout = "";
	192	$tarfile = "";
	193	$fn_stderr = "Ultrascan.stderr";
	194	$fn_stdout = "Ultrascan.stdout";
	195	$fn_tarfile = "analysis-results.tar";
[27]	196	$secwait = 10;
[10]	197	$num_try = 0;
[26]	198	write_log( "$me: fn_tarfile=$fn_tarfile" );
[28]	199	while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
[10]	200	{
[27]	201	sleep( $secwait );
[10]	202	$num_try++;
[27]	203	$secwait *= 2;
[25]	204	write_log( "$me: tar-exists: num_try=$num_try" );
[10]	205	}
	206
	207	$ofiles = scandir( $output_dir );
	208	foreach ( $ofiles as $ofile )
	209	{
[27]	210	if ( preg_match( "/^.*stderr$/", $ofile ) )
[10]	211	$fn_stderr = $ofile;
[27]	212	if ( preg_match( "/^.*stdout$/", $ofile ) )
[10]	213	$fn_stdout = $ofile;
	214	//write_log( "$me: ofile=$ofile" );
	215	}
	216	write_log( "$me: fn_stderr=$fn_stderr" );
	217	write_log( "$me: fn_stdout=$fn_stdout" );
	218	if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
	219	else write_log( "$me: NOT FOUND: $fn_tarfile" );
	220
[26]	221	$stderr = '';
	222	$stdout = '';
	223	$tarfile = '';
[10]	224	if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
	225	if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
	226	if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
[31]	227	write_log( "$me(0): length contents stderr,stdout,tarfile -- "
	228	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
[26]	229	// If stdout,stderr have no content, retry after delay
	230	if ( strlen( $stdout ) == 0 \|\| strlen( $stderr ) == 0 )
	231	{
	232	sleep( 20 );
	233	if ( file_exists( $fn_stderr ) )
	234	$stderr = file_get_contents( $fn_stderr );
	235	if ( file_exists( $fn_stdout ) )
	236	$stdout = file_get_contents( $fn_stdout );
	237	}
[10]	238
[26]	239	write_log( "$me: length contents stderr,stdout,tarfile -- "
	240	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
	241
[10]	242	if ( $cluster == 'alamo' \|\| $cluster == 'alamo-local' )
	243	{ // Filter "ipath_userinit" lines out of alamo stdout lines
	244	$prefln = strlen( $stdout );
	245	$output = array();
	246	exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
	247	$stdout = implode( "\n", $output );
	248	$posfln = strlen( $stdout );
	249	write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
	250	}
	251
[1]	252	// Save queue messages for post-mortem analysis
	253	$query = "SELECT message, time FROM queue_messages " .
	254	"WHERE analysisID = $analysisID " .
	255	"ORDER BY time ";
	256	$result = mysql_query( $query, $gfac_link );
	257
	258	if ( ! $result )
	259	{
	260	// Just log it and continue
	261	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	262	}
	263
	264	$now = date( 'Y-m-d H:i:s' );
	265	$message_log = "US3 DB: $db\n" .
	266	"RequestID: $requestID\n" .
	267	"GFAC ID: $gfacID\n" .
	268	"Processed: $now\n\n" .
	269	"Queue Messages\n\n" ;
	270	if ( mysql_num_rows( $result ) > 0 )
	271	{
	272	while ( list( $message, $time ) = mysql_fetch_array( $result ) )
	273	$message_log .= "$time $message\n";
	274	}
	275
	276	$query = "DELETE FROM queue_messages " .
	277	"WHERE analysisID = $analysisID ";
	278
	279	$result = mysql_query( $query, $gfac_link );
	280
	281	if ( ! $result )
	282	{
	283	// Just log it and continue
	284	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	285	}
	286
[10]	287	$query = "SELECT queue_msg FROM analysis " .
	288	"WHERE gfacID='$gfacID' ";
	289
[1]	290	$result = mysql_query( $query, $gfac_link );
[10]	291	list( $queue_msg ) = mysql_fetch_array( $result );
[1]	292
	293	// But let's allow for investigation of other large stdout and/or stderr
	294	if ( strlen( $stdout ) > 20480000 \|\|
	295	strlen( $stderr ) > 20480000 )
	296	write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
	297
	298	$message_log .= "\n\n\nStdout Contents\n\n" .
	299	$stdout .
	300	"\n\n\nStderr Contents\n\n" .
	301	$stderr .
	302	"\n\n\nGFAC Status: $status\n" .
	303	"GFAC message field: $queue_msg\n";
	304
	305	// Delete data from GFAC DB
	306	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
	307
	308	$result = mysql_query( $query, $gfac_link );
	309
	310	if ( ! $result )
	311	{
	312	// Just log it and continue
	313	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	314	}
	315
	316
	317	// Try to create it if necessary, and write the file
	318	// Let's use FILE_APPEND, in case this is the second time around and the
	319	// GFAC job status was INSERTed, rather than UPDATEd
	320	if ( ! is_dir( $output_dir ) )
	321	mkdir( $output_dir, 0775, true );
	322	$message_filename = "$output_dir/$db-$requestID-messages.txt";
	323	file_put_contents( $message_filename, $message_log, FILE_APPEND );
	324	// mysql_close( $gfac_link );
	325
	326	/////////
	327	// Insert data into HPCAnalysis
	328
	329	$query = "UPDATE HPCAnalysisResult SET " .
	330	"stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
[10]	331	"stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
	332	"queueStatus='completed' " .
[1]	333	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	334
	335	$result = mysql_query( $query, $us3_link );
	336
	337	if ( ! $result )
	338	{
	339	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	340	mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
	341	return( -1 );
	342	}
	343
[10]	344	// Delete data from GFAC DB
	345	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
[1]	346
[10]	347	$result = mysql_query( $query, $gfac_link );
	348
	349	if ( ! $result )
	350	{
	351	// Just log it and continue
	352	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	353	}
	354
	355	// Expand the tar file
	356
[1]	357	if ( strlen( $tarfile ) == 0 )
	358	{
	359	write_log( "$me: No tarfile" );
	360	mail_to_user( "fail", "No results" );
	361	return( -1 );
	362	}
	363
	364	$tar_out = array();
[10]	365	exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
[1]	366
	367	// Insert the model files and noise files
	368	$files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
	369	$noiseIDs = array();
	370	$modelGUIDs = array();
[17]	371	$mrecsIDs = array();
	372	$fns_used = array();
[1]	373
	374	foreach ( $files as $file )
	375	{
	376	$split = explode( ";", $file );
	377
	378	if ( count( $split ) > 1 )
	379	{
	380	list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
	381
	382	list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
	383	list( $other, $variance ) = explode( "=", $variance );
	384	list( $other, $meniscus ) = explode( "=", $meniscus );
	385	}
	386	else
	387	$fn = $file;
	388
[10]	389	if ( preg_match( "/mdl.tmp$/", $fn ) )
	390	continue;
	391
[17]	392	if ( in_array( $fn, $fns_used ) )
	393	continue;
	394
	395	$fns_used[] = $fn;
	396
[1]	397	if ( filesize( $fn ) < 100 )
	398	{
[10]	399	write_log( "$me:fn is invalid $fn size filesize($fn)" );
[1]	400	mail_to_user( "fail", "Internal error\n$fn is invalid" );
	401	return( -1 );
	402	}
	403
	404	if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
	405	{
	406	$xml = file_get_contents( $fn );
	407	$statistics = parse_xml( $xml, 'statistics' );
[10]	408	// $ntries = 0;
	409	//
	410	// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
	411	// { // job_statistics file not totally copied, so retry
	412	// sleep( 10 );
	413	// $xml = file_get_contents( $fn );
	414	// $statistics = parse_xml( $xml, 'statistics' );
	415	// $ntries++;
	416	//write_log( "$me:jobstats retry $ntries" );
	417	// }
	418	//write_log( "$me:cputime=$statistics['cputime']" );
	419
[1]	420	$otherdata = parse_xml( $xml, 'id' );
	421
	422	$query = "UPDATE HPCAnalysisResult SET " .
	423	"wallTime = {$statistics['walltime']}, " .
	424	"CPUTime = {$statistics['cputime']}, " .
	425	"CPUCount = {$statistics['cpucount']}, " .
	426	"max_rss = {$statistics['maxmemory']}, " .
	427	"startTime = '{$otherdata['starttime']}', " .
	428	"endTime = '{$otherdata['endtime']}', " .
	429	"mgroupcount = {$otherdata['groupcount']} " .
	430	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	431	$result = mysql_query( $query, $us3_link );
	432
	433	if ( ! $result )
	434	{
	435	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	436	}
	437
	438	file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
	439
[24]	440	$file_type = "job_stats";
	441	$id = 1;
[1]	442	}
	443
	444	else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
	445	{
	446	$xml = file_get_contents( $fn );
	447	$noise_data = parse_xml( $xml, "noise" );
	448	$type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
	449	$desc = $noise_data[ 'description' ];
	450	$modelGUID = $noise_data[ 'modelGUID' ];
	451	$noiseGUID = $noise_data[ 'noiseGUID' ];
	452
	453	$query = "INSERT INTO noise SET " .
	454	"noiseGUID='$noiseGUID'," .
	455	"modelGUID='$modelGUID'," .
	456	"editedDataID=1, " .
	457	"modelID=1, " .
	458	"noiseType='$type'," .
	459	"description='$desc'," .
	460	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	461
	462	// Add later after all files are processed: editDataID, modelID
	463
	464	$result = mysql_query( $query, $us3_link );
	465
	466	if ( ! $result )
	467	{
	468	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	469	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	470	return( -1 );
	471	}
	472
	473	$id = mysql_insert_id( $us3_link );
	474	$file_type = "noise";
	475	$noiseIDs[] = $id;
	476
	477	// Keep track of modelGUIDs for later, when we replace them
	478	$modelGUIDs[ $id ] = $modelGUID;
	479
	480	}
[10]	481
	482	else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
[1]	483	{
	484	$xml = file_get_contents( $fn );
[10]	485	$mrecs_data = parse_xml( $xml, "modelrecords" );
	486	$desc = $mrecs_data[ 'description' ];
	487	$editGUID = $mrecs_data[ 'editGUID' ];
	488	write_log( "$me: mrecs file editGUID=$editGUID" );
	489	if ( strlen( $editGUID ) < 36 )
	490	$editGUID = "12345678-0123-5678-0123-567890123456";
	491	$mrecGUID = $mrecs_data[ 'mrecGUID' ];
	492	$modelGUID = $mrecs_data[ 'modelGUID' ];
	493
	494	$query = "INSERT INTO pcsa_modelrecs SET " .
	495	"editedDataID=" .
	496	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	497	"modelID=0, " .
	498	"mrecsGUID='$mrecGUID'," .
	499	"description='$desc'," .
	500	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	501
	502	// Add later after all files are processed: editDataID, modelID
	503
	504	$result = mysql_query( $query, $us3_link );
	505
	506	if ( ! $result )
	507	{
	508	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	509	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	510	return( -1 );
	511	}
	512
	513	$id = mysql_insert_id( $us3_link );
	514	$file_type = "mrecs";
	515	$mrecsIDs[] = $id;
	516
	517	// Keep track of modelGUIDs for later, when we replace them
	518	$rmodlGUIDs[ $id ] = $modelGUID;
	519	//write_log( "$me: mrecs file inserted into DB : id=$id" );
	520	}
	521
[23]	522	else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
[10]	523	{
	524	$xml = file_get_contents( $fn );
[1]	525	$model_data = parse_xml( $xml, "model" );
	526	$description = $model_data[ 'description' ];
	527	$modelGUID = $model_data[ 'modelGUID' ];
	528	$editGUID = $model_data[ 'editGUID' ];
	529
[10]	530	if ( $mc_iteration > 1 )
	531	{
	532	$miter = sprintf( "_mcN%03d", $mc_iteration );
	533	$description = preg_replace( "/_mc[0-9]+/", $miter, $description );
	534	write_log( "$me: MODELUpd: O:description=$description" );
	535	}
	536
[1]	537	$query = "INSERT INTO model SET " .
	538	"modelGUID='$modelGUID'," .
	539	"editedDataID=" .
	540	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	541	"description='$description'," .
	542	"MCIteration='$mc_iteration'," .
	543	"meniscus='$meniscus'," .
	544	"variance='$variance'," .
	545	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	546
	547	$result = mysql_query( $query, $us3_link );
	548
	549	if ( ! $result )
	550	{
	551	write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
	552	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	553	return( -1 );
	554	}
	555
	556	$modelID = mysql_insert_id( $us3_link );
	557	$id = $modelID;
	558	$file_type = "model";
	559
	560	$query = "INSERT INTO modelPerson SET " .
	561	"modelID=$modelID, personID=$personID";
	562	$result = mysql_query( $query, $us3_link );
	563	}
	564
[24]	565	else // Undetermined type: skip result data update
	566	continue;
	567
[1]	568	$query = "INSERT INTO HPCAnalysisResultData SET " .
	569	"HPCAnalysisResultID='$HPCAnalysisResultID', " .
	570	"HPCAnalysisResultType='$file_type', " .
	571	"resultID=$id";
	572
	573	$result = mysql_query( $query, $us3_link );
	574
	575	if ( ! $result )
	576	{
	577	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	578	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	579	return( -1 );
	580	}
	581	}
	582
	583	// Now fix up noise entries
	584	// For noise files, there is, at most two: ti_noise and ri_noise
	585	// In this case there will only be one modelID
	586
	587	foreach ( $noiseIDs as $noiseID )
	588	{
	589	$modelGUID = $modelGUIDs[ $noiseID ];
	590	$query = "UPDATE noise SET " .
	591	"editedDataID=" .
[6]	592	"(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
[1]	593	"modelID=" .
	594	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	595	"WHERE noiseID=$noiseID";
	596
	597	$result = mysql_query( $query, $us3_link );
	598
	599	if ( ! $result )
	600	{
	601	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	602	mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
	603	return( -1 );
	604	}
	605	}
	606
[13]	607	// Now possibly fix up mrecs entries
	608
	609	foreach ( $mrecsIDs as $mrecsID )
	610	{
	611	$modelGUID = $rmodlGUIDs[ $mrecsID ];
	612	$query = "UPDATE pcsa_modelrecs SET " .
	613	"modelID=" .
	614	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	615	"WHERE mrecsID=$mrecsID";
	616
	617	$result = mysql_query( $query, $us3_link );
	618
	619	if ( ! $result )
	620	{
	621	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	622	mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
	623	return( -1 );
	624	}
	625	write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
	626	}
	627	//write_log( "$me: mrecs entries updated" );
	628
[1]	629	// Copy results to LIMS submit directory (files there are deleted after 7 days)
	630	global $submit_dir; // LIMS submit files dir
	631
	632	// Get the request guid (LIMS submit dir name)
	633	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	634	"WHERE HPCAnalysisRequestID = $requestID ";
	635	$result = mysql_query( $query, $us3_link );
	636
	637	if ( ! $result )
	638	{
	639	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	640	}
	641
[10]	642	// list( $requestGUID ) = mysql_fetch_array( $result );
	643	//
	644	// chdir( "$submit_dir/$requestGUID" );
	645	// $f = fopen( "analysis-results.tar", "w" );
	646	// fwrite( $f, $tarfile );
	647	// fclose( $f );
[1]	648
	649	// Clean up
[10]	650	// chdir ( $work );
[1]	651	// exec( "rm -rf $gfacID" );
	652
	653	mysql_close( $us3_link );
	654
	655	/////////
	656	// Send email
	657
	658	mail_to_user( "success", "" );
	659	}
	660
	661	function mail_to_user( $type, $msg )
	662	{
	663	// Note to me. Just changed subject line to include a modified $status instead
	664	// of the $type variable passed. More informative than just "fail" or "success."
	665	// See how it works for awhile and then consider removing $type parameter from
	666	// function.
	667	global $email_address;
	668	global $submittime;
	669	global $queuestatus;
	670	global $status;
	671	global $cluster;
	672	global $jobtype;
	673	global $org_name;
	674	global $admin_email;
[6]	675	global $db;
[1]	676	global $dbhost;
	677	global $requestID;
	678	global $gfacID;
	679	global $editXMLFilename;
[5]	680	global $stdout;
[25]	681	global $org_domain;
[1]	682
	683	global $me;
	684	write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
	685
	686	// Get GFAC status and message
[4]	687	// function get_gfac_message() also sets global $status
	688	$gfac_message = get_gfac_message( $gfacID );
[10]	689	if ( $gfac_message === false ) $gfac_message = "Job Finished";
[1]	690
	691	// Create a status to put in the subject line
	692	switch ( $status )
	693	{
	694	case "COMPLETE":
	695	$subj_status = 'completed';
	696	break;
	697
	698	case "CANCELLED":
	699	case "CANCELED":
	700	$subj_status = 'canceled';
	701	break;
	702
	703	case "FAILED":
	704	$subj_status = 'failed';
[10]	705	if ( preg_match( "/^US3-A/i", $gfacID ) )
	706	{ // For A/Thrift FAIL, get error message
	707	$gfac_message = getExperimentErrors( $gfacID );
	708	//$gfac_message .= "Test ERROR MESSAGE";
	709	}
[1]	710	break;
	711
	712	case "ERROR":
	713	$subj_status = 'unknown error';
	714	break;
	715
	716	default:
	717	$subj_status = $status; // For now
	718	break;
	719
	720	}
	721
[10]	722	$queuestatus = $subj_status;
	723	$limshost = $dbhost;
	724	if ( $limshost == 'localhost' )
[25]	725	{
[10]	726	$limshost = gethostname();
[25]	727	if ( ! preg_match( "/\./", $limshost ) )
	728	$limshost = $limshost . $org_domain;
	729	}
[10]	730
[1]	731	// Parse the editXMLFilename
	732	list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
	733	explode( ".", $editXMLFilename );
	734
	735	$headers = "From: $org_name Admin<$admin_email>" . "\n";
	736	$headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
	737
	738	// Set the reply address
	739	$headers .= "Reply-To: $org_name<$admin_email>" . "\n";
	740	$headers .= "Return-Path: $org_name<$admin_email>" . "\n";
	741
	742	// Try to avoid spam filters
	743	$now = time();
	744	$headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
	745	$headers .= "X-Mailer: PHP v" . phpversion() . "\n";
	746	$headers .= "MIME-Version: 1.0" . "\n";
	747	$headers .= "Content-Transfer-Encoding: 8bit" . "\n";
	748
	749	$subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
	750	$message = "
	751	Your UltraScan job is complete:
	752
	753	Submission Time : $submittime
[10]	754	LIMS Host : $limshost
[1]	755	Analysis ID : $gfacID
[6]	756	Request ID : $requestID ( $db )
[1]	757	RunID : $runID
	758	EditID : $editID
	759	Data Type : $dataType
	760	Cell/Channel/Wl : $cell / $channel / $wl
	761	Status : $queuestatus
	762	Cluster : $cluster
	763	Job Type : $jobtype
	764	GFAC Status : $status
[4]	765	GFAC Message : $gfac_message
[5]	766	Stdout : $stdout
[1]	767	";
	768
[4]	769	if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
[1]	770
	771	// Handle the error case where an error occurs before fetching the
	772	// user's email address
	773	if ( $email_address == "" ) $email_address = $admin_email;
	774
	775	mail( $email_address, $subject, $message, $headers );
	776	}
	777
	778	function parse_xml( $xml, $type )
	779	{
	780	$parser = new XMLReader();
	781	$parser->xml( $xml );
	782
	783	$results = array();
	784
	785	while ( $parser->read() )
	786	{
	787	if ( $parser->name == $type )
	788	{
	789	while ( $parser->moveToNextAttribute() )
	790	{
	791	$results[ $parser->name ] = $parser->value;
	792	}
	793
	794	break;
	795	}
	796	}
	797
	798	$parser->close();
	799	return $results;
	800	}
	801
	802	// Function to get information about the current job GFAC
	803	function get_gfac_message( $gfacID )
	804	{
	805	global $serviceURL;
	806	global $me;
	807
	808	$hex = "[0-9a-fA-F]";
	809	if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
	810	! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
	811	{
	812	// Then it's not a GFAC job
	813	return false;
	814	}
	815
	816	$url = "$serviceURL/jobstatus/$gfacID";
	817	try
	818	{
	819	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	820	$http = $post->send();
	821	$xml = $post->getResponseBody();
	822	}
	823	catch ( HttpException $e )
	824	{
	825	write_log( "$me: Job status not available - $gfacID" );
	826	return false;
	827	}
	828
	829	// Parse the result
	830	$gfac_message = parse_message( $xml );
	831
	832	return $gfac_message;
	833	}
	834
	835	function parse_message( $xml )
	836	{
[4]	837	global $status;
[1]	838	$status = "";
	839	$gfac_message = "";
	840
	841	$parser = new XMLReader();
	842	$parser->xml( $xml );
	843
	844	$results = array();
	845
	846	while( $parser->read() )
	847	{
	848	$type = $parser->nodeType;
	849
	850	if ( $type == XMLReader::ELEMENT )
	851	$name = $parser->name;
	852
	853	else if ( $type == XMLReader::TEXT )
	854	{
	855	if ( $name == "status" )
	856	$status = $parser->value;
	857	else
	858	$gfac_message = $parser->value;
	859	}
	860	}
	861
	862	$parser->close();
	863	return $gfac_message;
	864	}
	865
	866	function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
	867	{
	868	global $work;
	869	global $work_remote;
	870	global $me;
	871	global $db;
	872	global $status;
[25]	873	$is_us3iab = preg_match( "/us3iab/", $cluster );
[31]	874	$is_jetstr = preg_match( "/jetstream/", $cluster );
[1]	875
	876	// Figure out remote directory
	877	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
[28]	878	//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
[1]	879
	880	// Get stdout, stderr, output/analysis-results.tar
	881	$output = array();
	882
[25]	883	if ( $is_us3iab == 0 )
	884	{
[28]	885	// For "-local", recompute remote work directory
[31]	886	$clushost = "$cluster.uthscsa.edu";
	887	$lworkdir = "~us3/lims/work/local";
	888	if ( $is_jetstr )
	889	{
	890	$clushost = "js-157-184.jetstream-cloud.org";
	891	$lworkdir = "/N/us3_cluster/work/local";
	892	}
	893	$cmd = "ssh us3@$clushost 'ls -d $lworkdir' 2/dev/null";
[28]	894	exec( $cmd, $output, $stat );
	895	$work_remote = $output[ 0 ];
	896	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
	897	write_log( "$me: -LOCAL: remoteDir=$remoteDir" );
	898
[25]	899	// Figure out local working directory
	900	if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
	901	$pwd = chdir( "$work/$gfacID" );
[1]	902
[31]	903	$cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1";
[25]	904
[1]	905	exec( $cmd, $output, $stat );
[25]	906	if ( $stat != 0 )
[1]	907	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	908
[31]	909	$cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1";
[25]	910
[1]	911	exec( $cmd, $output, $stat );
[25]	912	if ( $stat != 0 )
	913	{
[1]	914	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	915	sleep( 10 );
	916	write_log( "$me: RETRY" );
	917	exec( $cmd, $output, $stat );
	918	if ( $stat != 0 )
	919	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	920	}
	921
[31]	922	$cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1";
[25]	923
	924	exec( $cmd, $output, $stat );
	925	if ( $stat != 0 )
	926	{
	927	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	928	sleep( 10 );
	929	write_log( "$me: RETRY" );
	930	exec( $cmd, $output, $stat );
	931	if ( $stat != 0 )
	932	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	933	}
[1]	934	}
[25]	935	else
	936	{
	937	$pwd = chdir( "$remoteDir" );
	938	write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
	939	}
[1]	940
[25]	941
[1]	942	// Write the files to gfacDB
	943
[25]	944	if ( file_exists( "stderr" ) )
	945	{ // Filter stderr to not have libnnls debug lines
[27]	946	// exec( "mv stderr stderr+nnls", $output, $stat );
	947	// exec( "grep -vi nnls stderr+nnls >stderr", $output, $stat );
[25]	948	$stderr = file_get_contents( "stderr" );
	949	}
	950	else
	951	$stderr = "";
[1]	952	if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
[25]	953	$fn1_tarfile = "analysis-results.tar";
	954	$fn2_tarfile = "output/" . $fn1_tarfile;
	955	if ( file_exists( $fn1_tarfile ) )
	956	$tarfile = file_get_contents( $fn1_tarfile );
	957	else if ( file_exists( $fn2_tarfile ) )
	958	$tarfile = file_get_contents( $fn2_tarfile );
[1]	959
[25]	960	$lense = strlen( $stderr );
[28]	961	if ( $lense > 1000000 )
[25]	962	{ // Replace exceptionally large stderr with smaller version
	963	exec( "mv stderr stderr-orig", $output, $stat );
	964	exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
	965	exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
	966	exec( "cat stderr-h stderr-t >stderr", $output, $stat );
	967	$stderr = file_get_contents( "stderr" );
	968	}
	969	$lent = strlen( $tarfile );
	970	write_log( "$me: tarfile size: $lent");
	971	$lene = strlen( $stderr );
	972	write_log( "$me: stderr size: $lene (was $lense)");
	973	$leno = strlen( $stdout );
	974	write_log( "$me: stdout size: $leno");
	975	$estarf=mysql_real_escape_string($tarfile,$gfac_link);
	976	$lenf = strlen($estarf);
	977	write_log( "$me: es-tarfile size: $lenf");
	978	$esstdo=mysql_real_escape_string($stdout,$gfac_link);
	979	$leno = strlen($esstdo);
	980	write_log( "$me: es-stdout size: $leno");
	981	$esstde=mysql_real_escape_string($stderr,$gfac_link);
	982	$lene = strlen($esstde);
	983	write_log( "$me: es-stderr size: $lene");
[1]	984	$query = "UPDATE analysis SET " .
	985	"stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
	986	"stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
	987	"tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
	988
	989	$result = mysql_query( $query, $gfac_link );
	990
	991	if ( ! $result )
	992	{
	993	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	994	echo "Bad query\n";
	995	return( -1 );
	996	}
	997	}
	998	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: