Context Navigation

source: trunk/cleanup.php@ 26

Last change on this file since 26 was 26, checked in by gegorbet, 8 years ago
various minor fixes/mods to gridctl
File size: 30.5 KB

Rev	Line
[1]	1	<?php
	2	/*
	3	* cleanup.php
	4	*
	5	* functions relating to copying results and cleaning up the gfac DB
[10]	6	* where the job used an Airavata interface.
[1]	7	*
	8	*/
	9
	10	$email_address = '';
	11	$queuestatus = '';
	12	$jobtype = '';
	13	$db = '';
	14	$editXMLFilename = '';
[4]	15	$status = '';
[1]	16
[10]	17	function aira_cleanup( $us3_db, $reqID, $gfac_link )
[1]	18	{
[25]	19	global $org_domain;
[1]	20	global $dbhost;
	21	global $user;
	22	global $passwd;
	23	global $db;
	24	global $guser;
	25	global $gpasswd;
	26	global $gDB;
	27	global $me;
	28	global $work;
	29	global $email_address;
	30	global $queuestatus;
	31	global $jobtype;
	32	global $editXMLFilename;
	33	global $submittime;
[4]	34	global $status;
[10]	35	global $stderr;
[5]	36	global $stdout;
[10]	37	global $tarfile;
[6]	38	global $requestID;
[10]	39	global $submit_dir;
	40	$me = 'cleanup_aira.php';
[1]	41
[6]	42	$requestID = $reqID;
[1]	43	$db = $us3_db;
	44	write_log( "$me: debug db=$db; requestID=$requestID" );
	45
	46	$us3_link = mysql_connect( $dbhost, $user, $passwd );
	47
	48	if ( ! $us3_link )
	49	{
	50	write_log( "$me: could not connect: $dbhost, $user, $passwd" );
	51	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
	52	return( -1 );
	53	}
	54
	55	$result = mysql_select_db( $db, $us3_link );
	56
	57	if ( ! $result )
	58	{
	59	write_log( "$me: could not select DB $db" );
	60	mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
	61	return( -1 );
	62	}
	63
	64	// First get basic info for email messages
	65	$query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
	66	"WHERE HPCAnalysisRequestID=$requestID";
	67	$result = mysql_query( $query, $us3_link );
	68
	69	if ( ! $result )
	70	{
	71	write_log( "$me: Bad query: $query" );
	72	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
	73	return( -1 );
	74	}
	75
	76	list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
	77
	78	$query = "SELECT personID FROM people " .
	79	"WHERE personGUID='$investigatorGUID'";
	80	$result = mysql_query( $query, $us3_link );
	81
	82	list( $personID ) = mysql_fetch_array( $result );
	83
	84	$query = "SELECT clusterName, submitTime, queueStatus, method " .
	85	"FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
	86	"WHERE h.HPCAnalysisRequestID=$requestID " .
	87	"AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
	88
	89	$result = mysql_query( $query, $us3_link );
	90
	91	if ( ! $result )
	92	{
	93	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	94	return( -1 );
	95	}
	96
	97	if ( mysql_num_rows( $result ) == 0 )
	98	{
	99	write_log( "$me: US3 Table error - No records for requestID: $requestID" );
	100	return( -1 );
	101	}
	102
	103	list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
	104
	105	// Get the GFAC ID
	106	$query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
	107	"WHERE HPCAnalysisRequestID=$requestID";
	108
	109	$result = mysql_query( $query, $us3_link );
	110
	111	if ( ! $result )
	112	{
	113	write_log( "$me: Bad query: $query" );
	114	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
	115	return( -1 );
	116	}
	117
	118	list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
	119
[10]	120	// Get data from global GFAC DB then insert it into US3 DB
[1]	121
	122	$result = mysql_select_db( $gDB, $gfac_link );
	123
	124	if ( ! $result )
	125	{
	126	write_log( "$me: Could not connect to DB $gDB" );
	127	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
	128	return( -1 );
	129	}
	130
	131	$query = "SELECT status, cluster, id FROM analysis " .
	132	"WHERE gfacID='$gfacID'";
	133
	134	$result = mysql_query( $query, $gfac_link );
	135	if ( ! $result )
	136	{
	137	write_log( "$me: Could not select GFAC status for $gfacID" );
	138	mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
	139	return( -1 );
	140	}
	141
	142	list( $status, $cluster, $id ) = mysql_fetch_array( $result );
	143
[25]	144	$is_us3iab = preg_match( "/us3iab/", $cluster );
	145	$is_local = preg_match( "/-local/", $cluster );
	146
	147	if ( $is_us3iab \|\| $is_local )
[1]	148	{
	149	$clushost = $cluster;
	150	$clushost = preg_replace( "/\-local/", "", $clushost );
	151	get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
	152	}
	153
[10]	154
	155	$query = "SELECT id FROM analysis " .
[1]	156	"WHERE gfacID='$gfacID'";
	157
	158	$result = mysql_query( $query, $gfac_link );
	159
	160	if ( ! $result )
	161	{
	162	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	163	mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
	164	return( -1 );
	165	}
	166
[10]	167	list( $analysisID ) = mysql_fetch_array( $result );
[1]	168
[10]	169	// Get the request guid (LIMS submit dir name)
	170	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	171	"WHERE HPCAnalysisRequestID = $requestID ";
	172	$result = mysql_query( $query, $us3_link );
	173
	174	if ( ! $result )
	175	{
	176	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	177	}
	178
	179	list( $requestGUID ) = mysql_fetch_array( $result );
	180	$output_dir = "$submit_dir/$requestGUID";
	181
	182	// Get stderr,stdout,tarfile from work directory
	183	if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
	184	chdir( "$output_dir" );
	185	//write_log( "$me: gfacID=$gfacID" );
	186	//write_log( "$me: submit_dir=$submit_dir" );
	187	//write_log( "$me: requestGUID=$requestGUID" );
	188	write_log( "$me: output_dir=$output_dir" );
	189
	190	$stderr = "";
	191	$stdout = "";
	192	$tarfile = "";
	193	$fn_stderr = "Ultrascan.stderr";
	194	$fn_stdout = "Ultrascan.stdout";
	195	$fn_tarfile = "analysis-results.tar";
	196	$num_try = 0;
[26]	197	write_log( "$me: fn_tarfile=$fn_tarfile" );
[10]	198	while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
	199	{
	200	sleep( 10 );
	201	$num_try++;
[25]	202	write_log( "$me: tar-exists: num_try=$num_try" );
[10]	203	}
	204
	205	$ofiles = scandir( $output_dir );
	206	foreach ( $ofiles as $ofile )
	207	{
[25]	208	if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
[10]	209	$fn_stderr = $ofile;
[25]	210	if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
[10]	211	$fn_stdout = $ofile;
	212	//write_log( "$me: ofile=$ofile" );
	213	}
	214	write_log( "$me: fn_stderr=$fn_stderr" );
	215	write_log( "$me: fn_stdout=$fn_stdout" );
	216	if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
	217	else write_log( "$me: NOT FOUND: $fn_tarfile" );
	218
[26]	219	$stderr = '';
	220	$stdout = '';
	221	$tarfile = '';
[10]	222	if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
	223	if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
	224	if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
[26]	225	// If stdout,stderr have no content, retry after delay
	226	if ( strlen( $stdout ) == 0 \|\| strlen( $stderr ) == 0 )
	227	{
	228	sleep( 20 );
	229	if ( file_exists( $fn_stderr ) )
	230	$stderr = file_get_contents( $fn_stderr );
	231	if ( file_exists( $fn_stdout ) )
	232	$stdout = file_get_contents( $fn_stdout );
	233	}
[10]	234
[26]	235	write_log( "$me: length contents stderr,stdout,tarfile -- "
	236	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
	237
[10]	238	if ( $cluster == 'alamo' \|\| $cluster == 'alamo-local' )
	239	{ // Filter "ipath_userinit" lines out of alamo stdout lines
	240	$prefln = strlen( $stdout );
	241	$output = array();
	242	exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
	243	$stdout = implode( "\n", $output );
	244	$posfln = strlen( $stdout );
	245	write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
	246	}
	247
[1]	248	// Save queue messages for post-mortem analysis
	249	$query = "SELECT message, time FROM queue_messages " .
	250	"WHERE analysisID = $analysisID " .
	251	"ORDER BY time ";
	252	$result = mysql_query( $query, $gfac_link );
	253
	254	if ( ! $result )
	255	{
	256	// Just log it and continue
	257	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	258	}
	259
	260	$now = date( 'Y-m-d H:i:s' );
	261	$message_log = "US3 DB: $db\n" .
	262	"RequestID: $requestID\n" .
	263	"GFAC ID: $gfacID\n" .
	264	"Processed: $now\n\n" .
	265	"Queue Messages\n\n" ;
	266	if ( mysql_num_rows( $result ) > 0 )
	267	{
	268	while ( list( $message, $time ) = mysql_fetch_array( $result ) )
	269	$message_log .= "$time $message\n";
	270	}
	271
	272	$query = "DELETE FROM queue_messages " .
	273	"WHERE analysisID = $analysisID ";
	274
	275	$result = mysql_query( $query, $gfac_link );
	276
	277	if ( ! $result )
	278	{
	279	// Just log it and continue
	280	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	281	}
	282
[10]	283	$query = "SELECT queue_msg FROM analysis " .
	284	"WHERE gfacID='$gfacID' ";
	285
[1]	286	$result = mysql_query( $query, $gfac_link );
[10]	287	list( $queue_msg ) = mysql_fetch_array( $result );
[1]	288
	289	// But let's allow for investigation of other large stdout and/or stderr
	290	if ( strlen( $stdout ) > 20480000 \|\|
	291	strlen( $stderr ) > 20480000 )
	292	write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
	293
	294	$message_log .= "\n\n\nStdout Contents\n\n" .
	295	$stdout .
	296	"\n\n\nStderr Contents\n\n" .
	297	$stderr .
	298	"\n\n\nGFAC Status: $status\n" .
	299	"GFAC message field: $queue_msg\n";
	300
	301	// Delete data from GFAC DB
	302	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
	303
	304	$result = mysql_query( $query, $gfac_link );
	305
	306	if ( ! $result )
	307	{
	308	// Just log it and continue
	309	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	310	}
	311
	312
	313	// Try to create it if necessary, and write the file
	314	// Let's use FILE_APPEND, in case this is the second time around and the
	315	// GFAC job status was INSERTed, rather than UPDATEd
	316	if ( ! is_dir( $output_dir ) )
	317	mkdir( $output_dir, 0775, true );
	318	$message_filename = "$output_dir/$db-$requestID-messages.txt";
	319	file_put_contents( $message_filename, $message_log, FILE_APPEND );
	320	// mysql_close( $gfac_link );
	321
	322	/////////
	323	// Insert data into HPCAnalysis
	324
	325	$query = "UPDATE HPCAnalysisResult SET " .
	326	"stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
[10]	327	"stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
	328	"queueStatus='completed' " .
[1]	329	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	330
	331	$result = mysql_query( $query, $us3_link );
	332
	333	if ( ! $result )
	334	{
	335	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	336	mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
	337	return( -1 );
	338	}
	339
[10]	340	// Delete data from GFAC DB
	341	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
[1]	342
[10]	343	$result = mysql_query( $query, $gfac_link );
	344
	345	if ( ! $result )
	346	{
	347	// Just log it and continue
	348	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	349	}
	350
	351	// Expand the tar file
	352
[1]	353	if ( strlen( $tarfile ) == 0 )
	354	{
	355	write_log( "$me: No tarfile" );
	356	mail_to_user( "fail", "No results" );
	357	return( -1 );
	358	}
	359
	360	$tar_out = array();
[10]	361	exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
[1]	362
	363	// Insert the model files and noise files
	364	$files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
	365	$noiseIDs = array();
	366	$modelGUIDs = array();
[17]	367	$mrecsIDs = array();
	368	$fns_used = array();
[1]	369
	370	foreach ( $files as $file )
	371	{
	372	$split = explode( ";", $file );
	373
	374	if ( count( $split ) > 1 )
	375	{
	376	list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
	377
	378	list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
	379	list( $other, $variance ) = explode( "=", $variance );
	380	list( $other, $meniscus ) = explode( "=", $meniscus );
	381	}
	382	else
	383	$fn = $file;
	384
[10]	385	if ( preg_match( "/mdl.tmp$/", $fn ) )
	386	continue;
	387
[17]	388	if ( in_array( $fn, $fns_used ) )
	389	continue;
	390
	391	$fns_used[] = $fn;
	392
[1]	393	if ( filesize( $fn ) < 100 )
	394	{
[10]	395	write_log( "$me:fn is invalid $fn size filesize($fn)" );
[1]	396	mail_to_user( "fail", "Internal error\n$fn is invalid" );
	397	return( -1 );
	398	}
	399
	400	if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
	401	{
	402	$xml = file_get_contents( $fn );
	403	$statistics = parse_xml( $xml, 'statistics' );
[10]	404	// $ntries = 0;
	405	//
	406	// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
	407	// { // job_statistics file not totally copied, so retry
	408	// sleep( 10 );
	409	// $xml = file_get_contents( $fn );
	410	// $statistics = parse_xml( $xml, 'statistics' );
	411	// $ntries++;
	412	//write_log( "$me:jobstats retry $ntries" );
	413	// }
	414	//write_log( "$me:cputime=$statistics['cputime']" );
	415
[1]	416	$otherdata = parse_xml( $xml, 'id' );
	417
	418	$query = "UPDATE HPCAnalysisResult SET " .
	419	"wallTime = {$statistics['walltime']}, " .
	420	"CPUTime = {$statistics['cputime']}, " .
	421	"CPUCount = {$statistics['cpucount']}, " .
	422	"max_rss = {$statistics['maxmemory']}, " .
	423	"startTime = '{$otherdata['starttime']}', " .
	424	"endTime = '{$otherdata['endtime']}', " .
	425	"mgroupcount = {$otherdata['groupcount']} " .
	426	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	427	$result = mysql_query( $query, $us3_link );
	428
	429	if ( ! $result )
	430	{
	431	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	432	}
	433
	434	file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
	435
[24]	436	$file_type = "job_stats";
	437	$id = 1;
[1]	438	}
	439
	440	else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
	441	{
	442	$xml = file_get_contents( $fn );
	443	$noise_data = parse_xml( $xml, "noise" );
	444	$type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
	445	$desc = $noise_data[ 'description' ];
	446	$modelGUID = $noise_data[ 'modelGUID' ];
	447	$noiseGUID = $noise_data[ 'noiseGUID' ];
	448
	449	$query = "INSERT INTO noise SET " .
	450	"noiseGUID='$noiseGUID'," .
	451	"modelGUID='$modelGUID'," .
	452	"editedDataID=1, " .
	453	"modelID=1, " .
	454	"noiseType='$type'," .
	455	"description='$desc'," .
	456	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	457
	458	// Add later after all files are processed: editDataID, modelID
	459
	460	$result = mysql_query( $query, $us3_link );
	461
	462	if ( ! $result )
	463	{
	464	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	465	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	466	return( -1 );
	467	}
	468
	469	$id = mysql_insert_id( $us3_link );
	470	$file_type = "noise";
	471	$noiseIDs[] = $id;
	472
	473	// Keep track of modelGUIDs for later, when we replace them
	474	$modelGUIDs[ $id ] = $modelGUID;
	475
	476	}
[10]	477
	478	else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
[1]	479	{
	480	$xml = file_get_contents( $fn );
[10]	481	$mrecs_data = parse_xml( $xml, "modelrecords" );
	482	$desc = $mrecs_data[ 'description' ];
	483	$editGUID = $mrecs_data[ 'editGUID' ];
	484	write_log( "$me: mrecs file editGUID=$editGUID" );
	485	if ( strlen( $editGUID ) < 36 )
	486	$editGUID = "12345678-0123-5678-0123-567890123456";
	487	$mrecGUID = $mrecs_data[ 'mrecGUID' ];
	488	$modelGUID = $mrecs_data[ 'modelGUID' ];
	489
	490	$query = "INSERT INTO pcsa_modelrecs SET " .
	491	"editedDataID=" .
	492	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	493	"modelID=0, " .
	494	"mrecsGUID='$mrecGUID'," .
	495	"description='$desc'," .
	496	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	497
	498	// Add later after all files are processed: editDataID, modelID
	499
	500	$result = mysql_query( $query, $us3_link );
	501
	502	if ( ! $result )
	503	{
	504	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	505	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	506	return( -1 );
	507	}
	508
	509	$id = mysql_insert_id( $us3_link );
	510	$file_type = "mrecs";
	511	$mrecsIDs[] = $id;
	512
	513	// Keep track of modelGUIDs for later, when we replace them
	514	$rmodlGUIDs[ $id ] = $modelGUID;
	515	//write_log( "$me: mrecs file inserted into DB : id=$id" );
	516	}
	517
[23]	518	else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
[10]	519	{
	520	$xml = file_get_contents( $fn );
[1]	521	$model_data = parse_xml( $xml, "model" );
	522	$description = $model_data[ 'description' ];
	523	$modelGUID = $model_data[ 'modelGUID' ];
	524	$editGUID = $model_data[ 'editGUID' ];
	525
[10]	526	if ( $mc_iteration > 1 )
	527	{
	528	$miter = sprintf( "_mcN%03d", $mc_iteration );
	529	$description = preg_replace( "/_mc[0-9]+/", $miter, $description );
	530	write_log( "$me: MODELUpd: O:description=$description" );
	531	}
	532
[1]	533	$query = "INSERT INTO model SET " .
	534	"modelGUID='$modelGUID'," .
	535	"editedDataID=" .
	536	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	537	"description='$description'," .
	538	"MCIteration='$mc_iteration'," .
	539	"meniscus='$meniscus'," .
	540	"variance='$variance'," .
	541	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	542
	543	$result = mysql_query( $query, $us3_link );
	544
	545	if ( ! $result )
	546	{
	547	write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
	548	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	549	return( -1 );
	550	}
	551
	552	$modelID = mysql_insert_id( $us3_link );
	553	$id = $modelID;
	554	$file_type = "model";
	555
	556	$query = "INSERT INTO modelPerson SET " .
	557	"modelID=$modelID, personID=$personID";
	558	$result = mysql_query( $query, $us3_link );
	559	}
	560
[24]	561	else // Undetermined type: skip result data update
	562	continue;
	563
[1]	564	$query = "INSERT INTO HPCAnalysisResultData SET " .
	565	"HPCAnalysisResultID='$HPCAnalysisResultID', " .
	566	"HPCAnalysisResultType='$file_type', " .
	567	"resultID=$id";
	568
	569	$result = mysql_query( $query, $us3_link );
	570
	571	if ( ! $result )
	572	{
	573	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	574	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	575	return( -1 );
	576	}
	577	}
	578
	579	// Now fix up noise entries
	580	// For noise files, there is, at most two: ti_noise and ri_noise
	581	// In this case there will only be one modelID
	582
	583	foreach ( $noiseIDs as $noiseID )
	584	{
	585	$modelGUID = $modelGUIDs[ $noiseID ];
	586	$query = "UPDATE noise SET " .
	587	"editedDataID=" .
[6]	588	"(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
[1]	589	"modelID=" .
	590	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	591	"WHERE noiseID=$noiseID";
	592
	593	$result = mysql_query( $query, $us3_link );
	594
	595	if ( ! $result )
	596	{
	597	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	598	mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
	599	return( -1 );
	600	}
	601	}
	602
[13]	603	// Now possibly fix up mrecs entries
	604
	605	foreach ( $mrecsIDs as $mrecsID )
	606	{
	607	$modelGUID = $rmodlGUIDs[ $mrecsID ];
	608	$query = "UPDATE pcsa_modelrecs SET " .
	609	"modelID=" .
	610	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	611	"WHERE mrecsID=$mrecsID";
	612
	613	$result = mysql_query( $query, $us3_link );
	614
	615	if ( ! $result )
	616	{
	617	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	618	mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
	619	return( -1 );
	620	}
	621	write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
	622	}
	623	//write_log( "$me: mrecs entries updated" );
	624
[1]	625	// Copy results to LIMS submit directory (files there are deleted after 7 days)
	626	global $submit_dir; // LIMS submit files dir
	627
	628	// Get the request guid (LIMS submit dir name)
	629	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	630	"WHERE HPCAnalysisRequestID = $requestID ";
	631	$result = mysql_query( $query, $us3_link );
	632
	633	if ( ! $result )
	634	{
	635	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	636	}
	637
[10]	638	// list( $requestGUID ) = mysql_fetch_array( $result );
	639	//
	640	// chdir( "$submit_dir/$requestGUID" );
	641	// $f = fopen( "analysis-results.tar", "w" );
	642	// fwrite( $f, $tarfile );
	643	// fclose( $f );
[1]	644
	645	// Clean up
[10]	646	// chdir ( $work );
[1]	647	// exec( "rm -rf $gfacID" );
	648
	649	mysql_close( $us3_link );
	650
	651	/////////
	652	// Send email
	653
	654	mail_to_user( "success", "" );
	655	}
	656
	657	function mail_to_user( $type, $msg )
	658	{
	659	// Note to me. Just changed subject line to include a modified $status instead
	660	// of the $type variable passed. More informative than just "fail" or "success."
	661	// See how it works for awhile and then consider removing $type parameter from
	662	// function.
	663	global $email_address;
	664	global $submittime;
	665	global $queuestatus;
	666	global $status;
	667	global $cluster;
	668	global $jobtype;
	669	global $org_name;
	670	global $admin_email;
[6]	671	global $db;
[1]	672	global $dbhost;
	673	global $requestID;
	674	global $gfacID;
	675	global $editXMLFilename;
[5]	676	global $stdout;
[25]	677	global $org_domain;
[1]	678
	679	global $me;
	680	write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
	681
	682	// Get GFAC status and message
[4]	683	// function get_gfac_message() also sets global $status
	684	$gfac_message = get_gfac_message( $gfacID );
[10]	685	if ( $gfac_message === false ) $gfac_message = "Job Finished";
[1]	686
	687	// Create a status to put in the subject line
	688	switch ( $status )
	689	{
	690	case "COMPLETE":
	691	$subj_status = 'completed';
	692	break;
	693
	694	case "CANCELLED":
	695	case "CANCELED":
	696	$subj_status = 'canceled';
	697	break;
	698
	699	case "FAILED":
	700	$subj_status = 'failed';
[10]	701	if ( preg_match( "/^US3-A/i", $gfacID ) )
	702	{ // For A/Thrift FAIL, get error message
	703	$gfac_message = getExperimentErrors( $gfacID );
	704	//$gfac_message .= "Test ERROR MESSAGE";
	705	}
[1]	706	break;
	707
	708	case "ERROR":
	709	$subj_status = 'unknown error';
	710	break;
	711
	712	default:
	713	$subj_status = $status; // For now
	714	break;
	715
	716	}
	717
[10]	718	$queuestatus = $subj_status;
	719	$limshost = $dbhost;
	720	if ( $limshost == 'localhost' )
[25]	721	{
[10]	722	$limshost = gethostname();
[25]	723	if ( ! preg_match( "/\./", $limshost ) )
	724	$limshost = $limshost . $org_domain;
	725	}
[10]	726
[1]	727	// Parse the editXMLFilename
	728	list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
	729	explode( ".", $editXMLFilename );
	730
	731	$headers = "From: $org_name Admin<$admin_email>" . "\n";
	732	$headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
	733
	734	// Set the reply address
	735	$headers .= "Reply-To: $org_name<$admin_email>" . "\n";
	736	$headers .= "Return-Path: $org_name<$admin_email>" . "\n";
	737
	738	// Try to avoid spam filters
	739	$now = time();
	740	$headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
	741	$headers .= "X-Mailer: PHP v" . phpversion() . "\n";
	742	$headers .= "MIME-Version: 1.0" . "\n";
	743	$headers .= "Content-Transfer-Encoding: 8bit" . "\n";
	744
	745	$subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
	746	$message = "
	747	Your UltraScan job is complete:
	748
	749	Submission Time : $submittime
[10]	750	LIMS Host : $limshost
[1]	751	Analysis ID : $gfacID
[6]	752	Request ID : $requestID ( $db )
[1]	753	RunID : $runID
	754	EditID : $editID
	755	Data Type : $dataType
	756	Cell/Channel/Wl : $cell / $channel / $wl
	757	Status : $queuestatus
	758	Cluster : $cluster
	759	Job Type : $jobtype
	760	GFAC Status : $status
[4]	761	GFAC Message : $gfac_message
[5]	762	Stdout : $stdout
[1]	763	";
	764
[4]	765	if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
[1]	766
	767	// Handle the error case where an error occurs before fetching the
	768	// user's email address
	769	if ( $email_address == "" ) $email_address = $admin_email;
	770
	771	mail( $email_address, $subject, $message, $headers );
	772	}
	773
	774	function parse_xml( $xml, $type )
	775	{
	776	$parser = new XMLReader();
	777	$parser->xml( $xml );
	778
	779	$results = array();
	780
	781	while ( $parser->read() )
	782	{
	783	if ( $parser->name == $type )
	784	{
	785	while ( $parser->moveToNextAttribute() )
	786	{
	787	$results[ $parser->name ] = $parser->value;
	788	}
	789
	790	break;
	791	}
	792	}
	793
	794	$parser->close();
	795	return $results;
	796	}
	797
	798	// Function to get information about the current job GFAC
	799	function get_gfac_message( $gfacID )
	800	{
	801	global $serviceURL;
	802	global $me;
	803
	804	$hex = "[0-9a-fA-F]";
	805	if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
	806	! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
	807	{
	808	// Then it's not a GFAC job
	809	return false;
	810	}
	811
	812	$url = "$serviceURL/jobstatus/$gfacID";
	813	try
	814	{
	815	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	816	$http = $post->send();
	817	$xml = $post->getResponseBody();
	818	}
	819	catch ( HttpException $e )
	820	{
	821	write_log( "$me: Job status not available - $gfacID" );
	822	return false;
	823	}
	824
	825	// Parse the result
	826	$gfac_message = parse_message( $xml );
	827
	828	return $gfac_message;
	829	}
	830
	831	function parse_message( $xml )
	832	{
[4]	833	global $status;
[1]	834	$status = "";
	835	$gfac_message = "";
	836
	837	$parser = new XMLReader();
	838	$parser->xml( $xml );
	839
	840	$results = array();
	841
	842	while( $parser->read() )
	843	{
	844	$type = $parser->nodeType;
	845
	846	if ( $type == XMLReader::ELEMENT )
	847	$name = $parser->name;
	848
	849	else if ( $type == XMLReader::TEXT )
	850	{
	851	if ( $name == "status" )
	852	$status = $parser->value;
	853	else
	854	$gfac_message = $parser->value;
	855	}
	856	}
	857
	858	$parser->close();
	859	return $gfac_message;
	860	}
	861
	862	function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
	863	{
	864	global $work;
	865	global $work_remote;
	866	global $me;
	867	global $db;
	868	global $status;
[25]	869	$is_us3iab = preg_match( "/us3iab/", $cluster );
[1]	870
	871	// Figure out remote directory
	872	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
[25]	873	write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
[1]	874
	875	// Get stdout, stderr, output/analysis-results.tar
	876	$output = array();
	877
[25]	878	if ( $is_us3iab == 0 )
	879	{
	880	// Figure out local working directory
	881	if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
	882	$pwd = chdir( "$work/$gfacID" );
[1]	883
[25]	884	$cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
	885
[1]	886	exec( $cmd, $output, $stat );
[25]	887	if ( $stat != 0 )
[1]	888	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	889
	890	$cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
	891
[1]	892	exec( $cmd, $output, $stat );
[25]	893	if ( $stat != 0 )
	894	{
[1]	895	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	896	sleep( 10 );
	897	write_log( "$me: RETRY" );
	898	exec( $cmd, $output, $stat );
	899	if ( $stat != 0 )
	900	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	901	}
	902
	903	$cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
	904
	905	exec( $cmd, $output, $stat );
	906	if ( $stat != 0 )
	907	{
	908	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	909	sleep( 10 );
	910	write_log( "$me: RETRY" );
	911	exec( $cmd, $output, $stat );
	912	if ( $stat != 0 )
	913	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	914	}
[1]	915	}
[25]	916	else
	917	{
	918	$pwd = chdir( "$remoteDir" );
	919	write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
	920	}
[1]	921
[25]	922
[1]	923	// Write the files to gfacDB
	924
[25]	925	if ( file_exists( "stderr" ) )
	926	{ // Filter stderr to not have libnnls debug lines
	927	exec( "mv stderr stderr+nnls", $output, $stat );
	928	exec( "grep -vi nnls stderr+nnls >stderr", $output, $stat );
	929	$stderr = file_get_contents( "stderr" );
	930	}
	931	else
	932	$stderr = "";
[1]	933	if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
[25]	934	$fn1_tarfile = "analysis-results.tar";
	935	$fn2_tarfile = "output/" . $fn1_tarfile;
	936	if ( file_exists( $fn1_tarfile ) )
	937	$tarfile = file_get_contents( $fn1_tarfile );
	938	else if ( file_exists( $fn2_tarfile ) )
	939	$tarfile = file_get_contents( $fn2_tarfile );
[1]	940
[25]	941	$lense = strlen( $stderr );
	942	if ( $lense > 1000000 )
	943	{ // Replace exceptionally large stderr with smaller version
	944	exec( "mv stderr stderr-orig", $output, $stat );
	945	exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
	946	exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
	947	exec( "cat stderr-h stderr-t >stderr", $output, $stat );
	948	$stderr = file_get_contents( "stderr" );
	949	}
	950	$lent = strlen( $tarfile );
	951	write_log( "$me: tarfile size: $lent");
	952	$lene = strlen( $stderr );
	953	write_log( "$me: stderr size: $lene (was $lense)");
	954	$leno = strlen( $stdout );
	955	write_log( "$me: stdout size: $leno");
	956	$estarf=mysql_real_escape_string($tarfile,$gfac_link);
	957	$lenf = strlen($estarf);
	958	write_log( "$me: es-tarfile size: $lenf");
	959	$esstdo=mysql_real_escape_string($stdout,$gfac_link);
	960	$leno = strlen($esstdo);
	961	write_log( "$me: es-stdout size: $leno");
	962	$esstde=mysql_real_escape_string($stderr,$gfac_link);
	963	$lene = strlen($esstde);
	964	write_log( "$me: es-stderr size: $lene");
[1]	965	$query = "UPDATE analysis SET " .
	966	"stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
	967	"stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
	968	"tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
	969
	970	$result = mysql_query( $query, $gfac_link );
	971
	972	if ( ! $result )
	973	{
	974	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	975	echo "Bad query\n";
	976	return( -1 );
	977	}
	978	}
	979	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: