Context Navigation

source: trunk/cleanup.php@ 28

Last change on this file since 28 was 28, checked in by gegorbet, 8 years ago
fixes for alamo-local and airavata EXECUTING status
File size: 30.9 KB

Rev	Line
[1]	1	<?php
	2	/*
	3	* cleanup.php
	4	*
	5	* functions relating to copying results and cleaning up the gfac DB
[10]	6	* where the job used an Airavata interface.
[1]	7	*
	8	*/
	9
	10	$email_address = '';
	11	$queuestatus = '';
	12	$jobtype = '';
	13	$db = '';
	14	$editXMLFilename = '';
[4]	15	$status = '';
[1]	16
[10]	17	function aira_cleanup( $us3_db, $reqID, $gfac_link )
[1]	18	{
[25]	19	global $org_domain;
[1]	20	global $dbhost;
	21	global $user;
	22	global $passwd;
	23	global $db;
	24	global $guser;
	25	global $gpasswd;
	26	global $gDB;
	27	global $me;
	28	global $work;
	29	global $email_address;
	30	global $queuestatus;
	31	global $jobtype;
	32	global $editXMLFilename;
	33	global $submittime;
[4]	34	global $status;
[10]	35	global $stderr;
[5]	36	global $stdout;
[10]	37	global $tarfile;
[6]	38	global $requestID;
[10]	39	global $submit_dir;
	40	$me = 'cleanup_aira.php';
[1]	41
[6]	42	$requestID = $reqID;
[1]	43	$db = $us3_db;
	44	write_log( "$me: debug db=$db; requestID=$requestID" );
	45
	46	$us3_link = mysql_connect( $dbhost, $user, $passwd );
	47
	48	if ( ! $us3_link )
	49	{
	50	write_log( "$me: could not connect: $dbhost, $user, $passwd" );
	51	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
	52	return( -1 );
	53	}
	54
	55	$result = mysql_select_db( $db, $us3_link );
	56
	57	if ( ! $result )
	58	{
	59	write_log( "$me: could not select DB $db" );
	60	mail_to_user( "fail", "Internal Error $requestID\n$could not select DB $db" );
	61	return( -1 );
	62	}
	63
	64	// First get basic info for email messages
	65	$query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
	66	"WHERE HPCAnalysisRequestID=$requestID";
	67	$result = mysql_query( $query, $us3_link );
	68
	69	if ( ! $result )
	70	{
	71	write_log( "$me: Bad query: $query" );
	72	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
	73	return( -1 );
	74	}
	75
	76	list( $email_address, $investigatorGUID, $editXMLFilename ) = mysql_fetch_array( $result );
	77
	78	$query = "SELECT personID FROM people " .
	79	"WHERE personGUID='$investigatorGUID'";
	80	$result = mysql_query( $query, $us3_link );
	81
	82	list( $personID ) = mysql_fetch_array( $result );
	83
	84	$query = "SELECT clusterName, submitTime, queueStatus, method " .
	85	"FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
	86	"WHERE h.HPCAnalysisRequestID=$requestID " .
	87	"AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
	88
	89	$result = mysql_query( $query, $us3_link );
	90
	91	if ( ! $result )
	92	{
	93	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	94	return( -1 );
	95	}
	96
	97	if ( mysql_num_rows( $result ) == 0 )
	98	{
	99	write_log( "$me: US3 Table error - No records for requestID: $requestID" );
	100	return( -1 );
	101	}
	102
	103	list( $cluster, $submittime, $queuestatus, $jobtype ) = mysql_fetch_array( $result );
	104
	105	// Get the GFAC ID
	106	$query = "SELECT HPCAnalysisResultID, gfacID FROM HPCAnalysisResult " .
	107	"WHERE HPCAnalysisRequestID=$requestID";
	108
	109	$result = mysql_query( $query, $us3_link );
	110
	111	if ( ! $result )
	112	{
	113	write_log( "$me: Bad query: $query" );
	114	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysql_error( $us3_link ) );
	115	return( -1 );
	116	}
	117
	118	list( $HPCAnalysisResultID, $gfacID ) = mysql_fetch_array( $result );
	119
[10]	120	// Get data from global GFAC DB then insert it into US3 DB
[1]	121
	122	$result = mysql_select_db( $gDB, $gfac_link );
	123
	124	if ( ! $result )
	125	{
	126	write_log( "$me: Could not connect to DB $gDB" );
	127	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
	128	return( -1 );
	129	}
	130
	131	$query = "SELECT status, cluster, id FROM analysis " .
	132	"WHERE gfacID='$gfacID'";
	133
	134	$result = mysql_query( $query, $gfac_link );
	135	if ( ! $result )
	136	{
	137	write_log( "$me: Could not select GFAC status for $gfacID" );
	138	mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
	139	return( -1 );
	140	}
	141
	142	list( $status, $cluster, $id ) = mysql_fetch_array( $result );
	143
[25]	144	$is_us3iab = preg_match( "/us3iab/", $cluster );
	145	$is_local = preg_match( "/-local/", $cluster );
	146
	147	if ( $is_us3iab \|\| $is_local )
[1]	148	{
	149	$clushost = $cluster;
	150	$clushost = preg_replace( "/\-local/", "", $clushost );
	151	get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
	152	}
	153
[10]	154
	155	$query = "SELECT id FROM analysis " .
[1]	156	"WHERE gfacID='$gfacID'";
	157
	158	$result = mysql_query( $query, $gfac_link );
	159
	160	if ( ! $result )
	161	{
	162	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	163	mail_to_user( "fail", "Internal error " . mysql_error( $gfac_link ) );
	164	return( -1 );
	165	}
	166
[10]	167	list( $analysisID ) = mysql_fetch_array( $result );
[1]	168
[10]	169	// Get the request guid (LIMS submit dir name)
	170	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	171	"WHERE HPCAnalysisRequestID = $requestID ";
	172	$result = mysql_query( $query, $us3_link );
	173
	174	if ( ! $result )
	175	{
	176	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	177	}
	178
	179	list( $requestGUID ) = mysql_fetch_array( $result );
	180	$output_dir = "$submit_dir/$requestGUID";
	181
	182	// Get stderr,stdout,tarfile from work directory
	183	if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
	184	chdir( "$output_dir" );
	185	//write_log( "$me: gfacID=$gfacID" );
	186	//write_log( "$me: submit_dir=$submit_dir" );
	187	//write_log( "$me: requestGUID=$requestGUID" );
	188	write_log( "$me: output_dir=$output_dir" );
	189
	190	$stderr = "";
	191	$stdout = "";
	192	$tarfile = "";
	193	$fn_stderr = "Ultrascan.stderr";
	194	$fn_stdout = "Ultrascan.stdout";
	195	$fn_tarfile = "analysis-results.tar";
[27]	196	$secwait = 10;
[10]	197	$num_try = 0;
[26]	198	write_log( "$me: fn_tarfile=$fn_tarfile" );
[28]	199	while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
[10]	200	{
[27]	201	sleep( $secwait );
[10]	202	$num_try++;
[27]	203	$secwait *= 2;
[25]	204	write_log( "$me: tar-exists: num_try=$num_try" );
[10]	205	}
	206
	207	$ofiles = scandir( $output_dir );
	208	foreach ( $ofiles as $ofile )
	209	{
[27]	210	if ( preg_match( "/^.*stderr$/", $ofile ) )
[10]	211	$fn_stderr = $ofile;
[27]	212	if ( preg_match( "/^.*stdout$/", $ofile ) )
[10]	213	$fn_stdout = $ofile;
	214	//write_log( "$me: ofile=$ofile" );
	215	}
	216	write_log( "$me: fn_stderr=$fn_stderr" );
	217	write_log( "$me: fn_stdout=$fn_stdout" );
	218	if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
	219	else write_log( "$me: NOT FOUND: $fn_tarfile" );
	220
[26]	221	$stderr = '';
	222	$stdout = '';
	223	$tarfile = '';
[10]	224	if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
	225	if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
	226	if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
[26]	227	// If stdout,stderr have no content, retry after delay
	228	if ( strlen( $stdout ) == 0 \|\| strlen( $stderr ) == 0 )
	229	{
	230	sleep( 20 );
	231	if ( file_exists( $fn_stderr ) )
	232	$stderr = file_get_contents( $fn_stderr );
	233	if ( file_exists( $fn_stdout ) )
	234	$stdout = file_get_contents( $fn_stdout );
	235	}
[10]	236
[26]	237	write_log( "$me: length contents stderr,stdout,tarfile -- "
	238	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
	239
[10]	240	if ( $cluster == 'alamo' \|\| $cluster == 'alamo-local' )
	241	{ // Filter "ipath_userinit" lines out of alamo stdout lines
	242	$prefln = strlen( $stdout );
	243	$output = array();
	244	exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
	245	$stdout = implode( "\n", $output );
	246	$posfln = strlen( $stdout );
	247	write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
	248	}
	249
[1]	250	// Save queue messages for post-mortem analysis
	251	$query = "SELECT message, time FROM queue_messages " .
	252	"WHERE analysisID = $analysisID " .
	253	"ORDER BY time ";
	254	$result = mysql_query( $query, $gfac_link );
	255
	256	if ( ! $result )
	257	{
	258	// Just log it and continue
	259	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	260	}
	261
	262	$now = date( 'Y-m-d H:i:s' );
	263	$message_log = "US3 DB: $db\n" .
	264	"RequestID: $requestID\n" .
	265	"GFAC ID: $gfacID\n" .
	266	"Processed: $now\n\n" .
	267	"Queue Messages\n\n" ;
	268	if ( mysql_num_rows( $result ) > 0 )
	269	{
	270	while ( list( $message, $time ) = mysql_fetch_array( $result ) )
	271	$message_log .= "$time $message\n";
	272	}
	273
	274	$query = "DELETE FROM queue_messages " .
	275	"WHERE analysisID = $analysisID ";
	276
	277	$result = mysql_query( $query, $gfac_link );
	278
	279	if ( ! $result )
	280	{
	281	// Just log it and continue
	282	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	283	}
	284
[10]	285	$query = "SELECT queue_msg FROM analysis " .
	286	"WHERE gfacID='$gfacID' ";
	287
[1]	288	$result = mysql_query( $query, $gfac_link );
[10]	289	list( $queue_msg ) = mysql_fetch_array( $result );
[1]	290
	291	// But let's allow for investigation of other large stdout and/or stderr
	292	if ( strlen( $stdout ) > 20480000 \|\|
	293	strlen( $stderr ) > 20480000 )
	294	write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
	295
	296	$message_log .= "\n\n\nStdout Contents\n\n" .
	297	$stdout .
	298	"\n\n\nStderr Contents\n\n" .
	299	$stderr .
	300	"\n\n\nGFAC Status: $status\n" .
	301	"GFAC message field: $queue_msg\n";
	302
	303	// Delete data from GFAC DB
	304	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
	305
	306	$result = mysql_query( $query, $gfac_link );
	307
	308	if ( ! $result )
	309	{
	310	// Just log it and continue
	311	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	312	}
	313
	314
	315	// Try to create it if necessary, and write the file
	316	// Let's use FILE_APPEND, in case this is the second time around and the
	317	// GFAC job status was INSERTed, rather than UPDATEd
	318	if ( ! is_dir( $output_dir ) )
	319	mkdir( $output_dir, 0775, true );
	320	$message_filename = "$output_dir/$db-$requestID-messages.txt";
	321	file_put_contents( $message_filename, $message_log, FILE_APPEND );
	322	// mysql_close( $gfac_link );
	323
	324	/////////
	325	// Insert data into HPCAnalysis
	326
	327	$query = "UPDATE HPCAnalysisResult SET " .
	328	"stderr='" . mysql_real_escape_string( $stderr, $us3_link ) . "', " .
[10]	329	"stdout='" . mysql_real_escape_string( $stdout, $us3_link ) . "', " .
	330	"queueStatus='completed' " .
[1]	331	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	332
	333	$result = mysql_query( $query, $us3_link );
	334
	335	if ( ! $result )
	336	{
	337	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	338	mail_to_user( "fail", "Bad query:\n$query\n" . mysql_error( $us3_link ) );
	339	return( -1 );
	340	}
	341
[10]	342	// Delete data from GFAC DB
	343	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
[1]	344
[10]	345	$result = mysql_query( $query, $gfac_link );
	346
	347	if ( ! $result )
	348	{
	349	// Just log it and continue
	350	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	351	}
	352
	353	// Expand the tar file
	354
[1]	355	if ( strlen( $tarfile ) == 0 )
	356	{
	357	write_log( "$me: No tarfile" );
	358	mail_to_user( "fail", "No results" );
	359	return( -1 );
	360	}
	361
	362	$tar_out = array();
[10]	363	exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
[1]	364
	365	// Insert the model files and noise files
	366	$files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
	367	$noiseIDs = array();
	368	$modelGUIDs = array();
[17]	369	$mrecsIDs = array();
	370	$fns_used = array();
[1]	371
	372	foreach ( $files as $file )
	373	{
	374	$split = explode( ";", $file );
	375
	376	if ( count( $split ) > 1 )
	377	{
	378	list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
	379
	380	list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
	381	list( $other, $variance ) = explode( "=", $variance );
	382	list( $other, $meniscus ) = explode( "=", $meniscus );
	383	}
	384	else
	385	$fn = $file;
	386
[10]	387	if ( preg_match( "/mdl.tmp$/", $fn ) )
	388	continue;
	389
[17]	390	if ( in_array( $fn, $fns_used ) )
	391	continue;
	392
	393	$fns_used[] = $fn;
	394
[1]	395	if ( filesize( $fn ) < 100 )
	396	{
[10]	397	write_log( "$me:fn is invalid $fn size filesize($fn)" );
[1]	398	mail_to_user( "fail", "Internal error\n$fn is invalid" );
	399	return( -1 );
	400	}
	401
	402	if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
	403	{
	404	$xml = file_get_contents( $fn );
	405	$statistics = parse_xml( $xml, 'statistics' );
[10]	406	// $ntries = 0;
	407	//
	408	// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
	409	// { // job_statistics file not totally copied, so retry
	410	// sleep( 10 );
	411	// $xml = file_get_contents( $fn );
	412	// $statistics = parse_xml( $xml, 'statistics' );
	413	// $ntries++;
	414	//write_log( "$me:jobstats retry $ntries" );
	415	// }
	416	//write_log( "$me:cputime=$statistics['cputime']" );
	417
[1]	418	$otherdata = parse_xml( $xml, 'id' );
	419
	420	$query = "UPDATE HPCAnalysisResult SET " .
	421	"wallTime = {$statistics['walltime']}, " .
	422	"CPUTime = {$statistics['cputime']}, " .
	423	"CPUCount = {$statistics['cpucount']}, " .
	424	"max_rss = {$statistics['maxmemory']}, " .
	425	"startTime = '{$otherdata['starttime']}', " .
	426	"endTime = '{$otherdata['endtime']}', " .
	427	"mgroupcount = {$otherdata['groupcount']} " .
	428	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	429	$result = mysql_query( $query, $us3_link );
	430
	431	if ( ! $result )
	432	{
	433	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	434	}
	435
	436	file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
	437
[24]	438	$file_type = "job_stats";
	439	$id = 1;
[1]	440	}
	441
	442	else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
	443	{
	444	$xml = file_get_contents( $fn );
	445	$noise_data = parse_xml( $xml, "noise" );
	446	$type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
	447	$desc = $noise_data[ 'description' ];
	448	$modelGUID = $noise_data[ 'modelGUID' ];
	449	$noiseGUID = $noise_data[ 'noiseGUID' ];
	450
	451	$query = "INSERT INTO noise SET " .
	452	"noiseGUID='$noiseGUID'," .
	453	"modelGUID='$modelGUID'," .
	454	"editedDataID=1, " .
	455	"modelID=1, " .
	456	"noiseType='$type'," .
	457	"description='$desc'," .
	458	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	459
	460	// Add later after all files are processed: editDataID, modelID
	461
	462	$result = mysql_query( $query, $us3_link );
	463
	464	if ( ! $result )
	465	{
	466	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	467	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	468	return( -1 );
	469	}
	470
	471	$id = mysql_insert_id( $us3_link );
	472	$file_type = "noise";
	473	$noiseIDs[] = $id;
	474
	475	// Keep track of modelGUIDs for later, when we replace them
	476	$modelGUIDs[ $id ] = $modelGUID;
	477
	478	}
[10]	479
	480	else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
[1]	481	{
	482	$xml = file_get_contents( $fn );
[10]	483	$mrecs_data = parse_xml( $xml, "modelrecords" );
	484	$desc = $mrecs_data[ 'description' ];
	485	$editGUID = $mrecs_data[ 'editGUID' ];
	486	write_log( "$me: mrecs file editGUID=$editGUID" );
	487	if ( strlen( $editGUID ) < 36 )
	488	$editGUID = "12345678-0123-5678-0123-567890123456";
	489	$mrecGUID = $mrecs_data[ 'mrecGUID' ];
	490	$modelGUID = $mrecs_data[ 'modelGUID' ];
	491
	492	$query = "INSERT INTO pcsa_modelrecs SET " .
	493	"editedDataID=" .
	494	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	495	"modelID=0, " .
	496	"mrecsGUID='$mrecGUID'," .
	497	"description='$desc'," .
	498	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	499
	500	// Add later after all files are processed: editDataID, modelID
	501
	502	$result = mysql_query( $query, $us3_link );
	503
	504	if ( ! $result )
	505	{
	506	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	507	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	508	return( -1 );
	509	}
	510
	511	$id = mysql_insert_id( $us3_link );
	512	$file_type = "mrecs";
	513	$mrecsIDs[] = $id;
	514
	515	// Keep track of modelGUIDs for later, when we replace them
	516	$rmodlGUIDs[ $id ] = $modelGUID;
	517	//write_log( "$me: mrecs file inserted into DB : id=$id" );
	518	}
	519
[23]	520	else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
[10]	521	{
	522	$xml = file_get_contents( $fn );
[1]	523	$model_data = parse_xml( $xml, "model" );
	524	$description = $model_data[ 'description' ];
	525	$modelGUID = $model_data[ 'modelGUID' ];
	526	$editGUID = $model_data[ 'editGUID' ];
	527
[10]	528	if ( $mc_iteration > 1 )
	529	{
	530	$miter = sprintf( "_mcN%03d", $mc_iteration );
	531	$description = preg_replace( "/_mc[0-9]+/", $miter, $description );
	532	write_log( "$me: MODELUpd: O:description=$description" );
	533	}
	534
[1]	535	$query = "INSERT INTO model SET " .
	536	"modelGUID='$modelGUID'," .
	537	"editedDataID=" .
	538	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	539	"description='$description'," .
	540	"MCIteration='$mc_iteration'," .
	541	"meniscus='$meniscus'," .
	542	"variance='$variance'," .
	543	"xml='" . mysql_real_escape_string( $xml, $us3_link ) . "'";
	544
	545	$result = mysql_query( $query, $us3_link );
	546
	547	if ( ! $result )
	548	{
	549	write_log( "$me: Bad query:\n$query " . mysql_error( $us3_link ) );
	550	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	551	return( -1 );
	552	}
	553
	554	$modelID = mysql_insert_id( $us3_link );
	555	$id = $modelID;
	556	$file_type = "model";
	557
	558	$query = "INSERT INTO modelPerson SET " .
	559	"modelID=$modelID, personID=$personID";
	560	$result = mysql_query( $query, $us3_link );
	561	}
	562
[24]	563	else // Undetermined type: skip result data update
	564	continue;
	565
[1]	566	$query = "INSERT INTO HPCAnalysisResultData SET " .
	567	"HPCAnalysisResultID='$HPCAnalysisResultID', " .
	568	"HPCAnalysisResultType='$file_type', " .
	569	"resultID=$id";
	570
	571	$result = mysql_query( $query, $us3_link );
	572
	573	if ( ! $result )
	574	{
	575	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	576	mail_to_user( "fail", "Internal error\n$query\n" . mysql_error( $us3_link ) );
	577	return( -1 );
	578	}
	579	}
	580
	581	// Now fix up noise entries
	582	// For noise files, there is, at most two: ti_noise and ri_noise
	583	// In this case there will only be one modelID
	584
	585	foreach ( $noiseIDs as $noiseID )
	586	{
	587	$modelGUID = $modelGUIDs[ $noiseID ];
	588	$query = "UPDATE noise SET " .
	589	"editedDataID=" .
[6]	590	"(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
[1]	591	"modelID=" .
	592	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	593	"WHERE noiseID=$noiseID";
	594
	595	$result = mysql_query( $query, $us3_link );
	596
	597	if ( ! $result )
	598	{
	599	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	600	mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
	601	return( -1 );
	602	}
	603	}
	604
[13]	605	// Now possibly fix up mrecs entries
	606
	607	foreach ( $mrecsIDs as $mrecsID )
	608	{
	609	$modelGUID = $rmodlGUIDs[ $mrecsID ];
	610	$query = "UPDATE pcsa_modelrecs SET " .
	611	"modelID=" .
	612	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	613	"WHERE mrecsID=$mrecsID";
	614
	615	$result = mysql_query( $query, $us3_link );
	616
	617	if ( ! $result )
	618	{
	619	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	620	mail_to_user( "fail", "Bad query\n$query\n" . mysql_error( $us3_link ) );
	621	return( -1 );
	622	}
	623	write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
	624	}
	625	//write_log( "$me: mrecs entries updated" );
	626
[1]	627	// Copy results to LIMS submit directory (files there are deleted after 7 days)
	628	global $submit_dir; // LIMS submit files dir
	629
	630	// Get the request guid (LIMS submit dir name)
	631	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	632	"WHERE HPCAnalysisRequestID = $requestID ";
	633	$result = mysql_query( $query, $us3_link );
	634
	635	if ( ! $result )
	636	{
	637	write_log( "$me: Bad query:\n$query\n" . mysql_error( $us3_link ) );
	638	}
	639
[10]	640	// list( $requestGUID ) = mysql_fetch_array( $result );
	641	//
	642	// chdir( "$submit_dir/$requestGUID" );
	643	// $f = fopen( "analysis-results.tar", "w" );
	644	// fwrite( $f, $tarfile );
	645	// fclose( $f );
[1]	646
	647	// Clean up
[10]	648	// chdir ( $work );
[1]	649	// exec( "rm -rf $gfacID" );
	650
	651	mysql_close( $us3_link );
	652
	653	/////////
	654	// Send email
	655
	656	mail_to_user( "success", "" );
	657	}
	658
	659	function mail_to_user( $type, $msg )
	660	{
	661	// Note to me. Just changed subject line to include a modified $status instead
	662	// of the $type variable passed. More informative than just "fail" or "success."
	663	// See how it works for awhile and then consider removing $type parameter from
	664	// function.
	665	global $email_address;
	666	global $submittime;
	667	global $queuestatus;
	668	global $status;
	669	global $cluster;
	670	global $jobtype;
	671	global $org_name;
	672	global $admin_email;
[6]	673	global $db;
[1]	674	global $dbhost;
	675	global $requestID;
	676	global $gfacID;
	677	global $editXMLFilename;
[5]	678	global $stdout;
[25]	679	global $org_domain;
[1]	680
	681	global $me;
	682	write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
	683
	684	// Get GFAC status and message
[4]	685	// function get_gfac_message() also sets global $status
	686	$gfac_message = get_gfac_message( $gfacID );
[10]	687	if ( $gfac_message === false ) $gfac_message = "Job Finished";
[1]	688
	689	// Create a status to put in the subject line
	690	switch ( $status )
	691	{
	692	case "COMPLETE":
	693	$subj_status = 'completed';
	694	break;
	695
	696	case "CANCELLED":
	697	case "CANCELED":
	698	$subj_status = 'canceled';
	699	break;
	700
	701	case "FAILED":
	702	$subj_status = 'failed';
[10]	703	if ( preg_match( "/^US3-A/i", $gfacID ) )
	704	{ // For A/Thrift FAIL, get error message
	705	$gfac_message = getExperimentErrors( $gfacID );
	706	//$gfac_message .= "Test ERROR MESSAGE";
	707	}
[1]	708	break;
	709
	710	case "ERROR":
	711	$subj_status = 'unknown error';
	712	break;
	713
	714	default:
	715	$subj_status = $status; // For now
	716	break;
	717
	718	}
	719
[10]	720	$queuestatus = $subj_status;
	721	$limshost = $dbhost;
	722	if ( $limshost == 'localhost' )
[25]	723	{
[10]	724	$limshost = gethostname();
[25]	725	if ( ! preg_match( "/\./", $limshost ) )
	726	$limshost = $limshost . $org_domain;
	727	}
[10]	728
[1]	729	// Parse the editXMLFilename
	730	list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
	731	explode( ".", $editXMLFilename );
	732
	733	$headers = "From: $org_name Admin<$admin_email>" . "\n";
	734	$headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
	735
	736	// Set the reply address
	737	$headers .= "Reply-To: $org_name<$admin_email>" . "\n";
	738	$headers .= "Return-Path: $org_name<$admin_email>" . "\n";
	739
	740	// Try to avoid spam filters
	741	$now = time();
	742	$headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
	743	$headers .= "X-Mailer: PHP v" . phpversion() . "\n";
	744	$headers .= "MIME-Version: 1.0" . "\n";
	745	$headers .= "Content-Transfer-Encoding: 8bit" . "\n";
	746
	747	$subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
	748	$message = "
	749	Your UltraScan job is complete:
	750
	751	Submission Time : $submittime
[10]	752	LIMS Host : $limshost
[1]	753	Analysis ID : $gfacID
[6]	754	Request ID : $requestID ( $db )
[1]	755	RunID : $runID
	756	EditID : $editID
	757	Data Type : $dataType
	758	Cell/Channel/Wl : $cell / $channel / $wl
	759	Status : $queuestatus
	760	Cluster : $cluster
	761	Job Type : $jobtype
	762	GFAC Status : $status
[4]	763	GFAC Message : $gfac_message
[5]	764	Stdout : $stdout
[1]	765	";
	766
[4]	767	if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
[1]	768
	769	// Handle the error case where an error occurs before fetching the
	770	// user's email address
	771	if ( $email_address == "" ) $email_address = $admin_email;
	772
	773	mail( $email_address, $subject, $message, $headers );
	774	}
	775
	776	function parse_xml( $xml, $type )
	777	{
	778	$parser = new XMLReader();
	779	$parser->xml( $xml );
	780
	781	$results = array();
	782
	783	while ( $parser->read() )
	784	{
	785	if ( $parser->name == $type )
	786	{
	787	while ( $parser->moveToNextAttribute() )
	788	{
	789	$results[ $parser->name ] = $parser->value;
	790	}
	791
	792	break;
	793	}
	794	}
	795
	796	$parser->close();
	797	return $results;
	798	}
	799
	800	// Function to get information about the current job GFAC
	801	function get_gfac_message( $gfacID )
	802	{
	803	global $serviceURL;
	804	global $me;
	805
	806	$hex = "[0-9a-fA-F]";
	807	if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
	808	! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
	809	{
	810	// Then it's not a GFAC job
	811	return false;
	812	}
	813
	814	$url = "$serviceURL/jobstatus/$gfacID";
	815	try
	816	{
	817	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	818	$http = $post->send();
	819	$xml = $post->getResponseBody();
	820	}
	821	catch ( HttpException $e )
	822	{
	823	write_log( "$me: Job status not available - $gfacID" );
	824	return false;
	825	}
	826
	827	// Parse the result
	828	$gfac_message = parse_message( $xml );
	829
	830	return $gfac_message;
	831	}
	832
	833	function parse_message( $xml )
	834	{
[4]	835	global $status;
[1]	836	$status = "";
	837	$gfac_message = "";
	838
	839	$parser = new XMLReader();
	840	$parser->xml( $xml );
	841
	842	$results = array();
	843
	844	while( $parser->read() )
	845	{
	846	$type = $parser->nodeType;
	847
	848	if ( $type == XMLReader::ELEMENT )
	849	$name = $parser->name;
	850
	851	else if ( $type == XMLReader::TEXT )
	852	{
	853	if ( $name == "status" )
	854	$status = $parser->value;
	855	else
	856	$gfac_message = $parser->value;
	857	}
	858	}
	859
	860	$parser->close();
	861	return $gfac_message;
	862	}
	863
	864	function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
	865	{
	866	global $work;
	867	global $work_remote;
	868	global $me;
	869	global $db;
	870	global $status;
[25]	871	$is_us3iab = preg_match( "/us3iab/", $cluster );
[1]	872
	873	// Figure out remote directory
	874	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
[28]	875	//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
[1]	876
	877	// Get stdout, stderr, output/analysis-results.tar
	878	$output = array();
	879
[25]	880	if ( $is_us3iab == 0 )
	881	{
[28]	882	// For "-local", recompute remote work directory
	883	$cmd = "ssh us3@$cluster.uthscsa.edu 'ls -d ~us3/lims/work/local' 2/dev/null";
	884	exec( $cmd, $output, $stat );
	885	$work_remote = $output[ 0 ];
	886	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
	887	write_log( "$me: -LOCAL: remoteDir=$remoteDir" );
	888
[25]	889	// Figure out local working directory
	890	if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
	891	$pwd = chdir( "$work/$gfacID" );
[1]	892
[25]	893	$cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
	894
[1]	895	exec( $cmd, $output, $stat );
[25]	896	if ( $stat != 0 )
[1]	897	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	898
	899	$cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
	900
[1]	901	exec( $cmd, $output, $stat );
[25]	902	if ( $stat != 0 )
	903	{
[1]	904	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	905	sleep( 10 );
	906	write_log( "$me: RETRY" );
	907	exec( $cmd, $output, $stat );
	908	if ( $stat != 0 )
	909	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	910	}
	911
	912	$cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
	913
	914	exec( $cmd, $output, $stat );
	915	if ( $stat != 0 )
	916	{
	917	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	918	sleep( 10 );
	919	write_log( "$me: RETRY" );
	920	exec( $cmd, $output, $stat );
	921	if ( $stat != 0 )
	922	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	923	}
[1]	924	}
[25]	925	else
	926	{
	927	$pwd = chdir( "$remoteDir" );
	928	write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
	929	}
[1]	930
[25]	931
[1]	932	// Write the files to gfacDB
	933
[25]	934	if ( file_exists( "stderr" ) )
	935	{ // Filter stderr to not have libnnls debug lines
[27]	936	// exec( "mv stderr stderr+nnls", $output, $stat );
	937	// exec( "grep -vi nnls stderr+nnls >stderr", $output, $stat );
[25]	938	$stderr = file_get_contents( "stderr" );
	939	}
	940	else
	941	$stderr = "";
[1]	942	if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
[25]	943	$fn1_tarfile = "analysis-results.tar";
	944	$fn2_tarfile = "output/" . $fn1_tarfile;
	945	if ( file_exists( $fn1_tarfile ) )
	946	$tarfile = file_get_contents( $fn1_tarfile );
	947	else if ( file_exists( $fn2_tarfile ) )
	948	$tarfile = file_get_contents( $fn2_tarfile );
[1]	949
[25]	950	$lense = strlen( $stderr );
[28]	951	if ( $lense > 1000000 )
[25]	952	{ // Replace exceptionally large stderr with smaller version
	953	exec( "mv stderr stderr-orig", $output, $stat );
	954	exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
	955	exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
	956	exec( "cat stderr-h stderr-t >stderr", $output, $stat );
	957	$stderr = file_get_contents( "stderr" );
	958	}
	959	$lent = strlen( $tarfile );
	960	write_log( "$me: tarfile size: $lent");
	961	$lene = strlen( $stderr );
	962	write_log( "$me: stderr size: $lene (was $lense)");
	963	$leno = strlen( $stdout );
	964	write_log( "$me: stdout size: $leno");
	965	$estarf=mysql_real_escape_string($tarfile,$gfac_link);
	966	$lenf = strlen($estarf);
	967	write_log( "$me: es-tarfile size: $lenf");
	968	$esstdo=mysql_real_escape_string($stdout,$gfac_link);
	969	$leno = strlen($esstdo);
	970	write_log( "$me: es-stdout size: $leno");
	971	$esstde=mysql_real_escape_string($stderr,$gfac_link);
	972	$lene = strlen($esstde);
	973	write_log( "$me: es-stderr size: $lene");
[1]	974	$query = "UPDATE analysis SET " .
	975	"stderr='" . mysql_real_escape_string( $stderr, $gfac_link ) . "'," .
	976	"stdout='" . mysql_real_escape_string( $stdout, $gfac_link ) . "'," .
	977	"tarfile='" . mysql_real_escape_string( $tarfile, $gfac_link ) . "'";
	978
	979	$result = mysql_query( $query, $gfac_link );
	980
	981	if ( ! $result )
	982	{
	983	write_log( "$me: Bad query:\n$query\n" . mysql_error( $gfac_link ) );
	984	echo "Bad query\n";
	985	return( -1 );
	986	}
	987	}
	988	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: