Context Navigation

source: trunk/cleanup.php@ 41

Last change on this file since 41 was 41, checked in by gegorbet, 6 years ago
gridctl mods for uslims.uleth.ca demeler3 cluster
File size: 33.4 KB

Rev	Line
[1]	1	<?php
	2	/*
	3	* cleanup.php
	4	*
	5	* functions relating to copying results and cleaning up the gfac DB
[10]	6	* where the job used an Airavata interface.
[1]	7	*
	8	*/
	9
	10	$email_address = '';
	11	$queuestatus = '';
	12	$jobtype = '';
	13	$db = '';
	14	$editXMLFilename = '';
[4]	15	$status = '';
[1]	16
[10]	17	function aira_cleanup( $us3_db, $reqID, $gfac_link )
[1]	18	{
[25]	19	global $org_domain;
[1]	20	global $dbhost;
	21	global $user;
	22	global $passwd;
	23	global $db;
	24	global $guser;
	25	global $gpasswd;
	26	global $gDB;
	27	global $me;
	28	global $work;
	29	global $email_address;
	30	global $queuestatus;
	31	global $jobtype;
	32	global $editXMLFilename;
	33	global $submittime;
[40]	34	global $endtime;
[4]	35	global $status;
[10]	36	global $stderr;
[5]	37	global $stdout;
[10]	38	global $tarfile;
[6]	39	global $requestID;
[10]	40	global $submit_dir;
	41	$me = 'cleanup_aira.php';
[1]	42
[6]	43	$requestID = $reqID;
[1]	44	$db = $us3_db;
	45	write_log( "$me: debug db=$db; requestID=$requestID" );
	46
[35]	47	$us3_link = mysqli_connect( $dbhost, $user, $passwd, $db );
[1]	48
	49	if ( ! $us3_link )
	50	{
[35]	51	write_log( "$me: could not connect: $dbhost, $user, $passwd, $db" );
[1]	52	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB" );
	53	return( -1 );
	54	}
	55
	56	// First get basic info for email messages
	57	$query = "SELECT email, investigatorGUID, editXMLFilename FROM HPCAnalysisRequest " .
	58	"WHERE HPCAnalysisRequestID=$requestID";
[35]	59	$result = mysqli_query( $us3_link, $query );
[1]	60
	61	if ( ! $result )
	62	{
	63	write_log( "$me: Bad query: $query" );
[35]	64	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
[1]	65	return( -1 );
	66	}
	67
[35]	68	list( $email_address, $investigatorGUID, $editXMLFilename ) = mysqli_fetch_array( $result );
[1]	69
	70	$query = "SELECT personID FROM people " .
	71	"WHERE personGUID='$investigatorGUID'";
[35]	72	$result = mysqli_query( $us3_link, $query );
[1]	73
[35]	74	list( $personID ) = mysqli_fetch_array( $result );
[1]	75
[40]	76	$query = "SELECT clusterName, submitTime, queueStatus, analType " .
	77	"FROM HPCAnalysisRequest h, HPCAnalysisResult r " .
	78	"WHERE h.HPCAnalysisRequestID=$requestID " .
[1]	79	"AND h.HPCAnalysisRequestID=r.HPCAnalysisRequestID";
	80
[35]	81	$result = mysqli_query( $us3_link, $query );
[1]	82
	83	if ( ! $result )
	84	{
[35]	85	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	86	return( -1 );
	87	}
	88
[35]	89	if ( mysqli_num_rows( $result ) == 0 )
[1]	90	{
	91	write_log( "$me: US3 Table error - No records for requestID: $requestID" );
	92	return( -1 );
	93	}
	94
[35]	95	list( $cluster, $submittime, $queuestatus, $jobtype ) = mysqli_fetch_array( $result );
[1]	96
	97	// Get the GFAC ID
[40]	98	$query = "SELECT HPCAnalysisResultID, gfacID, endTime FROM HPCAnalysisResult " .
[1]	99	"WHERE HPCAnalysisRequestID=$requestID";
	100
[35]	101	$result = mysqli_query( $us3_link, $query );
[1]	102
	103	if ( ! $result )
	104	{
	105	write_log( "$me: Bad query: $query" );
[35]	106	mail_to_user( "fail", "Internal Error $requestID\n$query\n" . mysqli_error( $us3_link ) );
[1]	107	return( -1 );
	108	}
	109
[40]	110	list( $HPCAnalysisResultID, $gfacID, $endtime ) = mysqli_fetch_array( $result );
[1]	111
[10]	112	// Get data from global GFAC DB then insert it into US3 DB
[1]	113
[35]	114	/*
	115	$result = mysqli_select_db( $gfac_link, $gDB );
[1]	116
	117	if ( ! $result )
	118	{
	119	write_log( "$me: Could not connect to DB $gDB" );
	120	mail_to_user( "fail", "Internal Error $requestID\nCould not connect to DB $gDB" );
	121	return( -1 );
	122	}
[35]	123	*/
[1]	124
	125	$query = "SELECT status, cluster, id FROM analysis " .
	126	"WHERE gfacID='$gfacID'";
	127
[35]	128	$result = mysqli_query( $gfac_link, $query );
[1]	129	if ( ! $result )
	130	{
	131	write_log( "$me: Could not select GFAC status for $gfacID" );
	132	mail_to_user( "fail", "Could not select GFAC status for $gfacID" );
	133	return( -1 );
	134	}
	135
[35]	136	list( $status, $cluster, $id ) = mysqli_fetch_array( $result );
[1]	137
[25]	138	$is_us3iab = preg_match( "/us3iab/", $cluster );
	139	$is_local = preg_match( "/-local/", $cluster );
	140
	141	if ( $is_us3iab \|\| $is_local )
[1]	142	{
	143	$clushost = $cluster;
	144	$clushost = preg_replace( "/\-local/", "", $clushost );
	145	get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
	146	}
	147
[10]	148
	149	$query = "SELECT id FROM analysis " .
[1]	150	"WHERE gfacID='$gfacID'";
	151
[35]	152	$result = mysqli_query( $gfac_link, $query );
[1]	153
	154	if ( ! $result )
	155	{
[35]	156	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
	157	mail_to_user( "fail", "Internal error " . mysqli_error( $gfac_link ) );
[1]	158	return( -1 );
	159	}
	160
[35]	161	list( $analysisID ) = mysqli_fetch_array( $result );
[1]	162
[10]	163	// Get the request guid (LIMS submit dir name)
	164	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	165	"WHERE HPCAnalysisRequestID = $requestID ";
[35]	166	$result = mysqli_query( $us3_link, $query );
[10]	167
	168	if ( ! $result )
	169	{
[35]	170	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[10]	171	}
	172
[35]	173	list( $requestGUID ) = mysqli_fetch_array( $result );
[10]	174	$output_dir = "$submit_dir/$requestGUID";
	175
	176	// Get stderr,stdout,tarfile from work directory
	177	if ( ! is_dir( "$output_dir" ) ) mkdir( "$output_dir", 0770 );
	178	chdir( "$output_dir" );
	179	//write_log( "$me: gfacID=$gfacID" );
	180	//write_log( "$me: submit_dir=$submit_dir" );
	181	//write_log( "$me: requestGUID=$requestGUID" );
	182	write_log( "$me: output_dir=$output_dir" );
	183
	184	$stderr = "";
	185	$stdout = "";
	186	$tarfile = "";
	187	$fn_stderr = "Ultrascan.stderr";
	188	$fn_stdout = "Ultrascan.stdout";
	189	$fn_tarfile = "analysis-results.tar";
[27]	190	$secwait = 10;
[10]	191	$num_try = 0;
[26]	192	write_log( "$me: fn_tarfile=$fn_tarfile" );
[28]	193	while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
[10]	194	{
[27]	195	sleep( $secwait );
[10]	196	$num_try++;
[27]	197	$secwait *= 2;
[25]	198	write_log( "$me: tar-exists: num_try=$num_try" );
[10]	199	}
	200
	201	$ofiles = scandir( $output_dir );
	202	foreach ( $ofiles as $ofile )
	203	{
[27]	204	if ( preg_match( "/^.*stderr$/", $ofile ) )
[10]	205	$fn_stderr = $ofile;
[27]	206	if ( preg_match( "/^.*stdout$/", $ofile ) )
[10]	207	$fn_stdout = $ofile;
	208	//write_log( "$me: ofile=$ofile" );
	209	}
	210	write_log( "$me: fn_stderr=$fn_stderr" );
	211	write_log( "$me: fn_stdout=$fn_stdout" );
	212	if (file_exists($fn_tarfile)) write_log( "$me: fn_tarfile=$fn_tarfile" );
	213	else write_log( "$me: NOT FOUND: $fn_tarfile" );
	214
[26]	215	$stderr = '';
	216	$stdout = '';
	217	$tarfile = '';
[10]	218	if ( file_exists( $fn_stderr ) ) $stderr = file_get_contents( $fn_stderr );
	219	if ( file_exists( $fn_stdout ) ) $stdout = file_get_contents( $fn_stdout );
	220	if ( file_exists( $fn_tarfile ) ) $tarfile = file_get_contents( $fn_tarfile );
[31]	221	write_log( "$me(0): length contents stderr,stdout,tarfile -- "
	222	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
[26]	223	// If stdout,stderr have no content, retry after delay
	224	if ( strlen( $stdout ) == 0 \|\| strlen( $stderr ) == 0 )
	225	{
	226	sleep( 20 );
	227	if ( file_exists( $fn_stderr ) )
	228	$stderr = file_get_contents( $fn_stderr );
	229	if ( file_exists( $fn_stdout ) )
	230	$stdout = file_get_contents( $fn_stdout );
	231	}
[10]	232
[26]	233	write_log( "$me: length contents stderr,stdout,tarfile -- "
	234	. strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
	235
[10]	236	if ( $cluster == 'alamo' \|\| $cluster == 'alamo-local' )
	237	{ // Filter "ipath_userinit" lines out of alamo stdout lines
	238	$prefln = strlen( $stdout );
	239	$output = array();
	240	exec( "grep -v 'ipath_userinit' $fn_stdout 2>&1", $output, $err );
	241	$stdout = implode( "\n", $output );
	242	$posfln = strlen( $stdout );
	243	write_log( "$me: fn_stdout : filtered. Length $prefln -> $posfln ." );
	244	}
	245
[1]	246	// Save queue messages for post-mortem analysis
	247	$query = "SELECT message, time FROM queue_messages " .
	248	"WHERE analysisID = $analysisID " .
	249	"ORDER BY time ";
[35]	250	$result = mysqli_query( $gfac_link, $query );
[1]	251
	252	if ( ! $result )
	253	{
	254	// Just log it and continue
[35]	255	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	256	}
	257
	258	$now = date( 'Y-m-d H:i:s' );
	259	$message_log = "US3 DB: $db\n" .
	260	"RequestID: $requestID\n" .
	261	"GFAC ID: $gfacID\n" .
	262	"Processed: $now\n\n" .
	263	"Queue Messages\n\n" ;
[35]	264	if ( mysqli_num_rows( $result ) > 0 )
[1]	265	{
[35]	266	while ( list( $message, $time ) = mysqli_fetch_array( $result ) )
[1]	267	$message_log .= "$time $message\n";
	268	}
	269
	270	$query = "DELETE FROM queue_messages " .
	271	"WHERE analysisID = $analysisID ";
	272
[35]	273	$result = mysqli_query( $gfac_link, $query );
[1]	274
	275	if ( ! $result )
	276	{
	277	// Just log it and continue
[35]	278	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	279	}
	280
[10]	281	$query = "SELECT queue_msg FROM analysis " .
	282	"WHERE gfacID='$gfacID' ";
	283
[35]	284	$result = mysqli_query( $gfac_link, $query );
	285	list( $queue_msg ) = mysqli_fetch_array( $result );
[1]	286
	287	// But let's allow for investigation of other large stdout and/or stderr
	288	if ( strlen( $stdout ) > 20480000 \|\|
	289	strlen( $stderr ) > 20480000 )
	290	write_log( "$me: stdout + stderr larger than 20M - $gfacID\n" );
	291
	292	$message_log .= "\n\n\nStdout Contents\n\n" .
	293	$stdout .
	294	"\n\n\nStderr Contents\n\n" .
	295	$stderr .
	296	"\n\n\nGFAC Status: $status\n" .
	297	"GFAC message field: $queue_msg\n";
	298
	299	// Delete data from GFAC DB
	300	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
	301
[35]	302	$result = mysqli_query( $gfac_link, $query );
[1]	303
	304	if ( ! $result )
	305	{
	306	// Just log it and continue
[35]	307	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	308	}
	309
	310
	311	// Try to create it if necessary, and write the file
	312	// Let's use FILE_APPEND, in case this is the second time around and the
	313	// GFAC job status was INSERTed, rather than UPDATEd
	314	if ( ! is_dir( $output_dir ) )
	315	mkdir( $output_dir, 0775, true );
	316	$message_filename = "$output_dir/$db-$requestID-messages.txt";
	317	file_put_contents( $message_filename, $message_log, FILE_APPEND );
[35]	318	// mysqli_close( $gfac_link );
[1]	319
	320	/////////
	321	// Insert data into HPCAnalysis
	322
	323	$query = "UPDATE HPCAnalysisResult SET " .
[35]	324	"stderr='" . mysqli_real_escape_string( $us3_link, $stderr ) . "', " .
	325	"stdout='" . mysqli_real_escape_string( $us3_link, $stdout ) . "', " .
[10]	326	"queueStatus='completed' " .
[1]	327	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
	328
[35]	329	$result = mysqli_query( $us3_link, $query );
[1]	330
	331	if ( ! $result )
	332	{
[35]	333	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	334	mail_to_user( "fail", "Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	335	return( -1 );
	336	}
	337
[10]	338	// Delete data from GFAC DB
	339	$query = "DELETE from analysis WHERE gfacID='$gfacID'";
[1]	340
[35]	341	$result = mysqli_query( $gfac_link, $query );
[10]	342
	343	if ( ! $result )
	344	{
	345	// Just log it and continue
[35]	346	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[10]	347	}
	348
	349	// Expand the tar file
	350
[1]	351	if ( strlen( $tarfile ) == 0 )
	352	{
	353	write_log( "$me: No tarfile" );
	354	mail_to_user( "fail", "No results" );
	355	return( -1 );
	356	}
	357
	358	$tar_out = array();
[10]	359	exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
[1]	360
	361	// Insert the model files and noise files
	362	$files = file( "analysis_files.txt", FILE_IGNORE_NEW_LINES );
	363	$noiseIDs = array();
	364	$modelGUIDs = array();
[17]	365	$mrecsIDs = array();
	366	$fns_used = array();
[1]	367
	368	foreach ( $files as $file )
	369	{
	370	$split = explode( ";", $file );
	371
	372	if ( count( $split ) > 1 )
	373	{
	374	list( $fn, $meniscus, $mc_iteration, $variance ) = explode( ";", $file );
	375
	376	list( $other, $mc_iteration ) = explode( "=", $mc_iteration );
	377	list( $other, $variance ) = explode( "=", $variance );
	378	list( $other, $meniscus ) = explode( "=", $meniscus );
	379	}
	380	else
	381	$fn = $file;
	382
[10]	383	if ( preg_match( "/mdl.tmp$/", $fn ) )
	384	continue;
	385
[17]	386	if ( in_array( $fn, $fns_used ) )
	387	continue;
	388
	389	$fns_used[] = $fn;
	390
[1]	391	if ( filesize( $fn ) < 100 )
	392	{
[10]	393	write_log( "$me:fn is invalid $fn size filesize($fn)" );
[1]	394	mail_to_user( "fail", "Internal error\n$fn is invalid" );
	395	return( -1 );
	396	}
	397
	398	if ( preg_match( "/^job_statistics\.xml$/", $fn ) ) // Job statistics file
	399	{
	400	$xml = file_get_contents( $fn );
	401	$statistics = parse_xml( $xml, 'statistics' );
[10]	402	// $ntries = 0;
	403	//
	404	// while ( $statistics['cpucount'] < 1 && $ntries < 3 )
	405	// { // job_statistics file not totally copied, so retry
	406	// sleep( 10 );
	407	// $xml = file_get_contents( $fn );
	408	// $statistics = parse_xml( $xml, 'statistics' );
	409	// $ntries++;
	410	//write_log( "$me:jobstats retry $ntries" );
	411	// }
	412	//write_log( "$me:cputime=$statistics['cputime']" );
	413
[1]	414	$otherdata = parse_xml( $xml, 'id' );
	415
	416	$query = "UPDATE HPCAnalysisResult SET " .
	417	"wallTime = {$statistics['walltime']}, " .
	418	"CPUTime = {$statistics['cputime']}, " .
	419	"CPUCount = {$statistics['cpucount']}, " .
	420	"max_rss = {$statistics['maxmemory']}, " .
	421	"startTime = '{$otherdata['starttime']}', " .
	422	"endTime = '{$otherdata['endtime']}', " .
	423	"mgroupcount = {$otherdata['groupcount']} " .
	424	"WHERE HPCAnalysisResultID=$HPCAnalysisResultID";
[35]	425	$result = mysqli_query( $us3_link, $query );
[1]	426
	427	if ( ! $result )
	428	{
[35]	429	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	430	}
	431
	432	file_put_contents( "$output_dir/$fn", $xml ); // Copy to submit dir
	433
[24]	434	$file_type = "job_stats";
	435	$id = 1;
[1]	436	}
	437
	438	else if ( preg_match( "/\.noise/", $fn ) > 0 ) // It's a noise file
	439	{
	440	$xml = file_get_contents( $fn );
	441	$noise_data = parse_xml( $xml, "noise" );
	442	$type = ( $noise_data[ 'type' ] == "ri" ) ? "ri_noise" : "ti_noise";
	443	$desc = $noise_data[ 'description' ];
	444	$modelGUID = $noise_data[ 'modelGUID' ];
	445	$noiseGUID = $noise_data[ 'noiseGUID' ];
[35]	446	$editGUID = '00000000-0000-0000-0000-000000000000';
	447	if ( isset( $model_data[ 'editGUID' ] ) )
	448	$editGUID = $model_data[ 'editGUID' ];
[1]	449
	450	$query = "INSERT INTO noise SET " .
	451	"noiseGUID='$noiseGUID'," .
	452	"modelGUID='$modelGUID'," .
[35]	453	"editedDataID=" .
	454	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
[1]	455	"modelID=1, " .
	456	"noiseType='$type'," .
	457	"description='$desc'," .
[35]	458	"xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
[1]	459
	460	// Add later after all files are processed: editDataID, modelID
	461
[35]	462	$result = mysqli_query( $us3_link, $query );
[1]	463
	464	if ( ! $result )
	465	{
[35]	466	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	467	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[1]	468	return( -1 );
	469	}
	470
[35]	471	$id = mysqli_insert_id( $us3_link );
[1]	472	$file_type = "noise";
	473	$noiseIDs[] = $id;
	474
	475	// Keep track of modelGUIDs for later, when we replace them
	476	$modelGUIDs[ $id ] = $modelGUID;
	477
	478	}
[10]	479
	480	else if ( preg_match( "/\.mrecs/", $fn ) > 0 ) // It's an mrecs file
[1]	481	{
	482	$xml = file_get_contents( $fn );
[10]	483	$mrecs_data = parse_xml( $xml, "modelrecords" );
	484	$desc = $mrecs_data[ 'description' ];
	485	$editGUID = $mrecs_data[ 'editGUID' ];
	486	write_log( "$me: mrecs file editGUID=$editGUID" );
	487	if ( strlen( $editGUID ) < 36 )
	488	$editGUID = "12345678-0123-5678-0123-567890123456";
	489	$mrecGUID = $mrecs_data[ 'mrecGUID' ];
	490	$modelGUID = $mrecs_data[ 'modelGUID' ];
	491
	492	$query = "INSERT INTO pcsa_modelrecs SET " .
	493	"editedDataID=" .
	494	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	495	"modelID=0, " .
	496	"mrecsGUID='$mrecGUID'," .
	497	"description='$desc'," .
[35]	498	"xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
[10]	499
	500	// Add later after all files are processed: editDataID, modelID
	501
[35]	502	$result = mysqli_query( $us3_link, $query );
[10]	503
	504	if ( ! $result )
	505	{
[35]	506	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	507	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[10]	508	return( -1 );
	509	}
	510
[35]	511	$id = mysqli_insert_id( $us3_link );
[10]	512	$file_type = "mrecs";
	513	$mrecsIDs[] = $id;
	514
	515	// Keep track of modelGUIDs for later, when we replace them
	516	$rmodlGUIDs[ $id ] = $modelGUID;
	517	//write_log( "$me: mrecs file inserted into DB : id=$id" );
	518	}
	519
[23]	520	else if ( preg_match( "/\.model/", $fn ) > 0 ) // It's a model file
[10]	521	{
	522	$xml = file_get_contents( $fn );
[1]	523	$model_data = parse_xml( $xml, "model" );
	524	$description = $model_data[ 'description' ];
	525	$modelGUID = $model_data[ 'modelGUID' ];
	526	$editGUID = $model_data[ 'editGUID' ];
	527
[10]	528	if ( $mc_iteration > 1 )
	529	{
	530	$miter = sprintf( "_mcN%03d", $mc_iteration );
	531	$description = preg_replace( "/_mc[0-9]+/", $miter, $description );
	532	write_log( "$me: MODELUpd: O:description=$description" );
	533	}
	534
[1]	535	$query = "INSERT INTO model SET " .
	536	"modelGUID='$modelGUID'," .
	537	"editedDataID=" .
	538	"(SELECT editedDataID FROM editedData WHERE editGUID='$editGUID')," .
	539	"description='$description'," .
	540	"MCIteration='$mc_iteration'," .
	541	"meniscus='$meniscus'," .
	542	"variance='$variance'," .
[35]	543	"xml='" . mysqli_real_escape_string( $us3_link, $xml ) . "'";
[1]	544
[35]	545	$result = mysqli_query( $us3_link, $query );
[1]	546
	547	if ( ! $result )
	548	{
[35]	549	write_log( "$me: Bad query:\n$query " . mysqli_error( $us3_link ) );
	550	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[1]	551	return( -1 );
	552	}
	553
[35]	554	$modelID = mysqli_insert_id( $us3_link );
[1]	555	$id = $modelID;
	556	$file_type = "model";
	557
	558	$query = "INSERT INTO modelPerson SET " .
	559	"modelID=$modelID, personID=$personID";
[35]	560	$result = mysqli_query( $us3_link, $query );
[1]	561	}
	562
[24]	563	else // Undetermined type: skip result data update
	564	continue;
	565
[1]	566	$query = "INSERT INTO HPCAnalysisResultData SET " .
	567	"HPCAnalysisResultID='$HPCAnalysisResultID', " .
	568	"HPCAnalysisResultType='$file_type', " .
	569	"resultID=$id";
	570
[35]	571	$result = mysqli_query( $us3_link, $query );
[1]	572
	573	if ( ! $result )
	574	{
[35]	575	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	576	mail_to_user( "fail", "Internal error\n$query\n" . mysqli_error( $us3_link ) );
[1]	577	return( -1 );
	578	}
	579	}
	580
	581	// Now fix up noise entries
	582	// For noise files, there is, at most two: ti_noise and ri_noise
	583	// In this case there will only be one modelID
	584
	585	foreach ( $noiseIDs as $noiseID )
	586	{
	587	$modelGUID = $modelGUIDs[ $noiseID ];
	588	$query = "UPDATE noise SET " .
	589	"editedDataID=" .
[6]	590	"(SELECT editedDataID FROM model WHERE modelGUID='$modelGUID')," .
[1]	591	"modelID=" .
	592	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	593	"WHERE noiseID=$noiseID";
	594
[35]	595	$result = mysqli_query( $us3_link, $query );
[1]	596
	597	if ( ! $result )
	598	{
[35]	599	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	600	mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
[1]	601	return( -1 );
	602	}
	603	}
	604
[13]	605	// Now possibly fix up mrecs entries
	606
	607	foreach ( $mrecsIDs as $mrecsID )
	608	{
	609	$modelGUID = $rmodlGUIDs[ $mrecsID ];
	610	$query = "UPDATE pcsa_modelrecs SET " .
	611	"modelID=" .
	612	"(SELECT modelID FROM model WHERE modelGUID='$modelGUID')" .
	613	"WHERE mrecsID=$mrecsID";
	614
[35]	615	$result = mysqli_query( $us3_link, $query );
[13]	616
	617	if ( ! $result )
	618	{
[35]	619	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
	620	mail_to_user( "fail", "Bad query\n$query\n" . mysqli_error( $us3_link ) );
[13]	621	return( -1 );
	622	}
	623	write_log( "$me: mrecs entry updated : mrecsID=$mrecsID" );
	624	}
	625	//write_log( "$me: mrecs entries updated" );
	626
[1]	627	// Copy results to LIMS submit directory (files there are deleted after 7 days)
	628	global $submit_dir; // LIMS submit files dir
	629
	630	// Get the request guid (LIMS submit dir name)
	631	$query = "SELECT HPCAnalysisRequestGUID FROM HPCAnalysisRequest " .
	632	"WHERE HPCAnalysisRequestID = $requestID ";
[35]	633	$result = mysqli_query( $us3_link, $query );
[1]	634
	635	if ( ! $result )
	636	{
[35]	637	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $us3_link ) );
[1]	638	}
	639
[35]	640	// list( $requestGUID ) = mysqli_fetch_array( $result );
[10]	641	//
	642	// chdir( "$submit_dir/$requestGUID" );
	643	// $f = fopen( "analysis-results.tar", "w" );
	644	// fwrite( $f, $tarfile );
	645	// fclose( $f );
[1]	646
	647	// Clean up
[10]	648	// chdir ( $work );
[1]	649	// exec( "rm -rf $gfacID" );
	650
[35]	651	mysqli_close( $us3_link );
[1]	652
	653	/////////
	654	// Send email
	655
	656	mail_to_user( "success", "" );
	657	}
	658
	659	function mail_to_user( $type, $msg )
	660	{
	661	// Note to me. Just changed subject line to include a modified $status instead
	662	// of the $type variable passed. More informative than just "fail" or "success."
	663	// See how it works for awhile and then consider removing $type parameter from
	664	// function.
	665	global $email_address;
	666	global $submittime;
[40]	667	global $endtime;
[1]	668	global $queuestatus;
	669	global $status;
	670	global $cluster;
	671	global $jobtype;
	672	global $org_name;
	673	global $admin_email;
[6]	674	global $db;
[1]	675	global $dbhost;
	676	global $requestID;
	677	global $gfacID;
	678	global $editXMLFilename;
[5]	679	global $stdout;
[25]	680	global $org_domain;
[1]	681
	682	global $me;
	683	write_log( "$me mail_to_user(): sending email to $email_address for $gfacID" );
	684
	685	// Get GFAC status and message
[4]	686	// function get_gfac_message() also sets global $status
	687	$gfac_message = get_gfac_message( $gfacID );
[10]	688	if ( $gfac_message === false ) $gfac_message = "Job Finished";
[1]	689
	690	// Create a status to put in the subject line
	691	switch ( $status )
	692	{
	693	case "COMPLETE":
	694	$subj_status = 'completed';
	695	break;
	696
	697	case "CANCELLED":
	698	case "CANCELED":
	699	$subj_status = 'canceled';
	700	break;
	701
	702	case "FAILED":
	703	$subj_status = 'failed';
[10]	704	if ( preg_match( "/^US3-A/i", $gfacID ) )
	705	{ // For A/Thrift FAIL, get error message
	706	$gfac_message = getExperimentErrors( $gfacID );
	707	//$gfac_message .= "Test ERROR MESSAGE";
	708	}
[1]	709	break;
	710
	711	case "ERROR":
	712	$subj_status = 'unknown error';
	713	break;
	714
	715	default:
	716	$subj_status = $status; // For now
	717	break;
	718
	719	}
	720
[10]	721	$queuestatus = $subj_status;
	722	$limshost = $dbhost;
	723	if ( $limshost == 'localhost' )
[25]	724	{
[10]	725	$limshost = gethostname();
[35]	726	if ( preg_match( "/scyld/", $limshost ) )
	727	$limshost = 'alamo.uthscsa.edu';
[37]	728	else if ( preg_match( "/novalo/", $limshost ) )
	729	$limshost = 'uslims3.aucsolutions.com';
[35]	730	else if ( ! preg_match( "/\./", $limshost ) )
[25]	731	$limshost = $limshost . $org_domain;
	732	}
[10]	733
[1]	734	// Parse the editXMLFilename
	735	list( $runID, $editID, $dataType, $cell, $channel, $wl, $ext ) =
	736	explode( ".", $editXMLFilename );
	737
	738	$headers = "From: $org_name Admin<$admin_email>" . "\n";
	739	$headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
[41]	740	$headers .= "CC: $org_name Admin<alexsav.science@gmail.com>" . "\n";
	741	$headers .= "CC: $org_name Admin<gegorbet@gmail.com>" . "\n";
[1]	742
	743	// Set the reply address
	744	$headers .= "Reply-To: $org_name<$admin_email>" . "\n";
	745	$headers .= "Return-Path: $org_name<$admin_email>" . "\n";
	746
	747	// Try to avoid spam filters
[40]	748	$now = time();
	749	$tnow = date( 'Y-m-d H:i:s' );
[1]	750	$headers .= "Message-ID: <" . $now . "cleanup@$dbhost>\n";
	751	$headers .= "X-Mailer: PHP v" . phpversion() . "\n";
	752	$headers .= "MIME-Version: 1.0" . "\n";
	753	$headers .= "Content-Transfer-Encoding: 8bit" . "\n";
	754
	755	$subject = "UltraScan Job Notification - $subj_status - " . substr( $gfacID, 0, 16 );
	756	$message = "
	757	Your UltraScan job is complete:
	758
	759	Submission Time : $submittime
[40]	760	Job End Time : $endtime
	761	Mail Time : $tnow
[10]	762	LIMS Host : $limshost
[1]	763	Analysis ID : $gfacID
[6]	764	Request ID : $requestID ( $db )
[1]	765	RunID : $runID
	766	EditID : $editID
	767	Data Type : $dataType
	768	Cell/Channel/Wl : $cell / $channel / $wl
	769	Status : $queuestatus
	770	Cluster : $cluster
	771	Job Type : $jobtype
	772	GFAC Status : $status
[4]	773	GFAC Message : $gfac_message
[5]	774	Stdout : $stdout
[1]	775	";
	776
[4]	777	if ( $type != "success" ) $message .= "Grid Ctrl Error : $msg\n";
[1]	778
	779	// Handle the error case where an error occurs before fetching the
	780	// user's email address
	781	if ( $email_address == "" ) $email_address = $admin_email;
	782
	783	mail( $email_address, $subject, $message, $headers );
	784	}
	785
	786	function parse_xml( $xml, $type )
	787	{
	788	$parser = new XMLReader();
	789	$parser->xml( $xml );
	790
	791	$results = array();
	792
	793	while ( $parser->read() )
	794	{
	795	if ( $parser->name == $type )
	796	{
	797	while ( $parser->moveToNextAttribute() )
	798	{
	799	$results[ $parser->name ] = $parser->value;
	800	}
	801
	802	break;
	803	}
	804	}
	805
	806	$parser->close();
	807	return $results;
	808	}
	809
	810	// Function to get information about the current job GFAC
	811	function get_gfac_message( $gfacID )
	812	{
	813	global $serviceURL;
	814	global $me;
	815
	816	$hex = "[0-9a-fA-F]";
	817	if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
	818	! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
	819	{
	820	// Then it's not a GFAC job
	821	return false;
	822	}
	823
	824	$url = "$serviceURL/jobstatus/$gfacID";
	825	try
	826	{
	827	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	828	$http = $post->send();
	829	$xml = $post->getResponseBody();
	830	}
	831	catch ( HttpException $e )
	832	{
	833	write_log( "$me: Job status not available - $gfacID" );
	834	return false;
	835	}
	836
	837	// Parse the result
	838	$gfac_message = parse_message( $xml );
	839
	840	return $gfac_message;
	841	}
	842
	843	function parse_message( $xml )
	844	{
[4]	845	global $status;
[1]	846	$status = "";
	847	$gfac_message = "";
	848
	849	$parser = new XMLReader();
	850	$parser->xml( $xml );
	851
	852	$results = array();
	853
	854	while( $parser->read() )
	855	{
	856	$type = $parser->nodeType;
	857
	858	if ( $type == XMLReader::ELEMENT )
	859	$name = $parser->name;
	860
	861	else if ( $type == XMLReader::TEXT )
	862	{
	863	if ( $name == "status" )
	864	$status = $parser->value;
	865	else
	866	$gfac_message = $parser->value;
	867	}
	868	}
	869
	870	$parser->close();
	871	return $gfac_message;
	872	}
	873
	874	function get_local_files( $gfac_link, $cluster, $requestID, $id, $gfacID )
	875	{
	876	global $work;
	877	global $work_remote;
	878	global $me;
	879	global $db;
[35]	880	global $dbhost;
[1]	881	global $status;
[41]	882
[25]	883	$is_us3iab = preg_match( "/us3iab/", $cluster );
[31]	884	$is_jetstr = preg_match( "/jetstream/", $cluster );
[41]	885	$is_demeler3 = preg_match( "/demeler3/", $cluster );
	886
[35]	887	$limshost = $dbhost;
	888	$stderr = '';
	889	$stdout = '';
	890	$tarfile = '';
[1]	891
[35]	892	if ( $limshost == 'localhost' )
	893	{ // If DB host is local host, get full LIMS host name
	894	$limshost = gethostname();
	895	if ( preg_match( "/scyld/", $limshost ) )
	896	$limshost = 'alamo.uthscsa.edu';
[37]	897	else if ( preg_match( "/novalo/", $limshost ) )
	898	$limshost = 'uslims3.aucsolutions.com';
[35]	899	else if ( ! preg_match( "/\./", $limshost ) )
	900	$limshost = $limshost . $org_domain;
	901	}
	902
	903	if ( preg_match( "/alamo/", $limshost ) &&
	904	preg_match( "/alamo/", $cluster ) )
	905	{ // If both LIMS and cluster are alamo, set up local transfers
	906	$is_us3iab = 1;
	907	if ( ! preg_match( "/\/local/", $work_remote ) )
	908	$work_remote = $work_remote . "/local";
	909	}
	910
	911	// Figure out job's remote (or local) work directory
[1]	912	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
[28]	913	//write_log( "$me: is_us3iab=$is_us3iab remoteDir=$remoteDir" );
[1]	914
	915	// Get stdout, stderr, output/analysis-results.tar
	916	$output = array();
	917
[25]	918	if ( $is_us3iab == 0 )
	919	{
[28]	920	// For "-local", recompute remote work directory
[31]	921	$clushost = "$cluster.uthscsa.edu";
	922	$lworkdir = "~us3/lims/work/local";
	923	if ( $is_jetstr )
	924	{
[32]	925	$clushost = "js-169-137.jetstream-cloud.org";
[31]	926	$lworkdir = "/N/us3_cluster/work/local";
	927	}
[41]	928
	929	if ( $is_demeler3 )
	930	{
	931	$clushost = "demeler3.uleth.ca";
	932	$lworkdir = "/home/us3/work";
	933	}
	934
[35]	935	$cmd = "ssh us3@$clushost 'ls -d $lworkdir' 2>/dev/null";
[28]	936	exec( $cmd, $output, $stat );
	937	$work_remote = $output[ 0 ];
	938	$remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
	939	write_log( "$me: -LOCAL: remoteDir=$remoteDir" );
	940
[25]	941	// Figure out local working directory
	942	if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
	943	$pwd = chdir( "$work/$gfacID" );
[1]	944
[31]	945	$cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1";
[25]	946
[1]	947	exec( $cmd, $output, $stat );
[25]	948	if ( $stat != 0 )
[1]	949	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	950
[31]	951	$cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1";
[25]	952
[1]	953	exec( $cmd, $output, $stat );
[25]	954	if ( $stat != 0 )
	955	{
[1]	956	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
[25]	957	sleep( 10 );
	958	write_log( "$me: RETRY" );
	959	exec( $cmd, $output, $stat );
	960	if ( $stat != 0 )
	961	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	962	}
	963
[31]	964	$cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1";
[25]	965
	966	exec( $cmd, $output, $stat );
	967	if ( $stat != 0 )
	968	{
	969	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	970	sleep( 10 );
	971	write_log( "$me: RETRY" );
	972	exec( $cmd, $output, $stat );
	973	if ( $stat != 0 )
	974	write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
	975	}
[1]	976	}
[25]	977	else
[35]	978	{ // Is US3IAB or alamo-to-alamo, so just change to local work directory
[25]	979	$pwd = chdir( "$remoteDir" );
	980	write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
	981	}
[1]	982
[25]	983
[1]	984	// Write the files to gfacDB
	985
[35]	986	$secwait = 10;
	987	$num_try = 0;
	988	while ( ! file_exists( "stderr" ) && $num_try < 3 )
	989	{ // Do waits and retries to let stderr appear
	990	sleep( $secwait );
	991	$num_try++;
	992	$secwait *= 2;
	993	write_log( "$me: not-exist-stderr: num_try=$num_try" );
	994	}
	995
	996	$lense = 0;
[25]	997	if ( file_exists( "stderr" ) )
[35]	998	{
	999	$lense = filesize( "stderr" );
	1000	if ( $lense > 1000000 )
	1001	{ // Replace exceptionally large stderr with smaller version
	1002	exec( "mv stderr stderr-orig", $output, $stat );
	1003	exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
	1004	exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
	1005	exec( "cat stderr-h stderr-t >stderr", $output, $stat );
	1006	}
[25]	1007	$stderr = file_get_contents( "stderr" );
	1008	}
	1009	else
[35]	1010	{
[25]	1011	$stderr = "";
[35]	1012	}
	1013
[1]	1014	if ( file_exists( "stdout" ) ) $stdout = file_get_contents( "stdout" );
[35]	1015
[25]	1016	$fn1_tarfile = "analysis-results.tar";
	1017	$fn2_tarfile = "output/" . $fn1_tarfile;
	1018	if ( file_exists( $fn1_tarfile ) )
	1019	$tarfile = file_get_contents( $fn1_tarfile );
	1020	else if ( file_exists( $fn2_tarfile ) )
	1021	$tarfile = file_get_contents( $fn2_tarfile );
[1]	1022
[35]	1023	// $lense = strlen( $stderr );
	1024	// if ( $lense > 1000000 )
	1025	// { // Replace exceptionally large stderr with smaller version
	1026	// exec( "mv stderr stderr-orig", $output, $stat );
	1027	// exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
	1028	// exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
	1029	// exec( "cat stderr-h stderr-t >stderr", $output, $stat );
	1030	// $stderr = file_get_contents( "stderr" );
	1031	// }
[25]	1032	$lene = strlen( $stderr );
	1033	write_log( "$me: stderr size: $lene (was $lense)");
	1034	$leno = strlen( $stdout );
	1035	write_log( "$me: stdout size: $leno");
[35]	1036	$lent = strlen( $tarfile );
	1037	write_log( "$me: tarfile size: $lent");
	1038	$esstde = mysqli_real_escape_string( $gfac_link, $stderr );
	1039	$esstdo = mysqli_real_escape_string( $gfac_link, $stdout );
	1040	$estarf = mysqli_real_escape_string( $gfac_link, $tarfile );
	1041	$lene = strlen($esstde);
	1042	write_log( "$me: es-stderr size: $lene");
	1043	$leno = strlen($esstdo);
	1044	write_log( "$me: es-stdout size: $leno");
[25]	1045	$lenf = strlen($estarf);
[35]	1046	write_log( "$me: es-tarfile size: $lenf");
[1]	1047	$query = "UPDATE analysis SET " .
[35]	1048	"stderr='" . $esstde . "'," .
	1049	"stdout='" . $esstdo . "'," .
	1050	"tarfile='" . $estarf . "'";
[1]	1051
[35]	1052	$result = mysqli_query( $gfac_link, $query );
[1]	1053
	1054	if ( ! $result )
	1055	{
[35]	1056	write_log( "$me: Bad query:\n$query\n" . mysqli_error( $gfac_link ) );
[1]	1057	echo "Bad query\n";
	1058	return( -1 );
	1059	}
	1060	}
	1061	?>

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: trunk/cleanup.php@ 41

Download in other formats: