Context Navigation

source: trunk/gridctl.php@ 17

Last change on this file since 17 was 17, checked in by gegorbet, 9 years ago
misc. fixes plus mods for new lonestar5 cluster
File size: 29.9 KB

Rev	Line
[1]	1	<?php
	2
	3	// Global variables
	4	$gfac_message = "";
	5	$updateTime = 0;
	6	$submittime = 0;
	7	$cluster = '';
	8
[6]	9	//global $self;
	10
[1]	11	// Produce some output temporarily, so cron will send me message
	12	$now = time();
[6]	13	echo "Time started: " . date( 'Y-m-d H:i:s', $now ) . "\n";
[1]	14
	15	// Get data from global GFAC DB
	16	$gLink = mysql_connect( $dbhost, $guser, $gpasswd );
	17
	18	if ( ! mysql_select_db( $gDB, $gLink ) )
	19	{
[3]	20	write_log( "$self: Could not select DB $gDB - " . mysql_error() );
[1]	21	mail_to_admin( "fail", "Internal Error: Could not select DB $gDB" );
	22	exit();
	23	}
	24
	25	$query = "SELECT gfacID, us3_db, cluster, status, queue_msg, " .
	26	"UNIX_TIMESTAMP(time), time from analysis";
	27	$result = mysql_query( $query, $gLink );
	28
	29	if ( ! $result )
	30	{
	31	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	32	mail_to_admin( "fail", "Query failed $query\n" . mysql_error( $gLink ) );
	33	exit();
	34	}
	35
	36	if ( mysql_num_rows( $result ) == 0 )
[6]	37	{
	38	//write_log( "$self: analysis read got numrows==0" );
[1]	39	exit(); // Nothing to do
[6]	40	}
[1]	41
[14]	42	$me_devel = preg_match( "/class_devel/", $class_dir );
	43
[1]	44	while ( list( $gfacID, $us3_db, $cluster, $status, $queue_msg, $time, $updateTime )
	45	= mysql_fetch_array( $result ) )
	46	{
[14]	47	// If this entry does not match class/class_devel, skip processing
	48
	49	if ( preg_match( "/US3-A/i", $gfacID ) )
	50	{ // For thrift, job and gridctl must match
	51	$job_devel = preg_match( "/US3-ADEV/i", $gfacID );
	52	if ( ( $me_devel && !$job_devel ) \|\|
	53	( !$me_devel && $job_devel ) )
	54	{ // If job not from appropriate Airavata server, skip processing
	55	continue;
	56	}
	57	}
	58	else if ( $me_devel )
	59	{ // For non-thrift and development, skip
	60	continue;
	61	}
	62
[1]	63	// Checking we need to do for each entry
[6]	64	echo "us3db=$us3_db gfid=$gfacID\n";
	65	switch ( $us3_db )
	66	{
	67	case 'Xuslims3_cauma3' :
	68	case 'Xuslims3_cauma3d' :
	69	case 'Xuslims3_HHU' :
	70	case 'Xuslims3_Uni_KN' :
	71	$serviceURL = "http://gridfarm005.ucs.indiana.edu:9090/ogce-rest/job";
	72	break;
[1]	73
[6]	74	default :
	75	// $serviceURL = "http://gridfarm005.ucs.indiana.edu:8080/ogce-rest/job";
	76	break;
	77	}
	78
	79	$awork = array();
	80	$awork = explode( "-", $gfacID );
	81	$gfacLabl = $awork[0] . "-" . $awork[1] . "-" . $awork[2];
	82	$loghdr = $self . ":" . $gfacLabl . "...:";
[14]	83	$status_in = $status;
	84	$status_gw = $status;
[17]	85	$status_ex = $status;
[6]	86
	87	// If entry is for Airvata/Thrift, get the true current status
	88
	89	if ( is_aira_job( $gfacID ) )
	90	{
	91	$status_in = $status;
	92	$status = aira_status( $gfacID, $status_in );
	93	if($status != $status_in )
	94	write_log( "$loghdr Set to $status from $status_in" );
	95	}
	96	else
	97	{
	98	$status_gw = $status;
	99	$status = get_gfac_status( $gfacID );
	100	//if ( $status == 'FINISHED' )
	101	if ( $status_gw == 'COMPLETE' )
	102	$status = $status_gw;
	103	write_log( "$loghdr non-AThrift status=$status status_gw=$status_gw" );
	104	}
	105
[1]	106	// Sometimes during testing, the us3_db entry is not set
	107	// If $status == 'ERROR' then the condition has been processed before
	108	if ( strlen( $us3_db ) == 0 && $status != 'ERROR' )
	109	{
[6]	110	write_log( "$loghdr GFAC DB is NULL - $gfacID" );
[1]	111	mail_to_admin( "fail", "GFAC DB is NULL\n$gfacID" );
	112
	113	$query2 = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
	114	$result2 = mysql_query( $query2, $gLink );
	115	$status = 'ERROR';
	116
	117	if ( ! $result2 )
[6]	118	write_log( "$loghdr Query failed $query2 - " . mysql_error( $gLink ) );
[1]	119
	120	}
	121
[6]	122	//echo " st=$status\n";
[1]	123	switch ( $status )
	124	{
	125	// Already been handled
	126	// Later update this condition to search for gfacID?
	127	case "ERROR":
	128	cleanup();
	129	break;
	130
	131	case "SUBMITTED":
	132	submitted( $time );
	133	break;
	134
	135	case "SUBMIT_TIMEOUT":
	136	submit_timeout( $time );
	137	break;
	138
	139	case "RUNNING":
[6]	140	case "STARTED":
	141	case "STAGING":
	142	case "ACTIVE":
[1]	143	running( $time );
	144	break;
	145
	146	case "RUN_TIMEOUT":
	147	run_timeout($time );
	148	break;
	149
	150	case "DATA":
[6]	151	case "RESULTS_GEN":
[1]	152	wait_data( $time );
	153	break;
	154
	155	case "DATA_TIMEOUT":
	156	data_timeout( $time );
	157	break;
	158
[6]	159	case "COMPLETED":
[1]	160	case "COMPLETE":
	161	complete();
	162	break;
	163
	164	case "CANCELLED":
	165	case "CANCELED":
	166	case "FAILED":
	167	failed();
	168	break;
	169
[6]	170	case "FINISHED":
	171	case "DONE":
[17]	172	if ( is_aira_job( $gfacID ) )
	173	{
	174	$status_ex = getExperimentStatus( $gfacID );
	175	write_log( "$loghdr status=$status status_ex=$status_ex" );
	176	if ( $status_ex === 'COMPLETED' )
	177	complete();
	178	}
[6]	179	case "PROCESSING":
[1]	180	default:
	181	break;
	182	}
	183	}
	184
	185	exit();
	186
	187	function submitted( $updatetime )
	188	{
	189	global $self;
	190	global $gLink;
	191	global $gfacID;
[6]	192	global $loghdr;
[1]	193
	194	$now = time();
	195
	196	if ( $updatetime + 600 > $now ) return; // < 10 minutes ago
	197
	198	if ( $updatetime + 86400 > $now ) // Within the first 24 hours
	199	{
	200	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
	201	$job_status = get_local_status( $gfacID );
	202
	203	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
	204	return;
	205
	206	if ( ! in_array( $job_status, array( 'SUBMITTED', 'INITIALIZED', 'PENDING' ) ) )
[6]	207	{
[14]	208	//write_log( "$loghdr submitted:job_status=$job_status" );
[1]	209	update_job_status( $job_status, $gfacID );
[6]	210	}
[1]	211
	212	return;
	213	}
	214
	215	$message = "Job listed submitted longer than 24 hours";
	216	write_log( "$self: $message - id: $gfacID" );
	217	mail_to_admin( "hang", "$message - id: $gfacID" );
	218	$query = "UPDATE analysis SET status='SUBMIT_TIMEOUT' WHERE gfacID='$gfacID'";
	219	$result = mysql_query( $query, $gLink );
	220
	221	if ( ! $result )
	222	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	223
	224	update_queue_messages( $message );
	225	update_db( $message );
	226	}
	227
	228	function submit_timeout( $updatetime )
	229	{
	230	global $self;
	231	global $gLink;
	232	global $gfacID;
[6]	233	global $loghdr;
[1]	234
	235	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
	236	$job_status = get_local_status( $gfacID );
	237
	238	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
	239	return;
	240
	241	if ( ! in_array( $job_status, array( 'SUBMITTED', 'INITIALIZED', 'PENDING' ) ) )
	242	{
[14]	243	//write_log( "$loghdr submit_timeout:job_status=$job_status" );
[1]	244	update_job_status( $job_status, $gfacID );
	245	return;
	246	}
	247
	248	$now = time();
	249
	250	if ( $updatetime + 86400 > $now ) return; // < 24 hours ago ( 48 total submitted )
	251
	252	$message = "Job listed submitted longer than 48 hours";
	253	write_log( "$self: $message - id: $gfacID" );
	254	mail_to_admin( "hang", "$message - id: $gfacID" );
	255	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
	256	$result = mysql_query( $query, $gLink );
	257
	258	if ( ! $result )
	259	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	260
	261	update_queue_messages( $message );
	262	update_db( $message );
	263	}
	264
	265	function running( $updatetime )
	266	{
	267	global $self;
	268	global $gLink;
	269	global $gfacID;
[6]	270	global $loghdr;
[1]	271
	272	$now = time();
	273
	274	get_us3_data();
	275
	276	if ( $updatetime + 600 > $now ) return; // message received < 10 minutes ago
	277
	278	if ( $updatetime + 86400 > $now ) // Within the first 24 hours
	279	{
	280	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
	281	$job_status = get_local_status( $gfacID );
	282
	283	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
	284	return;
	285
[6]	286	if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
[14]	287	{
	288	//write_log( "$loghdr running:job_status=$job_status" );
[1]	289	update_job_status( $job_status, $gfacID );
[14]	290	}
[1]	291
	292	return;
	293	}
	294
	295	$message = "Job listed running longer than 24 hours";
	296	write_log( "$self: $message - id: $gfacID" );
	297	mail_to_admin( "hang", "$message - id: $gfacID" );
	298	$query = "UPDATE analysis SET status='RUN_TIMEOUT' WHERE gfacID='$gfacID'";
	299	$result = mysql_query( $query, $gLink );
	300
	301	if ( ! $result )
	302	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	303
	304	update_queue_messages( $message );
	305	update_db( $message );
	306	}
	307
	308	function run_timeout( $updatetime )
	309	{
	310	global $self;
	311	global $gLink;
	312	global $gfacID;
[6]	313	global $loghdr;
[1]	314
	315	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
	316	$job_status = get_local_status( $gfacID );
	317
	318	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
	319	return;
	320
[6]	321	if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
[1]	322	{
[14]	323	//write_log( "$loghdr run_timeout:job_status=$job_status" );
[1]	324	update_job_status( $job_status, $gfacID );
	325	return;
	326	}
	327
	328	$now = time();
	329
	330	get_us3_data();
	331
	332	if ( $updatetime + 172800 > $now ) return; // < 48 hours ago
	333
	334	$message = "Job listed running longer than 48 hours";
	335	write_log( "$self: $message - id: $gfacID" );
	336	mail_to_admin( "hang", "$message - id: $gfacID" );
	337	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
	338	$result = mysql_query( $query, $gLink );
	339
	340	if ( ! $result )
	341	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	342
	343	update_queue_messages( $message );
	344	update_db( $message );
	345	}
	346
	347	function wait_data( $updatetime )
	348	{
	349	global $self;
	350	global $gLink;
	351	global $gfacID;
[6]	352	global $loghdr;
[1]	353
	354	$now = time();
	355
	356	if ( $updatetime + 3600 > $now ) // < Within the first hour
	357	{
	358	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
	359	$job_status = get_local_status( $gfacID );
	360
	361	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
	362	return;
	363
	364	if ( $job_status != 'DATA' )
	365	{
[14]	366	//write_log( "$loghdr wait_data:job_status=$job_status" );
[1]	367	update_job_status( $job_status, $gfacID );
	368	return;
	369	}
	370
	371	// Request to resend data, but only request every 5 minutes
	372	$minute = date( 'i' ) * 1; // Makes it an int
	373	if ( $minute % 5 ) return;
	374
	375	$output_status = get_gfac_outputs( $gfacID );
	376
	377	if ( $output_status !== false )
	378	mail_to_admin( "debug", "wait_data/$gfacID/$output_status" );
	379
	380	return;
	381	}
	382
	383	$message = "Waiting for data longer than 1 hour";
	384	write_log( "$self: $message - id: $gfacID" );
	385	mail_to_admin( "hang", "$message - id: $gfacID" );
	386	$query = "UPDATE analysis SET status='DATA_TIMEOUT' WHERE gfacID='$gfacID'";
	387	$result = mysql_query( $query, $gLink );
	388
	389	if ( ! $result )
	390	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	391
	392	update_queue_messages( $message );
	393	update_db( $message );
	394	}
	395
	396	function data_timeout( $updatetime )
	397	{
	398	global $self;
	399	global $gLink;
	400	global $gfacID;
[6]	401	global $loghdr;
[1]	402
	403	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
	404	$job_status = get_local_status( $gfacID );
	405
	406	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
	407	return;
	408
	409	if ( $job_status != 'DATA' )
	410	{
[14]	411	//write_log( "$loghdr data_timeout:job_status=$job_status" );
[1]	412	update_job_status( $job_status, $gfacID );
	413	return;
	414	}
	415
	416	$now = time();
	417
	418	if ( $updatetime + 86400 > $now ) // < 24 hours ago
	419	{
	420	// Request to resend data, but only request every 15 minutes
	421	$minute = date( 'i' ) * 1; // Makes it an int
	422	if ( $minute % 15 ) return;
	423
	424	$output_status = get_gfac_outputs( $gfacID );
	425
	426	if ( $output_status !== false )
	427	mail_to_admin( "debug", "data_timeout/$gfacID/$output_status" );
	428
	429	return;
	430	}
	431
	432	$message = "Waiting for data longer than 24 hours";
	433	write_log( "$self: $message - id: $gfacID" );
	434	mail_to_admin( "hang", "$message - id: $gfacID" );
	435	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
	436	$result = mysql_query( $query, $gLink );
	437
	438	if ( ! $result )
	439	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	440
	441	update_queue_messages( $message );
	442	update_db( $message );
	443	}
	444
	445	function complete()
	446	{
	447	// Just cleanup
	448	cleanup();
	449	}
	450
	451	function failed()
	452	{
	453	// Just cleanup
	454	cleanup();
	455	}
	456
	457	function cleanup()
	458	{
	459	global $self;
	460	global $gLink;
	461	global $gfacID;
	462	global $us3_db;
[6]	463	global $loghdr;
[10]	464	global $class_dir;
[1]	465
	466	// Double check that the gfacID exists
	467	$query = "SELECT count(*) FROM analysis WHERE gfacID='$gfacID'";
	468	$result = mysql_query( $query, $gLink );
	469
	470	if ( ! $result )
	471	{
	472	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	473	mail_to_admin( "fail", "Query failed $query\n" . mysql_error( $gLink ) );
	474	return;
	475	}
	476
	477	list( $count ) = mysql_fetch_array( $result );
	478
[6]	479	if ($count==0)
	480	write_log( "$loghdr count = $count gfacID = $gfacID" );
[1]	481	if ( $count == 0 ) return;
	482
	483	// Now check the us3 instance
	484	$requestID = get_us3_data();
[6]	485	//write_log( "$loghdr requestID = $requestID gfacID = $gfacID" );
[1]	486	if ( $requestID == 0 ) return;
	487
[14]	488	$me_devel = preg_match( "/class_devel/", $class_dir );
	489
[10]	490	if ( preg_match( "/US3-A/i", $gfacID ) )
[6]	491	{
[10]	492	$job_devel = preg_match( "/US3-ADEV/i", $gfacID );
	493	if ( ( !$me_devel && !$job_devel ) \|\|
	494	( $me_devel && $job_devel ) )
	495	{ // If job from appropriate Airavata server, process it
[6]	496	//write_log( "$loghdr CALLING aira_cleanup()" );
[10]	497	aira_cleanup( $us3_db, $requestID, $gLink );
	498	}
[6]	499	//write_log( "$loghdr RTN FR aira_cleanup()" );
	500	}
[14]	501
	502	else if ( ! $me_devel )
	503	{ // If this is gridctl_pro and gfac (jureca), do GFAC cleanup
[6]	504	//write_log( "$loghdr CALLING gfac_cleanup()" );
	505	gfac_cleanup( $us3_db, $requestID, $gLink );
	506	}
[1]	507	}
	508
	509	// Function to update status of job
	510	function update_job_status( $job_status, $gfacID )
	511	{
	512	global $gLink;
[6]	513	global $query;
	514	global $self;
	515	global $loghdr;
[1]	516
	517	switch ( $job_status )
	518	{
	519	case 'SUBMITTED' :
	520	case 'SUBMITED' :
	521	case 'INITIALIZED' :
	522	$query = "UPDATE analysis SET status='SUBMITTED' WHERE gfacID='$gfacID'";
[14]	523	$message = "Job status request reports job is " . $job_status;
	524	//write_log( "$loghdr update_job_status(SUBM) job_status=$job_status" );
[1]	525	break;
	526
[6]	527	case 'STARTED' :
	528	case 'RUNNING' :
[1]	529	case 'ACTIVE' :
	530	$query = "UPDATE analysis SET status='RUNNING' WHERE gfacID='$gfacID'";
	531	$message = "Job status request reports job is RUNNING";
	532	break;
	533
[6]	534	case 'FINISHED' :
	535	$query = "UPDATE analysis SET status='FINISHED' WHERE gfacID='$gfacID'";
	536	$message = "NONE";
	537	break;
	538
	539	case 'DONE' :
	540	$query = "UPDATE analysis SET status='DONE' WHERE gfacID='$gfacID'";
	541	$message = "NONE";
	542	break;
	543
[1]	544	case 'COMPLETED' :
[6]	545	case 'COMPLETE' :
[1]	546	$query = "UPDATE analysis SET status='COMPLETE' WHERE gfacID='$gfacID'";
[6]	547	$message = "Job status request reports job is COMPLETED";
[1]	548	break;
	549
[6]	550	case 'DATA' :
[1]	551	$query = "UPDATE analysis SET status='DATA' WHERE gfacID='$gfacID'";
	552	$message = "Job status request reports job is COMPLETE, waiting for data";
	553	break;
	554
	555	case 'CANCELED' :
[6]	556	case 'CANCELLED' :
[1]	557	$query = "UPDATE analysis SET status='CANCELED' WHERE gfacID='$gfacID'";
	558	$message = "Job status request reports job is CANCELED";
	559	break;
	560
	561	case 'FAILED' :
	562	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
	563	$message = "Job status request reports job is FAILED";
	564	break;
	565
	566	case 'UNKNOWN' :
[6]	567	write_log( "$loghdr job_status='UNKNOWN', reset to 'ERROR' " );
	568	$query = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
[1]	569	$message = "Job status request reports job is not in the queue";
	570	break;
	571
[14]	572	case 'UPDATING' :
	573	case 'PENDING' :
	574	$message = "Job status request reports job is " . $job_status;
	575	break;
	576
[1]	577	default :
[3]	578	// We shouldn't ever get here
[6]	579	$query = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
[1]	580	$message = "Job status was not recognized - $job_status";
[6]	581	write_log( "$loghdr update_job_status: " .
[3]	582	"Job status was not recognized - $job_status\n" .
	583	"gfacID = $gfacID\n" );
[1]	584	break;
	585
	586	}
	587
	588	$result = mysql_query( $query, $gLink );
	589	if ( ! $result )
[6]	590	write_log( "$loghdr Query failed $query - " . mysql_error( $gLink ) );
[1]	591
[6]	592	if ( $message != 'NONE' )
	593	{
	594	update_queue_messages( $message );
	595	update_db( $message );
	596	}
[1]	597	}
	598
	599	function get_us3_data()
	600	{
	601	global $self;
	602	global $gfacID;
	603	global $dbhost;
	604	global $user;
	605	global $passwd;
	606	global $us3_db;
	607	global $updateTime;
[6]	608	global $loghdr;
[1]	609
	610	$us3_link = mysql_connect( $dbhost, $user, $passwd );
	611
	612	if ( ! $us3_link )
	613	{
[6]	614	write_log( "$loghdr could not connect: $dbhost, $user, $passwd" );
[1]	615	mail_to_admin( "fail", "Could not connect to $dbhost" );
	616	return 0;
	617	}
	618
	619
	620	$result = mysql_select_db( $us3_db, $us3_link );
	621
	622	if ( ! $result )
	623	{
[6]	624	write_log( "$loghdr could not select DB $us3_db" );
[1]	625	mail_to_admin( "fail", "Could not select DB $us3_db, $dbhost, $user, $passwd" );
	626	return 0;
	627	}
	628
	629	$query = "SELECT HPCAnalysisRequestID, UNIX_TIMESTAMP(updateTime) " .
	630	"FROM HPCAnalysisResult WHERE gfacID='$gfacID'";
	631	$result = mysql_query( $query, $us3_link );
	632
	633	if ( ! $result )
	634	{
	635	write_log( "$self: Query failed $query - " . mysql_error( $us3_link ) );
	636	mail_to_admin( "fail", "Query failed $query\n" . mysql_error( $us3_link ) );
	637	return 0;
	638	}
	639
	640	list( $requestID, $updateTime ) = mysql_fetch_array( $result );
	641	mysql_close( $us3_link );
	642
	643	return $requestID;
	644	}
	645
[6]	646	// Function to determine if this is a gfac job or not
[1]	647	function is_gfac_job( $gfacID )
	648	{
	649	$hex = "[0-9a-fA-F]";
	650	if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
	651	! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
	652	{
	653	// Then it's not a GFAC job
	654	return false;
	655	}
	656
	657	return true;
	658	}
	659
[6]	660	// Function to determine if this is an airavata/thrift job or not
	661	function is_aira_job( $gfacID )
	662	{
	663	global $cluster;
	664
[15]	665	if ( preg_match( "/US3-A/i", $gfacID ) )
[6]	666	{
	667	// Then it's an Airavata/Thrift job
	668	return true;
	669	}
	670
	671	return false;
	672	}
	673
[1]	674	// Function to get the current job status from GFAC
	675	function get_gfac_status( $gfacID )
	676	{
	677	global $serviceURL;
[6]	678	global $self;
	679	global $loghdr;
	680	global $cluster;
[1]	681
[6]	682	if ( is_aira_job( $gfacID ) )
	683	{
	684	$status_ex = getExperimentStatus( $gfacID );
[17]	685
	686	if ( $status_ex == 'EXECUTING' )
	687	{
	688	if ( $gfac_status == 'SUBMITTED' )
	689	$status_ex = 'QUEUED';
	690	if ( $gfac_status == 'RUNNING' )
	691	$status_ex = 'ACTIVE';
	692	}
	693
[6]	694	$gfac_status = standard_status( $status_ex );
[14]	695	write_log( "$loghdr get_gfac_status: status_ex=$status_ex gfac_status=$gfac_status" );
[6]	696	return $gfac_status;
	697	}
	698
	699	else if ( ! is_gfac_job( $gfacID ) )
[1]	700	return false;
	701
	702	$url = "$serviceURL/jobstatus/$gfacID";
	703	try
	704	{
	705	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	706	$http = $post->send();
	707	$xml = $post->getResponseBody();
	708	}
	709	catch ( HttpException $e )
	710	{
[6]	711	write_log( "$loghdr Status not available - marking failed - $gfacID" );
[1]	712	return 'GFAC_STATUS_UNAVAILABLE';
	713	}
	714
	715	// Parse the result
	716	$gfac_status = parse_response( $xml );
	717
[3]	718	// This may not seem like the best place to do this, but here we have
	719	// the xml straight from GFAC
	720	$status_types = array('SUBMITTED',
	721	'SUBMITED',
	722	'INITIALIZED',
	723	'PENDING',
[6]	724	'RUNNING',
[3]	725	'ACTIVE',
[6]	726	'STARTED',
[3]	727	'COMPLETED',
[6]	728	'FINISHED',
[3]	729	'DONE',
	730	'DATA',
[6]	731	'RESULTS_GEN',
[3]	732	'CANCELED',
	733	'CANCELLED',
	734	'FAILED',
[6]	735	'STAGING',
[3]	736	'UNKNOWN');
	737	if ( ! in_array( $gfac_status, $status_types ) )
	738	mail_to_admin( 'debug', "gfacID: /$gfacID/\n" .
	739	"XML: /$xml/\n" .
	740	"Status: /$gfac_status/\n" );
	741
[6]	742	if ( in_array( $gfac_status, array( 'DONE', 'DATA', 'RESULTS_GEN' ) ) )
	743	$gfac_status = 'DATA';
	744
[1]	745	return $gfac_status;
	746	}
	747
	748	// Function to request data outputs from GFAC
	749	function get_gfac_outputs( $gfacID )
	750	{
	751	global $serviceURL;
[6]	752	global $self;
[1]	753
	754	// Make sure it's a GFAC job and status is appropriate for this call
	755	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
	756	{
	757	// Then it's not a GFAC job
	758	return false;
	759	}
	760
[6]	761	if ( ! in_array( $job_status, array( 'DONE', 'FAILED', 'COMPLETE', 'FINISHED' ) ) )
[1]	762	{
	763	// Then it's not appropriate to request data
	764	return false;
	765	}
	766
	767	$url = "$serviceURL/registeroutput/$gfacID";
	768	try
	769	{
	770	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	771	$http = $post->send();
	772	$xml = $post->getResponseBody();
	773	}
	774	catch ( HttpException $e )
	775	{
	776	write_log( "$self: Data not available - request failed - $gfacID" );
	777	return false;
	778	}
	779
	780	mail_to_admin( "debug", "get_gfac_outputs/\n$xml/" ); // Temporary, to see what the xml looks like,
	781	// if we ever get one
	782
	783	// Parse the result
	784	$gfac_status = parse_response( $xml );
	785
	786	return $gfac_status;
	787	}
	788
	789	function parse_response( $xml )
	790	{
	791	global $gfac_message;
	792
	793	$status = "";
	794	$gfac_message = "";
	795
	796	$parser = new XMLReader();
	797	$parser->xml( $xml );
	798
	799	while( $parser->read() )
	800	{
	801	$type = $parser->nodeType;
	802
	803	if ( $type == XMLReader::ELEMENT )
	804	$name = $parser->name;
	805
	806	else if ( $type == XMLReader::TEXT )
	807	{
	808	if ( $name == "status" )
	809	$status = $parser->value;
	810	else
	811	$gfac_message = $parser->value;
	812	}
	813	}
	814
	815	$parser->close();
	816	return $status;
	817	}
	818
	819	// Function to get status from local cluster
	820	function get_local_status( $gfacID )
	821	{
	822	global $cluster;
[6]	823	global $self;
[1]	824
	825	$system = "$cluster.uthscsa.edu";
	826	$system = preg_replace( "/\-local/", "", $system );
	827	$cmd = "/usr/bin/ssh -x us3@$system qstat -a $gfacID 2>&1";
	828
	829	$result = exec( $cmd );
	830
	831	if ( $result == "" \|\| preg_match( "/^qstat: Unknown/", $result ) )
	832	{
	833	write_log( "$self get_local_status: Local job $gfacID unknown" );
	834	return 'UNKNOWN';
	835	}
	836
	837	$values = preg_split( "/\s+/", $result );
	838	// write_log( "$self: get_local_status: job status = /{$values[9]}/");
	839	switch ( $values[ 9 ] )
	840	{
	841	case "W" : // Waiting for execution time to be reached
	842	case "E" : // Job is exiting after having run
	843	case "R" : // Still running
	844	$status = 'ACTIVE';
	845	break;
	846
	847	case "C" : // Job has completed
	848	$status = 'COMPLETED';
	849	break;
	850
	851	case "T" : // Job is being moved
	852	case "H" : // Held
	853	case "Q" : // Queued
	854	$status = 'SUBMITTED';
	855	break;
	856
	857	default :
	858	$status = 'UNKNOWN'; // This should not occur
	859	break;
	860	}
	861
	862	return $status;
	863	}
	864
	865	function update_queue_messages( $message )
	866	{
	867	global $self;
	868	global $gLink;
	869	global $gfacID;
	870
	871	// Get analysis table ID
	872	$query = "SELECT id FROM analysis " .
	873	"WHERE gfacID = '$gfacID' ";
	874	$result = mysql_query( $query, $gLink );
	875	if ( ! $result )
	876	{
	877	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	878	return;
	879	}
	880	list( $analysisID ) = mysql_fetch_array( $result );
	881
	882	// Insert message into queue_message table
	883	$query = "INSERT INTO queue_messages SET " .
[6]	884	"message = '" . mysql_real_escape_string( $message, $gLink ) . "', " .
	885	"analysisID = '$analysisID' ";
[1]	886	$result = mysql_query( $query, $gLink );
	887	if ( ! $result )
	888	{
	889	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
	890	return;
	891	}
	892	}
	893
	894	function update_db( $message )
	895	{
	896	global $self;
	897	global $gfacID;
	898	global $dbhost;
	899	global $user;
	900	global $passwd;
	901	global $us3_db;
	902
	903	$us3_link = mysql_connect( $dbhost, $user, $passwd );
	904
	905	if ( ! $us3_link )
	906	{
	907	write_log( "$self: could not connect: $dbhost, $user, $passwd" );
	908	mail_to_admin( "fail", "Could not connect to $dbhost" );
	909	return 0;
	910	}
	911
	912
	913	$result = mysql_select_db( $us3_db, $us3_link );
	914
	915	if ( ! $result )
	916	{
	917	write_log( "$self: could not select DB $us3_db" );
	918	mail_to_admin( "fail", "Could not select DB $us3_db, $dbhost, $user, $passwd" );
	919	return 0;
	920	}
	921
	922	$query = "UPDATE HPCAnalysisResult SET " .
	923	"lastMessage='" . mysql_real_escape_string( $message, $us3_link ) . "'" .
	924	"WHERE gfacID = '$gfacID' ";
	925
	926	mysql_query( $query, $us3_link );
	927	mysql_close( $us3_link );
	928	}
	929
	930	function mail_to_admin( $type, $msg )
	931	{
	932	global $updateTime;
	933	global $status;
	934	global $cluster;
	935	global $org_name;
	936	global $admin_email;
	937	global $dbhost;
	938	global $requestID;
	939
	940	$headers = "From: $org_name Admin<$admin_email>" . "\n";
	941	$headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
[6]	942	$headers .= "Bcc: Gary Gorbet<gegorbet@gmail.com>" . "\n"; // make sure
[1]	943
	944	// Set the reply address
	945	$headers .= "Reply-To: $org_name<$admin_email>" . "\n";
	946	$headers .= "Return-Path: $org_name<$admin_email>" . "\n";
	947
	948	// Try to avoid spam filters
	949	$now = time();
	950	$headers .= "Message-ID: <" . $now . "gridctl@$dbhost>$requestID\n";
	951	$headers .= "X-Mailer: PHP v" . phpversion() . "\n";
	952	$headers .= "MIME-Version: 1.0" . "\n";
	953	$headers .= "Content-Transfer-Encoding: 8bit" . "\n";
	954
	955	$subject = "US3 Error Notification";
	956	$message = "
	957	UltraScan job error notification from gridctl.php:
	958
	959	Update Time : $updateTime
	960	GFAC Status : $status
	961	Cluster : $cluster
	962	";
	963
	964	$message .= "Error Message : $msg\n";
	965
	966	mail( $admin_email, $subject, $message, $headers );
	967	}
[6]	968
	969	// Convert a status string to one of the standard DB status strings
	970	function standard_status( $status_in )
	971	{
	972	switch ( $status_in )
	973	{ // Map variations to standard gateway status values
	974	case 'QUEUED' :
	975	case 'LAUNCHED' :
	976	case 'CREATED' :
	977	case 'VALIDATED' :
	978	case 'SCHEDULED' :
	979	case 'submitted' :
	980	case '' :
	981	$status = 'SUBMITTED';
	982	break;
	983
	984	case 'EXECUTING' :
	985	case 'ACTIVE' :
	986	case 'running' :
	987	case 'executing' :
	988	$status = 'RUNNING';
	989	break;
	990
	991	case 'PENDING' :
	992	case 'CANCELING' :
	993	$status = 'UPDATING';
	994	break;
	995
	996	case 'CANCELLED' :
	997	case 'canceled' :
	998	$status = 'CANCELED';
	999	break;
	1000
	1001	case 'COMPLETED' :
	1002	case 'completed' :
	1003	$status = 'COMPLETE';
	1004	break;
	1005
	1006	case 'FAILED_DATA' :
	1007	case 'SUBMIT_TIMEOUT' :
	1008	case 'RUN_TIMEOUT' :
	1009	case 'DATA_TIMEOUT' :
	1010	$status = 'FAILED';
	1011	break;
	1012
	1013	case 'COMPLETE' :
	1014	$status = 'DONE';
	1015	break;
	1016
	1017	case 'UNKNOWN' :
	1018	$status = 'ERROR';
	1019	break;
	1020
	1021	// Where already standard value, retain value
	1022	case 'ERROR' :
	1023	case 'RUNNING' :
	1024	case 'SUBMITTED' :
	1025	case 'UPDATING' :
	1026	case 'CANCELED' :
	1027	case 'DATA' :
	1028	case 'FAILED' :
	1029	case 'DONE' :
	1030	case 'FINISHED' :
	1031	default :
	1032	$status = $status_in;
	1033	break;
	1034	}
	1035
	1036	return $status;
	1037	}
	1038
	1039	function aira_status( $gfacID, $status_in )
	1040	{
	1041	global $self;
	1042	global $loghdr;
[10]	1043	global $class_dir;
[6]	1044	//echo "a_st: st_in$status_in : $gfacID\n";
	1045	//$status_gw = standard_status( $status_in );
	1046	$status_gw = $status_in;
	1047	//echo "a_st: st_db=$status_gw\n";
	1048	$status = $status_gw;
[10]	1049	$me_devel = preg_match( "/class_devel/", $class_dir );
	1050	$job_devel = preg_match( "/US3-ADEV/i", $gfacID );
	1051	$devmatch = ( ( !$me_devel && !$job_devel ) \|\|
	1052	( $me_devel && $job_devel ) );
[6]	1053
[10]	1054	//write_log( "$loghdr gfacID=$gfacID devmatch=$devmatch" );
	1055	//write_log( "$loghdr me_d=$me_devel jo_d=$job_devel cd=$class_dir" );
	1056	if ( preg_match( "/US3-A/i", $gfacID ) && $devmatch )
	1057	{ // Airavata job and development/production type is right
[6]	1058	$status_ex = getExperimentStatus( $gfacID );
[10]	1059	//write_log( "$loghdr status_ex $status_ex" );
[6]	1060
	1061	if ( $status_ex == 'COMPLETED' )
	1062	{ // Experiment is COMPLETED: check for 'FINISHED' or 'DONE'
	1063	if ( $status_gw == 'FINISHED' \|\| $status_gw == 'DONE' )
	1064	{ // COMPLETED + FINISHED/DONE : gateway status is now COMPLETE
	1065	$status = 'COMPLETE';
	1066	}
	1067
	1068	else
	1069	{ // COMPLETED + NOT-FINISHED/DONE: gw status now DONE
	1070	$status = 'DONE';
	1071	}
	1072	}
	1073
	1074	else if ( $status_gw == 'FINISHED' \|\| $status_gw == 'DONE' )
	1075	{ // Gfac status == FINISHED/DONE: leave as is (unless FAILED)
	1076	$status = $status_gw;
	1077	if ( $status_ex == 'FAILED' )
	1078	{
	1079	sleep( 10 );
	1080	$status_ex = getExperimentStatus( $gfacID );
	1081	if ( $status_ex == 'FAILED' )
	1082	{
	1083	write_log( "$loghdr status still 'FAILED' after 10-second delay" );
	1084	sleep( 10 );
	1085	$status_ex = getExperimentStatus( $gfacID );
	1086	if ( $status_ex == 'FAILED' )
	1087	write_log( "$loghdr status still 'FAILED' after 20-second delay" );
	1088	else
	1089	write_log( "$loghdr status is $status_ex after 20-second delayed retry" );
	1090	}
	1091	write_log( "$loghdr status reset to 'COMPLETE'" );
	1092	$status = 'COMPLETE';
	1093	}
	1094	}
	1095
	1096	else
	1097	{ // Experiment not COMPLETED/FINISHED/DONE: use experiment status
	1098	$status = standard_status( $status_ex );
	1099	}
	1100
[15]	1101	if(preg_match("/US3-A/i",$gfacID))
[14]	1102	//if(preg_match("/US3-ADEV/i",$gfacID))
[12]	1103	write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
[10]	1104	//write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
	1105	//write_log( " me_d=$me_devel jo_d=$job_devel dm=$devmatch cd=$class_dir" );
[6]	1106	if ( $status != $status_gw )
	1107	{
	1108	update_job_status( $status, $gfacID );
	1109	}
	1110	}
	1111
	1112	return $status;
	1113	}
	1114
[1]	1115	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: