Context Navigation

source: trunk/gridctl.php@ 10

Last change on this file since 10 was 10, checked in by gegorbet, 9 years ago
fixes to allow simultaneous production/development jobs
File size: 28.2 KB

Line
1	<?php
2
3	// Global variables
4	$gfac_message = "";
5	$updateTime = 0;
6	$submittime = 0;
7	$cluster = '';
8
9	//global $self;
10
11	// Produce some output temporarily, so cron will send me message
12	$now = time();
13	echo "Time started: " . date( 'Y-m-d H:i:s', $now ) . "\n";
14
15	// Get data from global GFAC DB
16	$gLink = mysql_connect( $dbhost, $guser, $gpasswd );
17
18	if ( ! mysql_select_db( $gDB, $gLink ) )
19	{
20	write_log( "$self: Could not select DB $gDB - " . mysql_error() );
21	mail_to_admin( "fail", "Internal Error: Could not select DB $gDB" );
22	exit();
23	}
24
25	$query = "SELECT gfacID, us3_db, cluster, status, queue_msg, " .
26	"UNIX_TIMESTAMP(time), time from analysis";
27	$result = mysql_query( $query, $gLink );
28
29	if ( ! $result )
30	{
31	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
32	mail_to_admin( "fail", "Query failed $query\n" . mysql_error( $gLink ) );
33	exit();
34	}
35
36	if ( mysql_num_rows( $result ) == 0 )
37	{
38	//write_log( "$self: analysis read got numrows==0" );
39	exit(); // Nothing to do
40	}
41
42	while ( list( $gfacID, $us3_db, $cluster, $status, $queue_msg, $time, $updateTime )
43	= mysql_fetch_array( $result ) )
44	{
45	// Checking we need to do for each entry
46	echo "us3db=$us3_db gfid=$gfacID\n";
47	switch ( $us3_db )
48	{
49	case 'Xuslims3_cauma3' :
50	case 'Xuslims3_cauma3d' :
51	case 'Xuslims3_HHU' :
52	case 'Xuslims3_Uni_KN' :
53	$serviceURL = "http://gridfarm005.ucs.indiana.edu:9090/ogce-rest/job";
54	break;
55
56	default :
57	// $serviceURL = "http://gridfarm005.ucs.indiana.edu:8080/ogce-rest/job";
58	break;
59	}
60
61	$awork = array();
62	$awork = explode( "-", $gfacID );
63	$gfacLabl = $awork[0] . "-" . $awork[1] . "-" . $awork[2];
64	$loghdr = $self . ":" . $gfacLabl . "...:";
65
66	// If entry is for Airvata/Thrift, get the true current status
67
68	if ( is_aira_job( $gfacID ) )
69	{
70	$status_in = $status;
71	$status = aira_status( $gfacID, $status_in );
72	if($status != $status_in )
73	write_log( "$loghdr Set to $status from $status_in" );
74	}
75	else
76	{
77	$status_gw = $status;
78	$status = get_gfac_status( $gfacID );
79	//if ( $status == 'FINISHED' )
80	if ( $status_gw == 'COMPLETE' )
81	$status = $status_gw;
82	write_log( "$loghdr non-AThrift status=$status status_gw=$status_gw" );
83	}
84
85	// Sometimes during testing, the us3_db entry is not set
86	// If $status == 'ERROR' then the condition has been processed before
87	if ( strlen( $us3_db ) == 0 && $status != 'ERROR' )
88	{
89	write_log( "$loghdr GFAC DB is NULL - $gfacID" );
90	mail_to_admin( "fail", "GFAC DB is NULL\n$gfacID" );
91
92	$query2 = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
93	$result2 = mysql_query( $query2, $gLink );
94	$status = 'ERROR';
95
96	if ( ! $result2 )
97	write_log( "$loghdr Query failed $query2 - " . mysql_error( $gLink ) );
98
99	}
100
101	//echo " st=$status\n";
102	switch ( $status )
103	{
104	// Already been handled
105	// Later update this condition to search for gfacID?
106	case "ERROR":
107	cleanup();
108	break;
109
110	case "SUBMITTED":
111	submitted( $time );
112	break;
113
114	case "SUBMIT_TIMEOUT":
115	submit_timeout( $time );
116	break;
117
118	case "RUNNING":
119	case "STARTED":
120	case "STAGING":
121	case "ACTIVE":
122	running( $time );
123	break;
124
125	case "RUN_TIMEOUT":
126	run_timeout($time );
127	break;
128
129	case "DATA":
130	case "RESULTS_GEN":
131	wait_data( $time );
132	break;
133
134	case "DATA_TIMEOUT":
135	data_timeout( $time );
136	break;
137
138	case "COMPLETED":
139	case "COMPLETE":
140	complete();
141	break;
142
143	case "CANCELLED":
144	case "CANCELED":
145	case "FAILED":
146	failed();
147	break;
148
149	case "FINISHED":
150	case "DONE":
151	// if ( is_aira_job( $gfacID ) )
152	// {
153	// complete();
154	// }
155	case "PROCESSING":
156	default:
157	break;
158	}
159	}
160
161	exit();
162
163	function submitted( $updatetime )
164	{
165	global $self;
166	global $gLink;
167	global $gfacID;
168	global $loghdr;
169
170	$now = time();
171
172	if ( $updatetime + 600 > $now ) return; // < 10 minutes ago
173
174	if ( $updatetime + 86400 > $now ) // Within the first 24 hours
175	{
176	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
177	$job_status = get_local_status( $gfacID );
178
179	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
180	return;
181
182	if ( ! in_array( $job_status, array( 'SUBMITTED', 'INITIALIZED', 'PENDING' ) ) )
183	{
184	write_log( "$loghdr submitted:job_status=$job_status" );
185	update_job_status( $job_status, $gfacID );
186	}
187
188	return;
189	}
190
191	$message = "Job listed submitted longer than 24 hours";
192	write_log( "$self: $message - id: $gfacID" );
193	mail_to_admin( "hang", "$message - id: $gfacID" );
194	$query = "UPDATE analysis SET status='SUBMIT_TIMEOUT' WHERE gfacID='$gfacID'";
195	$result = mysql_query( $query, $gLink );
196
197	if ( ! $result )
198	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
199
200	update_queue_messages( $message );
201	update_db( $message );
202	}
203
204	function submit_timeout( $updatetime )
205	{
206	global $self;
207	global $gLink;
208	global $gfacID;
209	global $loghdr;
210
211	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
212	$job_status = get_local_status( $gfacID );
213
214	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
215	return;
216
217	if ( ! in_array( $job_status, array( 'SUBMITTED', 'INITIALIZED', 'PENDING' ) ) )
218	{
219	update_job_status( $job_status, $gfacID );
220	return;
221	}
222
223	$now = time();
224
225	if ( $updatetime + 86400 > $now ) return; // < 24 hours ago ( 48 total submitted )
226
227	$message = "Job listed submitted longer than 48 hours";
228	write_log( "$self: $message - id: $gfacID" );
229	mail_to_admin( "hang", "$message - id: $gfacID" );
230	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
231	$result = mysql_query( $query, $gLink );
232
233	if ( ! $result )
234	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
235
236	update_queue_messages( $message );
237	update_db( $message );
238	}
239
240	function running( $updatetime )
241	{
242	global $self;
243	global $gLink;
244	global $gfacID;
245	global $loghdr;
246
247	$now = time();
248
249	get_us3_data();
250
251	if ( $updatetime + 600 > $now ) return; // message received < 10 minutes ago
252
253	if ( $updatetime + 86400 > $now ) // Within the first 24 hours
254	{
255	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
256	$job_status = get_local_status( $gfacID );
257
258	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
259	return;
260
261	if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
262	update_job_status( $job_status, $gfacID );
263
264	return;
265	}
266
267	$message = "Job listed running longer than 24 hours";
268	write_log( "$self: $message - id: $gfacID" );
269	mail_to_admin( "hang", "$message - id: $gfacID" );
270	$query = "UPDATE analysis SET status='RUN_TIMEOUT' WHERE gfacID='$gfacID'";
271	$result = mysql_query( $query, $gLink );
272
273	if ( ! $result )
274	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
275
276	update_queue_messages( $message );
277	update_db( $message );
278	}
279
280	function run_timeout( $updatetime )
281	{
282	global $self;
283	global $gLink;
284	global $gfacID;
285	global $loghdr;
286
287	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
288	$job_status = get_local_status( $gfacID );
289
290	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
291	return;
292
293	if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
294	{
295	update_job_status( $job_status, $gfacID );
296	return;
297	}
298
299	$now = time();
300
301	get_us3_data();
302
303	if ( $updatetime + 172800 > $now ) return; // < 48 hours ago
304
305	$message = "Job listed running longer than 48 hours";
306	write_log( "$self: $message - id: $gfacID" );
307	mail_to_admin( "hang", "$message - id: $gfacID" );
308	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
309	$result = mysql_query( $query, $gLink );
310
311	if ( ! $result )
312	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
313
314	update_queue_messages( $message );
315	update_db( $message );
316	}
317
318	function wait_data( $updatetime )
319	{
320	global $self;
321	global $gLink;
322	global $gfacID;
323	global $loghdr;
324
325	$now = time();
326
327	if ( $updatetime + 3600 > $now ) // < Within the first hour
328	{
329	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
330	$job_status = get_local_status( $gfacID );
331
332	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
333	return;
334
335	if ( $job_status != 'DATA' )
336	{
337	update_job_status( $job_status, $gfacID );
338	return;
339	}
340
341	// Request to resend data, but only request every 5 minutes
342	$minute = date( 'i' ) * 1; // Makes it an int
343	if ( $minute % 5 ) return;
344
345	$output_status = get_gfac_outputs( $gfacID );
346
347	if ( $output_status !== false )
348	mail_to_admin( "debug", "wait_data/$gfacID/$output_status" );
349
350	return;
351	}
352
353	$message = "Waiting for data longer than 1 hour";
354	write_log( "$self: $message - id: $gfacID" );
355	mail_to_admin( "hang", "$message - id: $gfacID" );
356	$query = "UPDATE analysis SET status='DATA_TIMEOUT' WHERE gfacID='$gfacID'";
357	$result = mysql_query( $query, $gLink );
358
359	if ( ! $result )
360	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
361
362	update_queue_messages( $message );
363	update_db( $message );
364	}
365
366	function data_timeout( $updatetime )
367	{
368	global $self;
369	global $gLink;
370	global $gfacID;
371	global $loghdr;
372
373	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
374	$job_status = get_local_status( $gfacID );
375
376	if ( $job_status == 'GFAC_STATUS_UNAVAILABLE' )
377	return;
378
379	if ( $job_status != 'DATA' )
380	{
381	update_job_status( $job_status, $gfacID );
382	return;
383	}
384
385	$now = time();
386
387	if ( $updatetime + 86400 > $now ) // < 24 hours ago
388	{
389	// Request to resend data, but only request every 15 minutes
390	$minute = date( 'i' ) * 1; // Makes it an int
391	if ( $minute % 15 ) return;
392
393	$output_status = get_gfac_outputs( $gfacID );
394
395	if ( $output_status !== false )
396	mail_to_admin( "debug", "data_timeout/$gfacID/$output_status" );
397
398	return;
399	}
400
401	$message = "Waiting for data longer than 24 hours";
402	write_log( "$self: $message - id: $gfacID" );
403	mail_to_admin( "hang", "$message - id: $gfacID" );
404	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
405	$result = mysql_query( $query, $gLink );
406
407	if ( ! $result )
408	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
409
410	update_queue_messages( $message );
411	update_db( $message );
412	}
413
414	function complete()
415	{
416	// Just cleanup
417	cleanup();
418	}
419
420	function failed()
421	{
422	// Just cleanup
423	cleanup();
424	}
425
426	function cleanup()
427	{
428	global $self;
429	global $gLink;
430	global $gfacID;
431	global $us3_db;
432	global $loghdr;
433	global $class_dir;
434
435	// Double check that the gfacID exists
436	$query = "SELECT count(*) FROM analysis WHERE gfacID='$gfacID'";
437	$result = mysql_query( $query, $gLink );
438
439	if ( ! $result )
440	{
441	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
442	mail_to_admin( "fail", "Query failed $query\n" . mysql_error( $gLink ) );
443	return;
444	}
445
446	list( $count ) = mysql_fetch_array( $result );
447
448	if ($count==0)
449	write_log( "$loghdr count = $count gfacID = $gfacID" );
450	if ( $count == 0 ) return;
451
452	// Now check the us3 instance
453	$requestID = get_us3_data();
454	//write_log( "$loghdr requestID = $requestID gfacID = $gfacID" );
455	if ( $requestID == 0 ) return;
456
457	if ( preg_match( "/US3-A/i", $gfacID ) )
458	{
459	$me_devel = preg_match( "/class_devel/", $class_dir );
460	$job_devel = preg_match( "/US3-ADEV/i", $gfacID );
461	if ( ( !$me_devel && !$job_devel ) \|\|
462	( $me_devel && $job_devel ) )
463	{ // If job from appropriate Airavata server, process it
464	//write_log( "$loghdr CALLING aira_cleanup()" );
465	aira_cleanup( $us3_db, $requestID, $gLink );
466	}
467	//write_log( "$loghdr RTN FR aira_cleanup()" );
468	}
469	else
470	{
471	//write_log( "$loghdr CALLING gfac_cleanup()" );
472	gfac_cleanup( $us3_db, $requestID, $gLink );
473	}
474	}
475
476	// Function to update status of job
477	function update_job_status( $job_status, $gfacID )
478	{
479	global $gLink;
480	global $query;
481	global $self;
482	global $loghdr;
483
484	switch ( $job_status )
485	{
486	case 'SUBMITTED' :
487	case 'SUBMITED' :
488	case 'INITIALIZED' :
489	case 'UPDATING' :
490	case 'PENDING' :
491	$query = "UPDATE analysis SET status='SUBMITTED' WHERE gfacID='$gfacID'";
492	$message = "Job status request reports job is SUBMITTED";
493	break;
494
495	case 'STARTED' :
496	case 'RUNNING' :
497	case 'ACTIVE' :
498	$query = "UPDATE analysis SET status='RUNNING' WHERE gfacID='$gfacID'";
499	$message = "Job status request reports job is RUNNING";
500	break;
501
502	case 'FINISHED' :
503	$query = "UPDATE analysis SET status='FINISHED' WHERE gfacID='$gfacID'";
504	$message = "NONE";
505	break;
506
507	case 'DONE' :
508	$query = "UPDATE analysis SET status='DONE' WHERE gfacID='$gfacID'";
509	$message = "NONE";
510	break;
511
512	case 'COMPLETED' :
513	case 'COMPLETE' :
514	$query = "UPDATE analysis SET status='COMPLETE' WHERE gfacID='$gfacID'";
515	$message = "Job status request reports job is COMPLETED";
516	break;
517
518	case 'DATA' :
519	$query = "UPDATE analysis SET status='DATA' WHERE gfacID='$gfacID'";
520	$message = "Job status request reports job is COMPLETE, waiting for data";
521	break;
522
523	case 'CANCELED' :
524	case 'CANCELLED' :
525	$query = "UPDATE analysis SET status='CANCELED' WHERE gfacID='$gfacID'";
526	$message = "Job status request reports job is CANCELED";
527	break;
528
529	case 'FAILED' :
530	$query = "UPDATE analysis SET status='FAILED' WHERE gfacID='$gfacID'";
531	$message = "Job status request reports job is FAILED";
532	break;
533
534	case 'UNKNOWN' :
535	write_log( "$loghdr job_status='UNKNOWN', reset to 'ERROR' " );
536	$query = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
537	$message = "Job status request reports job is not in the queue";
538	break;
539
540	default :
541	// We shouldn't ever get here
542	$query = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
543	$message = "Job status was not recognized - $job_status";
544	write_log( "$loghdr update_job_status: " .
545	"Job status was not recognized - $job_status\n" .
546	"gfacID = $gfacID\n" );
547	break;
548
549	}
550
551	$result = mysql_query( $query, $gLink );
552	if ( ! $result )
553	write_log( "$loghdr Query failed $query - " . mysql_error( $gLink ) );
554
555	if ( $message != 'NONE' )
556	{
557	update_queue_messages( $message );
558	update_db( $message );
559	}
560	}
561
562	function get_us3_data()
563	{
564	global $self;
565	global $gfacID;
566	global $dbhost;
567	global $user;
568	global $passwd;
569	global $us3_db;
570	global $updateTime;
571	global $loghdr;
572
573	$us3_link = mysql_connect( $dbhost, $user, $passwd );
574
575	if ( ! $us3_link )
576	{
577	write_log( "$loghdr could not connect: $dbhost, $user, $passwd" );
578	mail_to_admin( "fail", "Could not connect to $dbhost" );
579	return 0;
580	}
581
582
583	$result = mysql_select_db( $us3_db, $us3_link );
584
585	if ( ! $result )
586	{
587	write_log( "$loghdr could not select DB $us3_db" );
588	mail_to_admin( "fail", "Could not select DB $us3_db, $dbhost, $user, $passwd" );
589	return 0;
590	}
591
592	$query = "SELECT HPCAnalysisRequestID, UNIX_TIMESTAMP(updateTime) " .
593	"FROM HPCAnalysisResult WHERE gfacID='$gfacID'";
594	$result = mysql_query( $query, $us3_link );
595
596	if ( ! $result )
597	{
598	write_log( "$self: Query failed $query - " . mysql_error( $us3_link ) );
599	mail_to_admin( "fail", "Query failed $query\n" . mysql_error( $us3_link ) );
600	return 0;
601	}
602
603	list( $requestID, $updateTime ) = mysql_fetch_array( $result );
604	mysql_close( $us3_link );
605
606	return $requestID;
607	}
608
609	// Function to determine if this is a gfac job or not
610	function is_gfac_job( $gfacID )
611	{
612	$hex = "[0-9a-fA-F]";
613	if ( ! preg_match( "/^US3-Experiment/i", $gfacID ) &&
614	! preg_match( "/^US3-$hex{8}-$hex{4}-$hex{4}-$hex{4}-$hex{12}$/", $gfacID ) )
615	{
616	// Then it's not a GFAC job
617	return false;
618	}
619
620	return true;
621	}
622
623	// Function to determine if this is an airavata/thrift job or not
624	function is_aira_job( $gfacID )
625	{
626	global $cluster;
627
628	if ( preg_match( "/US3-A/i", $gfacID ) &&
629	! preg_match( "/juropa/i", $cluster ) )
630	{
631	// Then it's an Airavata/Thrift job
632	return true;
633	}
634
635	return false;
636	}
637
638	// Function to get the current job status from GFAC
639	function get_gfac_status( $gfacID )
640	{
641	global $serviceURL;
642	global $self;
643	global $loghdr;
644	global $cluster;
645
646	if ( is_aira_job( $gfacID ) )
647	{
648	$status_ex = getExperimentStatus( $gfacID );
649	$gfac_status = standard_status( $status_ex );
650	return $gfac_status;
651	}
652
653	else if ( ! is_gfac_job( $gfacID ) )
654	return false;
655
656	$url = "$serviceURL/jobstatus/$gfacID";
657	try
658	{
659	$post = new HttpRequest( $url, HttpRequest::METH_GET );
660	$http = $post->send();
661	$xml = $post->getResponseBody();
662	}
663	catch ( HttpException $e )
664	{
665	write_log( "$loghdr Status not available - marking failed - $gfacID" );
666	return 'GFAC_STATUS_UNAVAILABLE';
667	}
668
669	// Parse the result
670	$gfac_status = parse_response( $xml );
671
672	// This may not seem like the best place to do this, but here we have
673	// the xml straight from GFAC
674	$status_types = array('SUBMITTED',
675	'SUBMITED',
676	'INITIALIZED',
677	'PENDING',
678	'RUNNING',
679	'ACTIVE',
680	'STARTED',
681	'COMPLETED',
682	'FINISHED',
683	'DONE',
684	'DATA',
685	'RESULTS_GEN',
686	'CANCELED',
687	'CANCELLED',
688	'FAILED',
689	'STAGING',
690	'UNKNOWN');
691	if ( ! in_array( $gfac_status, $status_types ) )
692	mail_to_admin( 'debug', "gfacID: /$gfacID/\n" .
693	"XML: /$xml/\n" .
694	"Status: /$gfac_status/\n" );
695
696	if ( in_array( $gfac_status, array( 'DONE', 'DATA', 'RESULTS_GEN' ) ) )
697	$gfac_status = 'DATA';
698
699	return $gfac_status;
700	}
701
702	// Function to request data outputs from GFAC
703	function get_gfac_outputs( $gfacID )
704	{
705	global $serviceURL;
706	global $self;
707
708	// Make sure it's a GFAC job and status is appropriate for this call
709	if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
710	{
711	// Then it's not a GFAC job
712	return false;
713	}
714
715	if ( ! in_array( $job_status, array( 'DONE', 'FAILED', 'COMPLETE', 'FINISHED' ) ) )
716	{
717	// Then it's not appropriate to request data
718	return false;
719	}
720
721	$url = "$serviceURL/registeroutput/$gfacID";
722	try
723	{
724	$post = new HttpRequest( $url, HttpRequest::METH_GET );
725	$http = $post->send();
726	$xml = $post->getResponseBody();
727	}
728	catch ( HttpException $e )
729	{
730	write_log( "$self: Data not available - request failed - $gfacID" );
731	return false;
732	}
733
734	mail_to_admin( "debug", "get_gfac_outputs/\n$xml/" ); // Temporary, to see what the xml looks like,
735	// if we ever get one
736
737	// Parse the result
738	$gfac_status = parse_response( $xml );
739
740	return $gfac_status;
741	}
742
743	function parse_response( $xml )
744	{
745	global $gfac_message;
746
747	$status = "";
748	$gfac_message = "";
749
750	$parser = new XMLReader();
751	$parser->xml( $xml );
752
753	while( $parser->read() )
754	{
755	$type = $parser->nodeType;
756
757	if ( $type == XMLReader::ELEMENT )
758	$name = $parser->name;
759
760	else if ( $type == XMLReader::TEXT )
761	{
762	if ( $name == "status" )
763	$status = $parser->value;
764	else
765	$gfac_message = $parser->value;
766	}
767	}
768
769	$parser->close();
770	return $status;
771	}
772
773	// Function to get status from local cluster
774	function get_local_status( $gfacID )
775	{
776	global $cluster;
777	global $self;
778
779	$system = "$cluster.uthscsa.edu";
780	$system = preg_replace( "/\-local/", "", $system );
781	$cmd = "/usr/bin/ssh -x us3@$system qstat -a $gfacID 2>&1";
782
783	$result = exec( $cmd );
784
785	if ( $result == "" \|\| preg_match( "/^qstat: Unknown/", $result ) )
786	{
787	write_log( "$self get_local_status: Local job $gfacID unknown" );
788	return 'UNKNOWN';
789	}
790
791	$values = preg_split( "/\s+/", $result );
792	// write_log( "$self: get_local_status: job status = /{$values[9]}/");
793	switch ( $values[ 9 ] )
794	{
795	case "W" : // Waiting for execution time to be reached
796	case "E" : // Job is exiting after having run
797	case "R" : // Still running
798	$status = 'ACTIVE';
799	break;
800
801	case "C" : // Job has completed
802	$status = 'COMPLETED';
803	break;
804
805	case "T" : // Job is being moved
806	case "H" : // Held
807	case "Q" : // Queued
808	$status = 'SUBMITTED';
809	break;
810
811	default :
812	$status = 'UNKNOWN'; // This should not occur
813	break;
814	}
815
816	return $status;
817	}
818
819	function update_queue_messages( $message )
820	{
821	global $self;
822	global $gLink;
823	global $gfacID;
824
825	// Get analysis table ID
826	$query = "SELECT id FROM analysis " .
827	"WHERE gfacID = '$gfacID' ";
828	$result = mysql_query( $query, $gLink );
829	if ( ! $result )
830	{
831	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
832	return;
833	}
834	list( $analysisID ) = mysql_fetch_array( $result );
835
836	// Insert message into queue_message table
837	$query = "INSERT INTO queue_messages SET " .
838	"message = '" . mysql_real_escape_string( $message, $gLink ) . "', " .
839	"analysisID = '$analysisID' ";
840	$result = mysql_query( $query, $gLink );
841	if ( ! $result )
842	{
843	write_log( "$self: Query failed $query - " . mysql_error( $gLink ) );
844	return;
845	}
846	}
847
848	function update_db( $message )
849	{
850	global $self;
851	global $gfacID;
852	global $dbhost;
853	global $user;
854	global $passwd;
855	global $us3_db;
856
857	$us3_link = mysql_connect( $dbhost, $user, $passwd );
858
859	if ( ! $us3_link )
860	{
861	write_log( "$self: could not connect: $dbhost, $user, $passwd" );
862	mail_to_admin( "fail", "Could not connect to $dbhost" );
863	return 0;
864	}
865
866
867	$result = mysql_select_db( $us3_db, $us3_link );
868
869	if ( ! $result )
870	{
871	write_log( "$self: could not select DB $us3_db" );
872	mail_to_admin( "fail", "Could not select DB $us3_db, $dbhost, $user, $passwd" );
873	return 0;
874	}
875
876	$query = "UPDATE HPCAnalysisResult SET " .
877	"lastMessage='" . mysql_real_escape_string( $message, $us3_link ) . "'" .
878	"WHERE gfacID = '$gfacID' ";
879
880	mysql_query( $query, $us3_link );
881	mysql_close( $us3_link );
882	}
883
884	function mail_to_admin( $type, $msg )
885	{
886	global $updateTime;
887	global $status;
888	global $cluster;
889	global $org_name;
890	global $admin_email;
891	global $dbhost;
892	global $requestID;
893
894	$headers = "From: $org_name Admin<$admin_email>" . "\n";
895	$headers .= "Cc: $org_name Admin<$admin_email>" . "\n";
896	$headers .= "Bcc: Gary Gorbet<gegorbet@gmail.com>" . "\n"; // make sure
897
898	// Set the reply address
899	$headers .= "Reply-To: $org_name<$admin_email>" . "\n";
900	$headers .= "Return-Path: $org_name<$admin_email>" . "\n";
901
902	// Try to avoid spam filters
903	$now = time();
904	$headers .= "Message-ID: <" . $now . "gridctl@$dbhost>$requestID\n";
905	$headers .= "X-Mailer: PHP v" . phpversion() . "\n";
906	$headers .= "MIME-Version: 1.0" . "\n";
907	$headers .= "Content-Transfer-Encoding: 8bit" . "\n";
908
909	$subject = "US3 Error Notification";
910	$message = "
911	UltraScan job error notification from gridctl.php:
912
913	Update Time : $updateTime
914	GFAC Status : $status
915	Cluster : $cluster
916	";
917
918	$message .= "Error Message : $msg\n";
919
920	mail( $admin_email, $subject, $message, $headers );
921	}
922
923	// Convert a status string to one of the standard DB status strings
924	function standard_status( $status_in )
925	{
926	switch ( $status_in )
927	{ // Map variations to standard gateway status values
928	case 'QUEUED' :
929	case 'LAUNCHED' :
930	case 'CREATED' :
931	case 'VALIDATED' :
932	case 'SCHEDULED' :
933	case 'submitted' :
934	case '' :
935	$status = 'SUBMITTED';
936	break;
937
938	case 'EXECUTING' :
939	case 'ACTIVE' :
940	case 'running' :
941	case 'executing' :
942	$status = 'RUNNING';
943	break;
944
945	case 'PENDING' :
946	case 'CANCELING' :
947	$status = 'UPDATING';
948	break;
949
950	case 'CANCELLED' :
951	case 'canceled' :
952	$status = 'CANCELED';
953	break;
954
955	$status = 'DATA';
956	break;
957
958	case 'COMPLETED' :
959	case 'completed' :
960	$status = 'COMPLETE';
961	break;
962
963	case 'FAILED_DATA' :
964	case 'SUBMIT_TIMEOUT' :
965	case 'RUN_TIMEOUT' :
966	case 'DATA_TIMEOUT' :
967	$status = 'FAILED';
968	break;
969
970	case 'COMPLETE' :
971	$status = 'DONE';
972	break;
973
974	case 'UNKNOWN' :
975	$status = 'ERROR';
976	break;
977
978	// Where already standard value, retain value
979	case 'ERROR' :
980	case 'RUNNING' :
981	case 'SUBMITTED' :
982	case 'UPDATING' :
983	case 'CANCELED' :
984	case 'DATA' :
985	case 'FAILED' :
986	case 'DONE' :
987	case 'FINISHED' :
988	default :
989	$status = $status_in;
990	break;
991	}
992
993	return $status;
994	}
995
996	function aira_status( $gfacID, $status_in )
997	{
998	global $self;
999	global $loghdr;
1000	global $class_dir;
1001	//echo "a_st: st_in$status_in : $gfacID\n";
1002	//$status_gw = standard_status( $status_in );
1003	$status_gw = $status_in;
1004	//echo "a_st: st_db=$status_gw\n";
1005	$status = $status_gw;
1006	$me_devel = preg_match( "/class_devel/", $class_dir );
1007	$job_devel = preg_match( "/US3-ADEV/i", $gfacID );
1008	$devmatch = ( ( !$me_devel && !$job_devel ) \|\|
1009	( $me_devel && $job_devel ) );
1010
1011	//write_log( "$loghdr gfacID=$gfacID devmatch=$devmatch" );
1012	//write_log( "$loghdr me_d=$me_devel jo_d=$job_devel cd=$class_dir" );
1013	if ( preg_match( "/US3-A/i", $gfacID ) && $devmatch )
1014	{ // Airavata job and development/production type is right
1015	$status_ex = getExperimentStatus( $gfacID );
1016	//write_log( "$loghdr status_ex $status_ex" );
1017
1018	if ( $status_ex == 'COMPLETED' )
1019	{ // Experiment is COMPLETED: check for 'FINISHED' or 'DONE'
1020	if ( $status_gw == 'FINISHED' \|\| $status_gw == 'DONE' )
1021	{ // COMPLETED + FINISHED/DONE : gateway status is now COMPLETE
1022	$status = 'COMPLETE';
1023	}
1024
1025	else
1026	{ // COMPLETED + NOT-FINISHED/DONE: gw status now DONE
1027	$status = 'DONE';
1028	}
1029	}
1030
1031	else if ( $status_gw == 'FINISHED' \|\| $status_gw == 'DONE' )
1032	{ // Gfac status == FINISHED/DONE: leave as is (unless FAILED)
1033	$status = $status_gw;
1034	if ( $status_ex == 'FAILED' )
1035	{
1036	sleep( 10 );
1037	$status_ex = getExperimentStatus( $gfacID );
1038	if ( $status_ex == 'FAILED' )
1039	{
1040	write_log( "$loghdr status still 'FAILED' after 10-second delay" );
1041	sleep( 10 );
1042	$status_ex = getExperimentStatus( $gfacID );
1043	if ( $status_ex == 'FAILED' )
1044	write_log( "$loghdr status still 'FAILED' after 20-second delay" );
1045	else
1046	write_log( "$loghdr status is $status_ex after 20-second delayed retry" );
1047	}
1048	write_log( "$loghdr status reset to 'COMPLETE'" );
1049	$status = 'COMPLETE';
1050	}
1051	}
1052
1053	else
1054	{ // Experiment not COMPLETED/FINISHED/DONE: use experiment status
1055	$status = standard_status( $status_ex );
1056	}
1057
1058	//write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
1059	//write_log( " me_d=$me_devel jo_d=$job_devel dm=$devmatch cd=$class_dir" );
1060	if ( $status != $status_gw )
1061	{
1062	update_job_status( $status, $gfacID );
1063	}
1064	}
1065
1066	return $status;
1067	}
1068
1069	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: