Context Navigation

source: trunk/cluster_status.php@ 36

Last change on this file since 36 was 36, checked in by gegorbet, 7 years ago
cluster status fix to insure queued,running counts are numeric
File size: 12.3 KB

Rev	Line
[1]	1	<?php
	2
[20]	3	$us3bin = exec( "ls -d ~us3/lims/bin" );
[6]	4	include "$us3bin/listen-config.php";
[1]	5
[26]	6	if ( ! preg_match( "/_local/", $class_dir ) )
[25]	7	{
	8	$xml = get_data();
[1]	9
[25]	10	if ( $xml != "" )
	11	parse( $xml );
	12	}
[1]	13
	14	$data = array();
	15
[6]	16	local_status();
	17
[1]	18	foreach ( $data as $item )
	19	{
[26]	20	if ( ! preg_match( "/error/", $item[ 'running' ] ) )
	21	update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
[1]	22	}
	23
[35]	24	//exit();
	25	exit(0);
[1]	26
	27	// Get the cluster status
	28
	29	function get_data()
	30	{
	31	global $self;
	32	$url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
	33
	34	try
	35	{
	36	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	37	$http = $post->send();
	38	$xml = $post->getResponseBody();
	39	}
	40	catch ( HttpException $e )
	41	{
[6]	42	// write_log( "$self: Cluster Status not available" );
[1]	43	return "";
	44	}
	45
	46	return $xml;
	47	}
	48
	49	// Parse the xml
	50
	51	function parse( $xml )
	52	{
	53	global $data;
	54
	55	$data = array();
	56
	57	$x = new XML_Array( $xml );
	58	$d = $x->ReturnArray();
	59
	60	if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
	61
	62	foreach ( $d[ 'summaries' ] as $item )
	63	{
	64	$a = Array();
	65
	66
	67	$a[ 'queued' ] = $item[ 'waitingJobs' ];
	68	$a[ 'running' ] = $item[ 'runningJobs' ];
	69
	70	if ( $a[ 'queued' ] == "" \|\| $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
	71	if ( $a[ 'running' ] == "" \|\| $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
	72
	73	$clusterParts = explode( ".", $item[ 'resourceId' ] );
	74	$cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
	75
	76	if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
	77	if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
	78
	79	$a[ 'cluster' ] = $cluster;
	80
	81	switch ( $item[ 'resourceStatus' ] )
	82	{
	83	case 'UP' :
	84	$status = "up";
	85	break;
	86
	87	case 'DOWN' :
	88	$status = "down";
	89	break;
	90
	91	case 'WARN' :
	92	$status = "warn";
	93	break;
	94
	95	case 'FAILED' :
	96	default :
	97	$status = "unknown";
	98	break;
	99	}
	100
	101	$a[ 'status' ] = $status;
	102
	103	$data[] = $a;
	104	}
	105	}
	106
	107	// Put it in the DB
	108
	109	function update( $cluster, $queued, $status, $running )
	110	{
	111	global $dbhost;
	112	global $guser;
	113	global $gpasswd;
	114	global $gDB;
	115	global $self;
[23]	116	//echo " $cluster $queued, $status, $running\n";
[1]	117
[35]	118	$gfac_link = mysqli_connect( $dbhost, $guser, $gpasswd, $gDB );
[1]	119
[35]	120	if ( ! $gfac_link )
[1]	121	{
	122	write_log( "$self: Could not connect to DB $gDB" );
	123	echo "Could not connect to DB $gDB.\n";
	124	exit();
	125	}
	126
	127	$query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
[35]	128	$result = mysqli_query( $gfac_link, $query );
[1]	129
	130	if ( ! $result )
	131	{
[35]	132	write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
	133	echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
[1]	134	exit();
	135	}
	136
[35]	137	$rows = mysqli_num_rows( $result );
[1]	138
	139	if ( $rows == 0 ) // INSERT
	140	{
	141	$query = "INSERT INTO cluster_status SET " .
	142	"cluster='$cluster', " .
	143	"queued=$queued, " .
	144	"running=$running, " .
	145	"status='$status'";
	146	}
	147	else // UPDATE
	148	{
	149	$query = "UPDATE cluster_status SET " .
	150	"queued=$queued, " .
	151	"running=$running, " .
	152	"status='$status' " .
	153	"WHERE cluster='$cluster'";
	154	}
	155
[35]	156	$result = mysqli_query( $gfac_link, $query );
[1]	157
	158	if ( ! $result )
	159	{
[35]	160	write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
	161	echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
[1]	162	}
	163	}
	164
[6]	165	// Get local cluster status
	166
	167	function local_status()
	168	{
	169	global $self;
	170	global $data;
[25]	171	global $dbhost;
	172	global $org_domain;
[26]	173	global $class_dir;
[6]	174
[26]	175	if ( preg_match( "/_local/", $class_dir ) )
[25]	176	{
	177	if ( preg_match( "/attlocal/", $org_domain ) )
[28]	178	$clusters = array( "us3iab-devel", "alamo-local" );
[25]	179	else
	180	$clusters = array( "us3iab-node0" );
	181	}
	182	else
	183	{
[35]	184	// $clusters = array( "alamo", "lonestar5", "comet",
	185	// "stampede2", "jetstream", "jureca", "jacinto" );
	186	$clusters = array( "alamo", "lonestar5", "comet",
	187	"stampede2", "jetstream" );
[25]	188	}
	189
[6]	190	foreach ( $clusters as $clname )
	191	{
	192	$a = Array();
[25]	193	//echo "$self: clname=$clname\n";
	194
[11]	195	switch( $clname )
	196	{
[25]	197	case 'us3iab-node0':
	198	case 'us3iab-node1':
	199	case 'us3iab-devel':
	200	{
	201	$qstat = `/usr/bin/qstat -B 2>&1\|tail -1`;
	202
	203	$sparts = preg_split( '/\s+/', $qstat );
	204	$que = $sparts[ 3 ];
	205	$run = $sparts[ 4 ];
	206	$sta = $sparts[ 10 ];
	207	if ( $sta == "Active" )
	208	$sta = "up";
	209	else
	210	$sta = "down";
	211	break;
	212	}
[28]	213	case 'alamo-local':
[11]	214	case 'alamo':
	215	{
	216	$host = "us3@alamo.uthscsa.edu";
	217	$qstat = `ssh $host '/usr/bin/qstat -B 2>&1\|tail -1'`;
	218	$sparts = preg_split( '/\s+/', $qstat );
	219	$que = $sparts[ 3 ];
	220	$run = $sparts[ 4 ];
	221	$sta = $sparts[ 10 ];
	222	if ( $sta == "Active" )
	223	$sta = "up";
	224	else
	225	$sta = "down";
	226	break;
	227	}
[17]	228	case 'jacinto':
	229	{
	230	$host = "us3@jacinto.uthscsa.edu";
	231	$qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1\|tail -1'`;
	232	$sparts = preg_split( '/\s+/', $qstat );
	233	$que = $sparts[ 3 ];
	234	$run = $sparts[ 4 ];
	235	$sta = $sparts[ 9 ];
	236	if ( $sta == "Active" )
	237	$sta = "up";
	238	else
	239	$sta = "down";
	240	break;
	241	}
[31]	242	case 'stampede2':
	243	{
	244	$host = "us3@stampede2.tacc.utexas.edu";
[35]	245	$qstat = `ssh $host '/usr/local/bin/showq 2>/dev/null\|grep "Total Jobs"'`;
[31]	246	$sparts = preg_split( '/\s+/', $qstat );
	247	$tot = $sparts[ 2 ];
	248	$run = $sparts[ 5 ];
	249	$que = $sparts[ 8 ];
	250	$sta = "up";
	251	if ( $tot == '' \|\| $tot == '0' )
	252	$sta = "down";
	253	break;
	254	}
[17]	255	case 'lonestar5':
	256	{
	257	$host = "us3@ls5.tacc.utexas.edu";
[35]	258	$qstat = `ssh $host '/usr/local/bin/showq 2>/dev/null\|grep "Total Jobs"'`;
[17]	259	$sparts = preg_split( '/\s+/', $qstat );
	260	$tot = $sparts[ 2 ];
	261	$run = '0';
	262	$que = '0';
	263	$sta = "up";
	264	if ( $tot == '' \|\| $tot == '0' )
	265	{
	266	$sta = "down";
	267	}
	268	else
	269	{
	270	$run = $sparts[ 5 ];
[35]	271	$que = $sparts[ 8 ];
	272	// $que = $sparts[ 11 ];
[17]	273	}
	274	break;
	275	}
[11]	276	case 'comet':
	277	{
	278	$host = "us3@comet.sdsc.edu";
[35]	279	//$qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" \|tail -1'`;
	280	$qstat = `ssh $host '/home/us3/scripts/cstat 2>&1'`;
[11]	281	$sparts = preg_split( '/\s+/', $qstat );
[35]	282	$tot = $sparts[ 1 ];
	283	$run = '0';
	284	$que = '0';
	285	$sta = "up";
	286	if ( $tot == '' \|\| $tot == '0' )
	287	{
[15]	288	$sta = "down";
[35]	289	}
[11]	290	else
[35]	291	{
	292	$run = $sparts[ 3 ];
	293	$que = $sparts[ 5 ];
	294	}
[11]	295	break;
	296	}
	297	case 'jureca':
	298	{
[15]	299	$host = "swus1@jureca.fz-juelich.de";
[17]	300	$qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
[11]	301	$sparts = preg_split( '/\s+/', $qstat );
	302	$sta = $sparts[ 0 ];
[17]	303	$run = $sparts[ 1 ];
	304	$que = $sparts[ 2 ];
[11]	305	break;
	306	}
[31]	307	case 'jetstream-local':
	308	case 'jetstream':
	309	{
[32]	310	$host = "us3@js-169-137.jetstream-cloud.org";
	311	// $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" \|tail -1'`;
	312	$qstat = `ssh $host '/home/us3/bin/clusstat \|tail -n 1'`;
[31]	313	$sparts = preg_split( '/\s+/', $qstat );
	314	$sta = $sparts[ 0 ];
	315	$knts = $sparts[ 1 ];
	316	$sparts = preg_split( '/\//', $knts );
	317	$run = $sparts[ 0 ];
[32]	318	$que = $sparts[ 2 ];
	319	$tot = $sparts[ 3 ];
[31]	320	if ( $sta == "" )
	321	$sta = "down";
	322	break;
	323	}
[11]	324	}
[6]	325
[17]	326	if ( $sta == "" )
	327	$sta = "down";
	328
[11]	329	if ( $sta == "down" )
	330	{
	331	$que = "0";
	332	$run = "0";
	333	}
	334
[36]	335	// Insure queued,running counts are numeric
	336	$que_s = $que;
	337	$run_s = $run;
	338	$que_e = preg_replace( '/=/', "", $que_s );
	339	$run_e = preg_replace( '/=/', "", $run_s );
	340	$que = intval( $que_e );
	341	$run = intval( $run_e );
	342	if($que!=$que_s \|\| $run!=$run_s)
	343	echo "$self: *** que s,e,i $que_s $que_e $que run s,e,i $run_s $run_e $run\n";
	344
	345	// Save cluster status values
[6]	346	$a[ 'cluster' ] = $clname;
	347	$a[ 'queued' ] = $que;
	348	$a[ 'running' ] = $run;
	349	$a[ 'status' ] = $sta;
[32]	350	echo "$self: $clname $que $run $sta\n";
[6]	351
	352	$data[] = $a;
[11]	353
[31]	354	if ( $clname == 'alamo' \|\|
	355	$clname == 'jacinto' \|\|
	356	$clname == 'jetstream' )
[11]	357	{
	358	$a[ 'cluster' ] = $clname . "-local";
	359	$data[] = $a;
	360	}
[6]	361	}
	362	}
	363
[1]	364	class XML_Array
	365	{
	366	var $_data = Array();
	367	var $_name = Array();
	368	var $_rep = Array();
	369	var $_parser = 0;
	370	var $_level = 0;
	371	var $_index = 0;
	372
	373	function XML_Array( &$data )
	374	{
	375	$this->_parser = xml_parser_create();
	376
	377	xml_set_object ( $this->_parser, $this );
	378	xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
	379	xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
	380	xml_set_character_data_handler( $this->_parser, "_cdata" );
	381
	382	$this->_data = array();
	383	$this->_level = 0;
	384
	385	if ( ! xml_parse( $this->_parser, $data, true ) )
	386	return false;
	387
	388	xml_parser_free( $this->_parser );
	389	}
	390
	391	function & ReturnArray()
	392	{
	393	return $this->_data[ 0 ];
	394	}
	395
	396	function _startElement( $parser, $name, $attrs )
	397	{
	398	if ( $name == "resourceHealth" )
	399	{
[6]	400	## $name .= $this->_index;
[1]	401	$this->_index++;
	402	}
	403
	404	if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
	405
	406	$this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
	407	$this->_name[ $this->_level ] = $name;
	408	$this->_level++;
	409	}
	410
	411	function _endElement( $parser, $name )
	412	{
	413	if ( isset( $this->_data[ $this->_level ] ) )
	414	{
	415	$this->_addElement( $this->_name[ $this->_level - 1 ],
	416	$this->_data[ $this->_level - 1 ],
	417	$this->_data[ $this->_level ]
	418	);
	419	}
	420
	421	unset( $this->_data[ $this->_level ] );
	422	$this->_level--;
	423	$this->_rep[ $name ]++;
	424	}
	425
	426	function _cdata( $parser, $data )
	427	{
	428	if ( $this->_name[ $this->_level - 1 ] )
	429	{
	430	$this->_addElement( $this->_name[ $this->_level - 1 ],
	431	$this->_data[ $this->_level - 1 ],
	432	str_replace( array( ">", "<",""", "&" ),
	433	array( ">" , "<" , '"' , "&" ),
	434	$data
	435	)
	436	);
	437	}
	438	}
	439
	440	function _addElement( &$name, &$start, $add = array() )
	441	{
	442	if ( ( sizeof( $add ) == 0 && is_array( $add ) ) \|\| ! $add )
	443	{
	444	if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
	445	$add = '';
	446	}
	447
	448	$update = &$start[ $name ];
	449
	450	if ( is_array( $add) &&
	451	is_array( $update ) ) $update += $add;
	452	elseif ( is_array( $update ) ) return;
	453	elseif ( is_array( $add ) ) $update = $add;
	454	elseif ( $add ) $update .= $add;
	455	}
	456	}
	457	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: