Context Navigation

source: trunk/cluster_status.php@ 41

Last change on this file since 41 was 41, checked in by gegorbet, 6 years ago
gridctl mods for uslims.uleth.ca demeler3 cluster
File size: 13.4 KB

Rev	Line
[1]	1	<?php
	2
[20]	3	$us3bin = exec( "ls -d ~us3/lims/bin" );
[6]	4	include "$us3bin/listen-config.php";
[1]	5
[26]	6	if ( ! preg_match( "/_local/", $class_dir ) )
[25]	7	{
	8	$xml = get_data();
[1]	9
[25]	10	if ( $xml != "" )
	11	parse( $xml );
	12	}
[1]	13
	14	$data = array();
	15
[6]	16	local_status();
	17
[1]	18	foreach ( $data as $item )
	19	{
[26]	20	if ( ! preg_match( "/error/", $item[ 'running' ] ) )
	21	update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
[1]	22	}
	23
[35]	24	//exit();
	25	exit(0);
[1]	26
	27	// Get the cluster status
	28
	29	function get_data()
	30	{
	31	global $self;
	32	$url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
	33
	34	try
	35	{
	36	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	37	$http = $post->send();
	38	$xml = $post->getResponseBody();
	39	}
	40	catch ( HttpException $e )
	41	{
[6]	42	// write_log( "$self: Cluster Status not available" );
[1]	43	return "";
	44	}
	45
	46	return $xml;
	47	}
	48
	49	// Parse the xml
	50
	51	function parse( $xml )
	52	{
	53	global $data;
	54
	55	$data = array();
	56
	57	$x = new XML_Array( $xml );
	58	$d = $x->ReturnArray();
	59
	60	if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
	61
	62	foreach ( $d[ 'summaries' ] as $item )
	63	{
	64	$a = Array();
	65
	66
	67	$a[ 'queued' ] = $item[ 'waitingJobs' ];
	68	$a[ 'running' ] = $item[ 'runningJobs' ];
	69
	70	if ( $a[ 'queued' ] == "" \|\| $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
	71	if ( $a[ 'running' ] == "" \|\| $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
	72
	73	$clusterParts = explode( ".", $item[ 'resourceId' ] );
	74	$cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
	75
	76	if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
	77	if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
	78
	79	$a[ 'cluster' ] = $cluster;
	80
	81	switch ( $item[ 'resourceStatus' ] )
	82	{
	83	case 'UP' :
	84	$status = "up";
	85	break;
	86
	87	case 'DOWN' :
	88	$status = "down";
	89	break;
	90
	91	case 'WARN' :
	92	$status = "warn";
	93	break;
	94
	95	case 'FAILED' :
	96	default :
	97	$status = "unknown";
	98	break;
	99	}
	100
	101	$a[ 'status' ] = $status;
	102
	103	$data[] = $a;
	104	}
	105	}
	106
	107	// Put it in the DB
	108
	109	function update( $cluster, $queued, $status, $running )
	110	{
	111	global $dbhost;
	112	global $guser;
	113	global $gpasswd;
	114	global $gDB;
	115	global $self;
[23]	116	//echo " $cluster $queued, $status, $running\n";
[1]	117
[35]	118	$gfac_link = mysqli_connect( $dbhost, $guser, $gpasswd, $gDB );
[1]	119
[35]	120	if ( ! $gfac_link )
[1]	121	{
	122	write_log( "$self: Could not connect to DB $gDB" );
	123	echo "Could not connect to DB $gDB.\n";
	124	exit();
	125	}
	126
	127	$query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
[35]	128	$result = mysqli_query( $gfac_link, $query );
[1]	129
	130	if ( ! $result )
	131	{
[35]	132	write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
	133	echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
[1]	134	exit();
	135	}
	136
[35]	137	$rows = mysqli_num_rows( $result );
[1]	138
	139	if ( $rows == 0 ) // INSERT
	140	{
	141	$query = "INSERT INTO cluster_status SET " .
	142	"cluster='$cluster', " .
	143	"queued=$queued, " .
	144	"running=$running, " .
	145	"status='$status'";
	146	}
	147	else // UPDATE
	148	{
	149	$query = "UPDATE cluster_status SET " .
	150	"queued=$queued, " .
	151	"running=$running, " .
	152	"status='$status' " .
	153	"WHERE cluster='$cluster'";
	154	}
	155
[35]	156	$result = mysqli_query( $gfac_link, $query );
[1]	157
	158	if ( ! $result )
	159	{
[35]	160	write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
	161	echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
[1]	162	}
	163	}
	164
[6]	165	// Get local cluster status
	166
	167	function local_status()
	168	{
	169	global $self;
	170	global $data;
[25]	171	global $dbhost;
	172	global $org_domain;
[26]	173	global $class_dir;
[6]	174
[26]	175	if ( preg_match( "/_local/", $class_dir ) )
[25]	176	{
	177	if ( preg_match( "/attlocal/", $org_domain ) )
[41]	178	$clusters = array( "us3iab-devel" );
[25]	179	else
[41]	180	$clusters = array( "us3iab-node0", "demeler3-local", "taito-local" );
[25]	181	}
	182	else
	183	{
[37]	184	$clusters = array( "lonestar5", "comet",
[35]	185	"stampede2", "jetstream" );
[25]	186	}
	187
[6]	188	foreach ( $clusters as $clname )
	189	{
	190	$a = Array();
[25]	191	//echo "$self: clname=$clname\n";
	192
[11]	193	switch( $clname )
	194	{
[25]	195	case 'us3iab-node0':
	196	case 'us3iab-node1':
	197	case 'us3iab-devel':
	198	{
	199	$qstat = `/usr/bin/qstat -B 2>&1\|tail -1`;
	200
	201	$sparts = preg_split( '/\s+/', $qstat );
	202	$que = $sparts[ 3 ];
	203	$run = $sparts[ 4 ];
	204	$sta = $sparts[ 10 ];
	205	if ( $sta == "Active" )
	206	$sta = "up";
	207	else
	208	$sta = "down";
	209	break;
	210	}
[28]	211	case 'alamo-local':
[11]	212	case 'alamo':
	213	{
	214	$host = "us3@alamo.uthscsa.edu";
	215	$qstat = `ssh $host '/usr/bin/qstat -B 2>&1\|tail -1'`;
	216	$sparts = preg_split( '/\s+/', $qstat );
	217	$que = $sparts[ 3 ];
	218	$run = $sparts[ 4 ];
	219	$sta = $sparts[ 10 ];
	220	if ( $sta == "Active" )
	221	$sta = "up";
	222	else
	223	$sta = "down";
	224	break;
	225	}
[41]	226
	227	case 'demeler3-local':
	228	{
	229	$host = "us3@demeler3.uleth.ca";
	230	$qstat = `ssh $host '/usr/bin/qstat -B 2>&1\|tail -1'`;
	231	$sparts = preg_split( '/\s+/', $qstat );
	232	$que = $sparts[ 3 ];
	233	$run = $sparts[ 4 ];
	234	$sta = $sparts[ 10 ];
	235	if ( $sta == "Active" )
	236	$sta = "up";
	237	else
	238	$sta = "down";
	239	break;
	240	}
	241
[17]	242	case 'jacinto':
	243	{
	244	$host = "us3@jacinto.uthscsa.edu";
	245	$qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1\|tail -1'`;
	246	$sparts = preg_split( '/\s+/', $qstat );
	247	$que = $sparts[ 3 ];
	248	$run = $sparts[ 4 ];
	249	$sta = $sparts[ 9 ];
	250	if ( $sta == "Active" )
	251	$sta = "up";
	252	else
	253	$sta = "down";
	254	break;
	255	}
[31]	256	case 'stampede2':
	257	{
	258	$host = "us3@stampede2.tacc.utexas.edu";
[39]	259	$qstat = `ssh $host '~us3/scripts/clusstat skx-normal 2>/dev/null'`;
[31]	260	$sparts = preg_split( '/\s+/', $qstat );
	261	$tot = $sparts[ 2 ];
	262	$run = $sparts[ 5 ];
	263	$que = $sparts[ 8 ];
	264	$sta = "up";
	265	if ( $tot == '' \|\| $tot == '0' )
	266	$sta = "down";
	267	break;
	268	}
[17]	269	case 'lonestar5':
	270	{
	271	$host = "us3@ls5.tacc.utexas.edu";
[39]	272	$qstat = `ssh $host '/opt/apps/tacc/bin/showq 2>/dev/null\|grep "Total Jobs"'`;
[17]	273	$sparts = preg_split( '/\s+/', $qstat );
	274	$tot = $sparts[ 2 ];
	275	$run = '0';
	276	$que = '0';
	277	$sta = "up";
	278	if ( $tot == '' \|\| $tot == '0' )
	279	{
	280	$sta = "down";
	281	}
	282	else
	283	{
	284	$run = $sparts[ 5 ];
[35]	285	$que = $sparts[ 8 ];
	286	// $que = $sparts[ 11 ];
[17]	287	}
	288	break;
	289	}
[11]	290	case 'comet':
	291	{
	292	$host = "us3@comet.sdsc.edu";
[35]	293	//$qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" \|tail -1'`;
	294	$qstat = `ssh $host '/home/us3/scripts/cstat 2>&1'`;
[11]	295	$sparts = preg_split( '/\s+/', $qstat );
[35]	296	$tot = $sparts[ 1 ];
	297	$run = '0';
	298	$que = '0';
	299	$sta = "up";
	300	if ( $tot == '' \|\| $tot == '0' )
	301	{
[15]	302	$sta = "down";
[35]	303	}
[11]	304	else
[35]	305	{
	306	$run = $sparts[ 3 ];
	307	$que = $sparts[ 5 ];
[39]	308	if ( $run == '0' && $que == '0' )
	309	{
	310	$sta = "down";
	311	}
[35]	312	}
[11]	313	break;
	314	}
	315	case 'jureca':
	316	{
[15]	317	$host = "swus1@jureca.fz-juelich.de";
[17]	318	$qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
[11]	319	$sparts = preg_split( '/\s+/', $qstat );
	320	$sta = $sparts[ 0 ];
[17]	321	$run = $sparts[ 1 ];
	322	$que = $sparts[ 2 ];
[11]	323	break;
	324	}
[31]	325	case 'jetstream-local':
	326	case 'jetstream':
	327	{
[32]	328	$host = "us3@js-169-137.jetstream-cloud.org";
	329	// $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" \|tail -1'`;
	330	$qstat = `ssh $host '/home/us3/bin/clusstat \|tail -n 1'`;
[31]	331	$sparts = preg_split( '/\s+/', $qstat );
	332	$sta = $sparts[ 0 ];
	333	$knts = $sparts[ 1 ];
	334	$sparts = preg_split( '/\//', $knts );
	335	$run = $sparts[ 0 ];
[32]	336	$que = $sparts[ 2 ];
	337	$tot = $sparts[ 3 ];
[31]	338	if ( $sta == "" )
	339	$sta = "down";
	340	break;
	341	}
[39]	342	case 'taito-local':
	343	{
	344	$host = "rb_2001068_taito01@taito.csc.fi";
	345	$qstat = `ssh -i /home/us3/.ssh/id_rsa_taito_robot $host '/homeappl/home/rb_2001068_taito01/scripts/cstat 2>&1'`;
	346	$sparts = preg_split( '/\s+/', $qstat );
	347	$tot = $sparts[ 1 ];
	348	$run = '0';
	349	$que = '0';
	350	$sta = "up";
	351	if ( $tot == '' \|\| $tot == '0' )
	352	{
	353	$sta = "down";
	354	}
	355	else
	356	{
	357	$run = $sparts[ 3 ];
	358	$que = $sparts[ 5 ];
	359	}
	360	break;
	361	}
	362
[11]	363	}
[6]	364
[17]	365	if ( $sta == "" )
	366	$sta = "down";
	367
[11]	368	if ( $sta == "down" )
	369	{
	370	$que = "0";
	371	$run = "0";
	372	}
	373
[36]	374	// Insure queued,running counts are numeric
	375	$que_s = $que;
	376	$run_s = $run;
	377	$que_e = preg_replace( '/=/', "", $que_s );
	378	$run_e = preg_replace( '/=/', "", $run_s );
	379	$que = intval( $que_e );
	380	$run = intval( $run_e );
	381	if($que!=$que_s \|\| $run!=$run_s)
	382	echo "$self: *** que s,e,i $que_s $que_e $que run s,e,i $run_s $run_e $run\n";
	383
	384	// Save cluster status values
[6]	385	$a[ 'cluster' ] = $clname;
	386	$a[ 'queued' ] = $que;
	387	$a[ 'running' ] = $run;
	388	$a[ 'status' ] = $sta;
[32]	389	echo "$self: $clname $que $run $sta\n";
[6]	390
	391	$data[] = $a;
[11]	392
[31]	393	if ( $clname == 'alamo' \|\|
	394	$clname == 'jacinto' \|\|
	395	$clname == 'jetstream' )
[11]	396	{
	397	$a[ 'cluster' ] = $clname . "-local";
	398	$data[] = $a;
	399	}
[6]	400	}
	401	}
	402
[1]	403	class XML_Array
	404	{
	405	var $_data = Array();
	406	var $_name = Array();
	407	var $_rep = Array();
	408	var $_parser = 0;
	409	var $_level = 0;
	410	var $_index = 0;
	411
	412	function XML_Array( &$data )
	413	{
	414	$this->_parser = xml_parser_create();
	415
	416	xml_set_object ( $this->_parser, $this );
	417	xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
	418	xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
	419	xml_set_character_data_handler( $this->_parser, "_cdata" );
	420
	421	$this->_data = array();
	422	$this->_level = 0;
	423
	424	if ( ! xml_parse( $this->_parser, $data, true ) )
	425	return false;
	426
	427	xml_parser_free( $this->_parser );
	428	}
	429
	430	function & ReturnArray()
	431	{
	432	return $this->_data[ 0 ];
	433	}
	434
	435	function _startElement( $parser, $name, $attrs )
	436	{
	437	if ( $name == "resourceHealth" )
	438	{
[6]	439	## $name .= $this->_index;
[1]	440	$this->_index++;
	441	}
	442
	443	if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
	444
	445	$this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
	446	$this->_name[ $this->_level ] = $name;
	447	$this->_level++;
	448	}
	449
	450	function _endElement( $parser, $name )
	451	{
	452	if ( isset( $this->_data[ $this->_level ] ) )
	453	{
	454	$this->_addElement( $this->_name[ $this->_level - 1 ],
	455	$this->_data[ $this->_level - 1 ],
	456	$this->_data[ $this->_level ]
	457	);
	458	}
	459
	460	unset( $this->_data[ $this->_level ] );
	461	$this->_level--;
	462	$this->_rep[ $name ]++;
	463	}
	464
	465	function _cdata( $parser, $data )
	466	{
	467	if ( $this->_name[ $this->_level - 1 ] )
	468	{
	469	$this->_addElement( $this->_name[ $this->_level - 1 ],
	470	$this->_data[ $this->_level - 1 ],
	471	str_replace( array( ">", "<",""", "&" ),
	472	array( ">" , "<" , '"' , "&" ),
	473	$data
	474	)
	475	);
	476	}
	477	}
	478
	479	function _addElement( &$name, &$start, $add = array() )
	480	{
	481	if ( ( sizeof( $add ) == 0 && is_array( $add ) ) \|\| ! $add )
	482	{
	483	if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
	484	$add = '';
	485	}
	486
	487	$update = &$start[ $name ];
	488
	489	if ( is_array( $add) &&
	490	is_array( $update ) ) $update += $add;
	491	elseif ( is_array( $update ) ) return;
	492	elseif ( is_array( $add ) ) $update = $add;
	493	elseif ( $add ) $update .= $add;
	494	}
	495	}
	496	?>

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: trunk/cluster_status.php@ 41

Download in other formats: