Context Navigation

source: trunk/cluster_status.php@ 37

Last change on this file since 37 was 37, checked in by gegorbet, 6 years ago
gridctl mods for mysqli and aucsolutions, plus new text for crontabs and cronic
File size: 12.1 KB

Rev	Line
[1]	1	<?php
	2
[20]	3	$us3bin = exec( "ls -d ~us3/lims/bin" );
[6]	4	include "$us3bin/listen-config.php";
[1]	5
[26]	6	if ( ! preg_match( "/_local/", $class_dir ) )
[25]	7	{
	8	$xml = get_data();
[1]	9
[25]	10	if ( $xml != "" )
	11	parse( $xml );
	12	}
[1]	13
	14	$data = array();
	15
[6]	16	local_status();
	17
[1]	18	foreach ( $data as $item )
	19	{
[26]	20	if ( ! preg_match( "/error/", $item[ 'running' ] ) )
	21	update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
[1]	22	}
	23
[35]	24	//exit();
	25	exit(0);
[1]	26
	27	// Get the cluster status
	28
	29	function get_data()
	30	{
	31	global $self;
	32	$url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
	33
	34	try
	35	{
	36	$post = new HttpRequest( $url, HttpRequest::METH_GET );
	37	$http = $post->send();
	38	$xml = $post->getResponseBody();
	39	}
	40	catch ( HttpException $e )
	41	{
[6]	42	// write_log( "$self: Cluster Status not available" );
[1]	43	return "";
	44	}
	45
	46	return $xml;
	47	}
	48
	49	// Parse the xml
	50
	51	function parse( $xml )
	52	{
	53	global $data;
	54
	55	$data = array();
	56
	57	$x = new XML_Array( $xml );
	58	$d = $x->ReturnArray();
	59
	60	if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
	61
	62	foreach ( $d[ 'summaries' ] as $item )
	63	{
	64	$a = Array();
	65
	66
	67	$a[ 'queued' ] = $item[ 'waitingJobs' ];
	68	$a[ 'running' ] = $item[ 'runningJobs' ];
	69
	70	if ( $a[ 'queued' ] == "" \|\| $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
	71	if ( $a[ 'running' ] == "" \|\| $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
	72
	73	$clusterParts = explode( ".", $item[ 'resourceId' ] );
	74	$cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
	75
	76	if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
	77	if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
	78
	79	$a[ 'cluster' ] = $cluster;
	80
	81	switch ( $item[ 'resourceStatus' ] )
	82	{
	83	case 'UP' :
	84	$status = "up";
	85	break;
	86
	87	case 'DOWN' :
	88	$status = "down";
	89	break;
	90
	91	case 'WARN' :
	92	$status = "warn";
	93	break;
	94
	95	case 'FAILED' :
	96	default :
	97	$status = "unknown";
	98	break;
	99	}
	100
	101	$a[ 'status' ] = $status;
	102
	103	$data[] = $a;
	104	}
	105	}
	106
	107	// Put it in the DB
	108
	109	function update( $cluster, $queued, $status, $running )
	110	{
	111	global $dbhost;
	112	global $guser;
	113	global $gpasswd;
	114	global $gDB;
	115	global $self;
[23]	116	//echo " $cluster $queued, $status, $running\n";
[1]	117
[35]	118	$gfac_link = mysqli_connect( $dbhost, $guser, $gpasswd, $gDB );
[1]	119
[35]	120	if ( ! $gfac_link )
[1]	121	{
	122	write_log( "$self: Could not connect to DB $gDB" );
	123	echo "Could not connect to DB $gDB.\n";
	124	exit();
	125	}
	126
	127	$query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
[35]	128	$result = mysqli_query( $gfac_link, $query );
[1]	129
	130	if ( ! $result )
	131	{
[35]	132	write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
	133	echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
[1]	134	exit();
	135	}
	136
[35]	137	$rows = mysqli_num_rows( $result );
[1]	138
	139	if ( $rows == 0 ) // INSERT
	140	{
	141	$query = "INSERT INTO cluster_status SET " .
	142	"cluster='$cluster', " .
	143	"queued=$queued, " .
	144	"running=$running, " .
	145	"status='$status'";
	146	}
	147	else // UPDATE
	148	{
	149	$query = "UPDATE cluster_status SET " .
	150	"queued=$queued, " .
	151	"running=$running, " .
	152	"status='$status' " .
	153	"WHERE cluster='$cluster'";
	154	}
	155
[35]	156	$result = mysqli_query( $gfac_link, $query );
[1]	157
	158	if ( ! $result )
	159	{
[35]	160	write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
	161	echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
[1]	162	}
	163	}
	164
[6]	165	// Get local cluster status
	166
	167	function local_status()
	168	{
	169	global $self;
	170	global $data;
[25]	171	global $dbhost;
	172	global $org_domain;
[26]	173	global $class_dir;
[6]	174
[26]	175	if ( preg_match( "/_local/", $class_dir ) )
[25]	176	{
	177	if ( preg_match( "/attlocal/", $org_domain ) )
[28]	178	$clusters = array( "us3iab-devel", "alamo-local" );
[25]	179	else
	180	$clusters = array( "us3iab-node0" );
	181	}
	182	else
	183	{
[37]	184	$clusters = array( "lonestar5", "comet",
[35]	185	"stampede2", "jetstream" );
[25]	186	}
	187
[6]	188	foreach ( $clusters as $clname )
	189	{
	190	$a = Array();
[25]	191	//echo "$self: clname=$clname\n";
	192
[11]	193	switch( $clname )
	194	{
[25]	195	case 'us3iab-node0':
	196	case 'us3iab-node1':
	197	case 'us3iab-devel':
	198	{
	199	$qstat = `/usr/bin/qstat -B 2>&1\|tail -1`;
	200
	201	$sparts = preg_split( '/\s+/', $qstat );
	202	$que = $sparts[ 3 ];
	203	$run = $sparts[ 4 ];
	204	$sta = $sparts[ 10 ];
	205	if ( $sta == "Active" )
	206	$sta = "up";
	207	else
	208	$sta = "down";
	209	break;
	210	}
[28]	211	case 'alamo-local':
[11]	212	case 'alamo':
	213	{
	214	$host = "us3@alamo.uthscsa.edu";
	215	$qstat = `ssh $host '/usr/bin/qstat -B 2>&1\|tail -1'`;
	216	$sparts = preg_split( '/\s+/', $qstat );
	217	$que = $sparts[ 3 ];
	218	$run = $sparts[ 4 ];
	219	$sta = $sparts[ 10 ];
	220	if ( $sta == "Active" )
	221	$sta = "up";
	222	else
	223	$sta = "down";
	224	break;
	225	}
[17]	226	case 'jacinto':
	227	{
	228	$host = "us3@jacinto.uthscsa.edu";
	229	$qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1\|tail -1'`;
	230	$sparts = preg_split( '/\s+/', $qstat );
	231	$que = $sparts[ 3 ];
	232	$run = $sparts[ 4 ];
	233	$sta = $sparts[ 9 ];
	234	if ( $sta == "Active" )
	235	$sta = "up";
	236	else
	237	$sta = "down";
	238	break;
	239	}
[31]	240	case 'stampede2':
	241	{
	242	$host = "us3@stampede2.tacc.utexas.edu";
[35]	243	$qstat = `ssh $host '/usr/local/bin/showq 2>/dev/null\|grep "Total Jobs"'`;
[31]	244	$sparts = preg_split( '/\s+/', $qstat );
	245	$tot = $sparts[ 2 ];
	246	$run = $sparts[ 5 ];
	247	$que = $sparts[ 8 ];
	248	$sta = "up";
	249	if ( $tot == '' \|\| $tot == '0' )
	250	$sta = "down";
	251	break;
	252	}
[17]	253	case 'lonestar5':
	254	{
	255	$host = "us3@ls5.tacc.utexas.edu";
[35]	256	$qstat = `ssh $host '/usr/local/bin/showq 2>/dev/null\|grep "Total Jobs"'`;
[17]	257	$sparts = preg_split( '/\s+/', $qstat );
	258	$tot = $sparts[ 2 ];
	259	$run = '0';
	260	$que = '0';
	261	$sta = "up";
	262	if ( $tot == '' \|\| $tot == '0' )
	263	{
	264	$sta = "down";
	265	}
	266	else
	267	{
	268	$run = $sparts[ 5 ];
[35]	269	$que = $sparts[ 8 ];
	270	// $que = $sparts[ 11 ];
[17]	271	}
	272	break;
	273	}
[11]	274	case 'comet':
	275	{
	276	$host = "us3@comet.sdsc.edu";
[35]	277	//$qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" \|tail -1'`;
	278	$qstat = `ssh $host '/home/us3/scripts/cstat 2>&1'`;
[11]	279	$sparts = preg_split( '/\s+/', $qstat );
[35]	280	$tot = $sparts[ 1 ];
	281	$run = '0';
	282	$que = '0';
	283	$sta = "up";
	284	if ( $tot == '' \|\| $tot == '0' )
	285	{
[15]	286	$sta = "down";
[35]	287	}
[11]	288	else
[35]	289	{
	290	$run = $sparts[ 3 ];
	291	$que = $sparts[ 5 ];
	292	}
[11]	293	break;
	294	}
	295	case 'jureca':
	296	{
[15]	297	$host = "swus1@jureca.fz-juelich.de";
[17]	298	$qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
[11]	299	$sparts = preg_split( '/\s+/', $qstat );
	300	$sta = $sparts[ 0 ];
[17]	301	$run = $sparts[ 1 ];
	302	$que = $sparts[ 2 ];
[11]	303	break;
	304	}
[31]	305	case 'jetstream-local':
	306	case 'jetstream':
	307	{
[32]	308	$host = "us3@js-169-137.jetstream-cloud.org";
	309	// $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" \|tail -1'`;
	310	$qstat = `ssh $host '/home/us3/bin/clusstat \|tail -n 1'`;
[31]	311	$sparts = preg_split( '/\s+/', $qstat );
	312	$sta = $sparts[ 0 ];
	313	$knts = $sparts[ 1 ];
	314	$sparts = preg_split( '/\//', $knts );
	315	$run = $sparts[ 0 ];
[32]	316	$que = $sparts[ 2 ];
	317	$tot = $sparts[ 3 ];
[31]	318	if ( $sta == "" )
	319	$sta = "down";
	320	break;
	321	}
[11]	322	}
[6]	323
[17]	324	if ( $sta == "" )
	325	$sta = "down";
	326
[11]	327	if ( $sta == "down" )
	328	{
	329	$que = "0";
	330	$run = "0";
	331	}
	332
[36]	333	// Insure queued,running counts are numeric
	334	$que_s = $que;
	335	$run_s = $run;
	336	$que_e = preg_replace( '/=/', "", $que_s );
	337	$run_e = preg_replace( '/=/', "", $run_s );
	338	$que = intval( $que_e );
	339	$run = intval( $run_e );
	340	if($que!=$que_s \|\| $run!=$run_s)
	341	echo "$self: *** que s,e,i $que_s $que_e $que run s,e,i $run_s $run_e $run\n";
	342
	343	// Save cluster status values
[6]	344	$a[ 'cluster' ] = $clname;
	345	$a[ 'queued' ] = $que;
	346	$a[ 'running' ] = $run;
	347	$a[ 'status' ] = $sta;
[32]	348	echo "$self: $clname $que $run $sta\n";
[6]	349
	350	$data[] = $a;
[11]	351
[31]	352	if ( $clname == 'alamo' \|\|
	353	$clname == 'jacinto' \|\|
	354	$clname == 'jetstream' )
[11]	355	{
	356	$a[ 'cluster' ] = $clname . "-local";
	357	$data[] = $a;
	358	}
[6]	359	}
	360	}
	361
[1]	362	class XML_Array
	363	{
	364	var $_data = Array();
	365	var $_name = Array();
	366	var $_rep = Array();
	367	var $_parser = 0;
	368	var $_level = 0;
	369	var $_index = 0;
	370
	371	function XML_Array( &$data )
	372	{
	373	$this->_parser = xml_parser_create();
	374
	375	xml_set_object ( $this->_parser, $this );
	376	xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
	377	xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
	378	xml_set_character_data_handler( $this->_parser, "_cdata" );
	379
	380	$this->_data = array();
	381	$this->_level = 0;
	382
	383	if ( ! xml_parse( $this->_parser, $data, true ) )
	384	return false;
	385
	386	xml_parser_free( $this->_parser );
	387	}
	388
	389	function & ReturnArray()
	390	{
	391	return $this->_data[ 0 ];
	392	}
	393
	394	function _startElement( $parser, $name, $attrs )
	395	{
	396	if ( $name == "resourceHealth" )
	397	{
[6]	398	## $name .= $this->_index;
[1]	399	$this->_index++;
	400	}
	401
	402	if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
	403
	404	$this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
	405	$this->_name[ $this->_level ] = $name;
	406	$this->_level++;
	407	}
	408
	409	function _endElement( $parser, $name )
	410	{
	411	if ( isset( $this->_data[ $this->_level ] ) )
	412	{
	413	$this->_addElement( $this->_name[ $this->_level - 1 ],
	414	$this->_data[ $this->_level - 1 ],
	415	$this->_data[ $this->_level ]
	416	);
	417	}
	418
	419	unset( $this->_data[ $this->_level ] );
	420	$this->_level--;
	421	$this->_rep[ $name ]++;
	422	}
	423
	424	function _cdata( $parser, $data )
	425	{
	426	if ( $this->_name[ $this->_level - 1 ] )
	427	{
	428	$this->_addElement( $this->_name[ $this->_level - 1 ],
	429	$this->_data[ $this->_level - 1 ],
	430	str_replace( array( ">", "<",""", "&" ),
	431	array( ">" , "<" , '"' , "&" ),
	432	$data
	433	)
	434	);
	435	}
	436	}
	437
	438	function _addElement( &$name, &$start, $add = array() )
	439	{
	440	if ( ( sizeof( $add ) == 0 && is_array( $add ) ) \|\| ! $add )
	441	{
	442	if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
	443	$add = '';
	444	}
	445
	446	$update = &$start[ $name ];
	447
	448	if ( is_array( $add) &&
	449	is_array( $update ) ) $update += $add;
	450	elseif ( is_array( $update ) ) return;
	451	elseif ( is_array( $add ) ) $update = $add;
	452	elseif ( $add ) $update .= $add;
	453	}
	454	}
	455	?>

Note: See TracBrowser for help on using the repository browser.

Download in other formats: