[1] | 1 | <?php
|
---|
| 2 |
|
---|
[20] | 3 | $us3bin = exec( "ls -d ~us3/lims/bin" );
|
---|
[6] | 4 | include "$us3bin/listen-config.php";
|
---|
[1] | 5 |
|
---|
[26] | 6 | if ( ! preg_match( "/_local/", $class_dir ) )
|
---|
[25] | 7 | {
|
---|
| 8 | $xml = get_data();
|
---|
[1] | 9 |
|
---|
[25] | 10 | if ( $xml != "" )
|
---|
| 11 | parse( $xml );
|
---|
| 12 | }
|
---|
[1] | 13 |
|
---|
| 14 | $data = array();
|
---|
| 15 |
|
---|
[6] | 16 | local_status();
|
---|
| 17 |
|
---|
[1] | 18 | foreach ( $data as $item )
|
---|
| 19 | {
|
---|
[26] | 20 | if ( ! preg_match( "/error/", $item[ 'running' ] ) )
|
---|
| 21 | update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
|
---|
[1] | 22 | }
|
---|
| 23 |
|
---|
| 24 | exit();
|
---|
| 25 |
|
---|
| 26 | // Get the cluster status
|
---|
| 27 |
|
---|
| 28 | function get_data()
|
---|
| 29 | {
|
---|
| 30 | global $self;
|
---|
| 31 | $url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
|
---|
| 32 |
|
---|
| 33 | try
|
---|
| 34 | {
|
---|
| 35 | $post = new HttpRequest( $url, HttpRequest::METH_GET );
|
---|
| 36 | $http = $post->send();
|
---|
| 37 | $xml = $post->getResponseBody();
|
---|
| 38 | }
|
---|
| 39 | catch ( HttpException $e )
|
---|
| 40 | {
|
---|
[6] | 41 | // write_log( "$self: Cluster Status not available" );
|
---|
[1] | 42 | return "";
|
---|
| 43 | }
|
---|
| 44 |
|
---|
| 45 | return $xml;
|
---|
| 46 | }
|
---|
| 47 |
|
---|
| 48 | // Parse the xml
|
---|
| 49 |
|
---|
| 50 | function parse( $xml )
|
---|
| 51 | {
|
---|
| 52 | global $data;
|
---|
| 53 |
|
---|
| 54 | $data = array();
|
---|
| 55 |
|
---|
| 56 | $x = new XML_Array( $xml );
|
---|
| 57 | $d = $x->ReturnArray();
|
---|
| 58 |
|
---|
| 59 | if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
|
---|
| 60 |
|
---|
| 61 | foreach ( $d[ 'summaries' ] as $item )
|
---|
| 62 | {
|
---|
| 63 | $a = Array();
|
---|
| 64 |
|
---|
| 65 |
|
---|
| 66 | $a[ 'queued' ] = $item[ 'waitingJobs' ];
|
---|
| 67 | $a[ 'running' ] = $item[ 'runningJobs' ];
|
---|
| 68 |
|
---|
| 69 | if ( $a[ 'queued' ] == "" || $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
|
---|
| 70 | if ( $a[ 'running' ] == "" || $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
|
---|
| 71 |
|
---|
| 72 | $clusterParts = explode( ".", $item[ 'resourceId' ] );
|
---|
| 73 | $cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
|
---|
| 74 |
|
---|
| 75 | if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
|
---|
| 76 | if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
|
---|
| 77 |
|
---|
| 78 | $a[ 'cluster' ] = $cluster;
|
---|
| 79 |
|
---|
| 80 | switch ( $item[ 'resourceStatus' ] )
|
---|
| 81 | {
|
---|
| 82 | case 'UP' :
|
---|
| 83 | $status = "up";
|
---|
| 84 | break;
|
---|
| 85 |
|
---|
| 86 | case 'DOWN' :
|
---|
| 87 | $status = "down";
|
---|
| 88 | break;
|
---|
| 89 |
|
---|
| 90 | case 'WARN' :
|
---|
| 91 | $status = "warn";
|
---|
| 92 | break;
|
---|
| 93 |
|
---|
| 94 | case 'FAILED' :
|
---|
| 95 | default :
|
---|
| 96 | $status = "unknown";
|
---|
| 97 | break;
|
---|
| 98 | }
|
---|
| 99 |
|
---|
| 100 | $a[ 'status' ] = $status;
|
---|
| 101 |
|
---|
| 102 | $data[] = $a;
|
---|
| 103 | }
|
---|
| 104 | }
|
---|
| 105 |
|
---|
| 106 | // Put it in the DB
|
---|
| 107 |
|
---|
| 108 | function update( $cluster, $queued, $status, $running )
|
---|
| 109 | {
|
---|
| 110 | global $dbhost;
|
---|
| 111 | global $guser;
|
---|
| 112 | global $gpasswd;
|
---|
| 113 | global $gDB;
|
---|
| 114 | global $self;
|
---|
[23] | 115 | //echo " $cluster $queued, $status, $running\n";
|
---|
[1] | 116 |
|
---|
| 117 | $gfac_link = mysql_connect( $dbhost, $guser, $gpasswd );
|
---|
| 118 | $result = mysql_select_db( $gDB, $gfac_link );
|
---|
| 119 |
|
---|
| 120 | if ( ! $result )
|
---|
| 121 | {
|
---|
| 122 | write_log( "$self: Could not connect to DB $gDB" );
|
---|
| 123 | echo "Could not connect to DB $gDB.\n";
|
---|
| 124 | exit();
|
---|
| 125 | }
|
---|
| 126 |
|
---|
| 127 | $query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
|
---|
| 128 | $result = mysql_query( $query, $gfac_link );
|
---|
| 129 |
|
---|
| 130 | if ( ! $result )
|
---|
| 131 | {
|
---|
| 132 | write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
|
---|
| 133 | echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
|
---|
| 134 | exit();
|
---|
| 135 | }
|
---|
| 136 |
|
---|
| 137 | $rows = mysql_num_rows( $result );
|
---|
| 138 |
|
---|
| 139 | if ( $rows == 0 ) // INSERT
|
---|
| 140 | {
|
---|
| 141 | $query = "INSERT INTO cluster_status SET " .
|
---|
| 142 | "cluster='$cluster', " .
|
---|
| 143 | "queued=$queued, " .
|
---|
| 144 | "running=$running, " .
|
---|
| 145 | "status='$status'";
|
---|
| 146 | }
|
---|
| 147 | else // UPDATE
|
---|
| 148 | {
|
---|
| 149 | $query = "UPDATE cluster_status SET " .
|
---|
| 150 | "queued=$queued, " .
|
---|
| 151 | "running=$running, " .
|
---|
| 152 | "status='$status' " .
|
---|
| 153 | "WHERE cluster='$cluster'";
|
---|
| 154 | }
|
---|
| 155 |
|
---|
| 156 | $result = mysql_query( $query, $gfac_link );
|
---|
| 157 |
|
---|
| 158 | if ( ! $result )
|
---|
| 159 | {
|
---|
| 160 | write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
|
---|
| 161 | echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
|
---|
| 162 | }
|
---|
| 163 | }
|
---|
| 164 |
|
---|
[6] | 165 | // Get local cluster status
|
---|
| 166 |
|
---|
| 167 | function local_status()
|
---|
| 168 | {
|
---|
| 169 | global $self;
|
---|
| 170 | global $data;
|
---|
[25] | 171 | global $dbhost;
|
---|
| 172 | global $org_domain;
|
---|
[26] | 173 | global $class_dir;
|
---|
[6] | 174 |
|
---|
[26] | 175 | if ( preg_match( "/_local/", $class_dir ) )
|
---|
[25] | 176 | {
|
---|
| 177 | if ( preg_match( "/attlocal/", $org_domain ) )
|
---|
[28] | 178 | $clusters = array( "us3iab-devel", "alamo-local" );
|
---|
[25] | 179 | else
|
---|
| 180 | $clusters = array( "us3iab-node0" );
|
---|
| 181 | }
|
---|
| 182 | else
|
---|
| 183 | {
|
---|
| 184 | $clusters = array( "alamo", "lonestar5", "stampede", "comet",
|
---|
[31] | 185 | "stampede2-b", "jetstream", "jureca", "jacinto-b" );
|
---|
[25] | 186 | }
|
---|
| 187 |
|
---|
[6] | 188 | foreach ( $clusters as $clname )
|
---|
| 189 | {
|
---|
| 190 | $a = Array();
|
---|
[25] | 191 | //echo "$self: clname=$clname\n";
|
---|
| 192 |
|
---|
[11] | 193 | switch( $clname )
|
---|
| 194 | {
|
---|
[25] | 195 | case 'us3iab-node0':
|
---|
| 196 | case 'us3iab-node1':
|
---|
| 197 | case 'us3iab-devel':
|
---|
| 198 | {
|
---|
| 199 | $qstat = `/usr/bin/qstat -B 2>&1|tail -1`;
|
---|
| 200 |
|
---|
| 201 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 202 | $que = $sparts[ 3 ];
|
---|
| 203 | $run = $sparts[ 4 ];
|
---|
| 204 | $sta = $sparts[ 10 ];
|
---|
| 205 | if ( $sta == "Active" )
|
---|
| 206 | $sta = "up";
|
---|
| 207 | else
|
---|
| 208 | $sta = "down";
|
---|
| 209 | break;
|
---|
| 210 | }
|
---|
[28] | 211 | case 'alamo-local':
|
---|
[11] | 212 | case 'alamo':
|
---|
| 213 | {
|
---|
| 214 | $host = "us3@alamo.uthscsa.edu";
|
---|
| 215 | $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
|
---|
| 216 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 217 | $que = $sparts[ 3 ];
|
---|
| 218 | $run = $sparts[ 4 ];
|
---|
| 219 | $sta = $sparts[ 10 ];
|
---|
| 220 | if ( $sta == "Active" )
|
---|
| 221 | $sta = "up";
|
---|
| 222 | else
|
---|
| 223 | $sta = "down";
|
---|
| 224 | break;
|
---|
| 225 | }
|
---|
[17] | 226 | case 'jacinto':
|
---|
| 227 | {
|
---|
| 228 | $host = "us3@jacinto.uthscsa.edu";
|
---|
| 229 | $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
|
---|
| 230 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 231 | $que = $sparts[ 3 ];
|
---|
| 232 | $run = $sparts[ 4 ];
|
---|
| 233 | $sta = $sparts[ 9 ];
|
---|
| 234 | if ( $sta == "Active" )
|
---|
| 235 | $sta = "up";
|
---|
| 236 | else
|
---|
| 237 | $sta = "down";
|
---|
| 238 | break;
|
---|
| 239 | }
|
---|
[11] | 240 | case 'stampede':
|
---|
| 241 | {
|
---|
| 242 | $host = "us3@stampede.tacc.utexas.edu";
|
---|
[23] | 243 | $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
|
---|
[11] | 244 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 245 | $tot = $sparts[ 2 ];
|
---|
| 246 | $run = $sparts[ 5 ];
|
---|
| 247 | $que = $sparts[ 8 ];
|
---|
| 248 | $sta = "up";
|
---|
| 249 | if ( $tot == '' || $tot == '0' )
|
---|
| 250 | $sta = "down";
|
---|
| 251 | break;
|
---|
| 252 | }
|
---|
[31] | 253 | case 'stampede2':
|
---|
| 254 | {
|
---|
| 255 | $host = "us3@stampede2.tacc.utexas.edu";
|
---|
| 256 | $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
|
---|
| 257 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 258 | $tot = $sparts[ 2 ];
|
---|
| 259 | $run = $sparts[ 5 ];
|
---|
| 260 | $que = $sparts[ 8 ];
|
---|
| 261 | $sta = "up";
|
---|
| 262 | if ( $tot == '' || $tot == '0' )
|
---|
| 263 | $sta = "down";
|
---|
| 264 | break;
|
---|
| 265 | }
|
---|
[17] | 266 | case 'lonestar5':
|
---|
| 267 | {
|
---|
| 268 | $host = "us3@ls5.tacc.utexas.edu";
|
---|
[23] | 269 | $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
|
---|
[17] | 270 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 271 | $tot = $sparts[ 2 ];
|
---|
| 272 | $run = '0';
|
---|
| 273 | $que = '0';
|
---|
| 274 | $sta = "up";
|
---|
| 275 | if ( $tot == '' || $tot == '0' )
|
---|
| 276 | {
|
---|
| 277 | $sta = "down";
|
---|
| 278 | }
|
---|
| 279 | else
|
---|
| 280 | {
|
---|
| 281 | $run = $sparts[ 5 ];
|
---|
[31] | 282 | // $que = $sparts[ 8 ];
|
---|
| 283 | $que = $sparts[ 11 ];
|
---|
[17] | 284 | }
|
---|
| 285 | break;
|
---|
| 286 | }
|
---|
[11] | 287 | case 'comet':
|
---|
| 288 | {
|
---|
| 289 | $host = "us3@comet.sdsc.edu";
|
---|
[15] | 290 | $qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" |tail -1'`;
|
---|
[11] | 291 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 292 | $sta = $sparts[ 0 ];
|
---|
| 293 | $knts = $sparts[ 1 ];
|
---|
| 294 | $sparts = preg_split( '/\//', $knts );
|
---|
| 295 | $run = $sparts[ 0 ];
|
---|
| 296 | $que = $sparts[ 1 ];
|
---|
[15] | 297 | if ( $sta == "" )
|
---|
| 298 | $sta = "down";
|
---|
[11] | 299 | break;
|
---|
| 300 | }
|
---|
| 301 | case 'gordon':
|
---|
| 302 | {
|
---|
| 303 | $host = "us3@gordon.sdsc.edu";
|
---|
| 304 | $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
|
---|
| 305 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 306 | $que = $sparts[ 3 ];
|
---|
| 307 | $run = $sparts[ 4 ];
|
---|
| 308 | $sta = $sparts[ 10 ];
|
---|
| 309 | if ( $sta == "Active" )
|
---|
| 310 | $sta = "up";
|
---|
| 311 | else
|
---|
| 312 | $sta = "down";
|
---|
| 313 | break;
|
---|
| 314 | }
|
---|
| 315 | case 'jureca':
|
---|
| 316 | {
|
---|
[15] | 317 | $host = "swus1@jureca.fz-juelich.de";
|
---|
[17] | 318 | $qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
|
---|
[11] | 319 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 320 | $sta = $sparts[ 0 ];
|
---|
[17] | 321 | $run = $sparts[ 1 ];
|
---|
| 322 | $que = $sparts[ 2 ];
|
---|
[11] | 323 | break;
|
---|
| 324 | }
|
---|
[31] | 325 | case 'jetstream-local':
|
---|
| 326 | case 'jetstream':
|
---|
| 327 | {
|
---|
| 328 | $host = "us3@js-157-184.jetstream-cloud.org";
|
---|
| 329 | $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" |tail -1'`;
|
---|
| 330 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 331 | $sta = $sparts[ 0 ];
|
---|
| 332 | $knts = $sparts[ 1 ];
|
---|
| 333 | $sparts = preg_split( '/\//', $knts );
|
---|
| 334 | $run = $sparts[ 0 ];
|
---|
| 335 | $que = $sparts[ 1 ];
|
---|
| 336 | if ( $sta == "" )
|
---|
| 337 | $sta = "down";
|
---|
| 338 | break;
|
---|
| 339 | }
|
---|
[11] | 340 | }
|
---|
[6] | 341 |
|
---|
[17] | 342 | if ( $sta == "" )
|
---|
| 343 | $sta = "down";
|
---|
| 344 |
|
---|
[11] | 345 | if ( $sta == "down" )
|
---|
| 346 | {
|
---|
| 347 | $que = "0";
|
---|
| 348 | $run = "0";
|
---|
| 349 | }
|
---|
| 350 |
|
---|
[6] | 351 | $a[ 'cluster' ] = $clname;
|
---|
| 352 | $a[ 'queued' ] = $que;
|
---|
| 353 | $a[ 'running' ] = $run;
|
---|
| 354 | $a[ 'status' ] = $sta;
|
---|
[25] | 355 | //echo "$self: $clname $que $run $sta\n";
|
---|
[6] | 356 |
|
---|
| 357 | $data[] = $a;
|
---|
[11] | 358 |
|
---|
[31] | 359 | if ( $clname == 'alamo' ||
|
---|
| 360 | $clname == 'jacinto' ||
|
---|
| 361 | $clname == 'jetstream' )
|
---|
[11] | 362 | {
|
---|
| 363 | $a[ 'cluster' ] = $clname . "-local";
|
---|
| 364 | $data[] = $a;
|
---|
| 365 | }
|
---|
[6] | 366 | }
|
---|
| 367 | }
|
---|
| 368 |
|
---|
[1] | 369 | class XML_Array
|
---|
| 370 | {
|
---|
| 371 | var $_data = Array();
|
---|
| 372 | var $_name = Array();
|
---|
| 373 | var $_rep = Array();
|
---|
| 374 | var $_parser = 0;
|
---|
| 375 | var $_level = 0;
|
---|
| 376 | var $_index = 0;
|
---|
| 377 |
|
---|
| 378 | function XML_Array( &$data )
|
---|
| 379 | {
|
---|
| 380 | $this->_parser = xml_parser_create();
|
---|
| 381 |
|
---|
| 382 | xml_set_object ( $this->_parser, $this );
|
---|
| 383 | xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
|
---|
| 384 | xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
|
---|
| 385 | xml_set_character_data_handler( $this->_parser, "_cdata" );
|
---|
| 386 |
|
---|
| 387 | $this->_data = array();
|
---|
| 388 | $this->_level = 0;
|
---|
| 389 |
|
---|
| 390 | if ( ! xml_parse( $this->_parser, $data, true ) )
|
---|
| 391 | return false;
|
---|
| 392 |
|
---|
| 393 | xml_parser_free( $this->_parser );
|
---|
| 394 | }
|
---|
| 395 |
|
---|
| 396 | function & ReturnArray()
|
---|
| 397 | {
|
---|
| 398 | return $this->_data[ 0 ];
|
---|
| 399 | }
|
---|
| 400 |
|
---|
| 401 | function _startElement( $parser, $name, $attrs )
|
---|
| 402 | {
|
---|
| 403 | if ( $name == "resourceHealth" )
|
---|
| 404 | {
|
---|
[6] | 405 | ## $name .= $this->_index;
|
---|
[1] | 406 | $this->_index++;
|
---|
| 407 | }
|
---|
| 408 |
|
---|
| 409 | if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
|
---|
| 410 |
|
---|
| 411 | $this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
|
---|
| 412 | $this->_name[ $this->_level ] = $name;
|
---|
| 413 | $this->_level++;
|
---|
| 414 | }
|
---|
| 415 |
|
---|
| 416 | function _endElement( $parser, $name )
|
---|
| 417 | {
|
---|
| 418 | if ( isset( $this->_data[ $this->_level ] ) )
|
---|
| 419 | {
|
---|
| 420 | $this->_addElement( $this->_name[ $this->_level - 1 ],
|
---|
| 421 | $this->_data[ $this->_level - 1 ],
|
---|
| 422 | $this->_data[ $this->_level ]
|
---|
| 423 | );
|
---|
| 424 | }
|
---|
| 425 |
|
---|
| 426 | unset( $this->_data[ $this->_level ] );
|
---|
| 427 | $this->_level--;
|
---|
| 428 | $this->_rep[ $name ]++;
|
---|
| 429 | }
|
---|
| 430 |
|
---|
| 431 | function _cdata( $parser, $data )
|
---|
| 432 | {
|
---|
| 433 | if ( $this->_name[ $this->_level - 1 ] )
|
---|
| 434 | {
|
---|
| 435 | $this->_addElement( $this->_name[ $this->_level - 1 ],
|
---|
| 436 | $this->_data[ $this->_level - 1 ],
|
---|
| 437 | str_replace( array( ">", "<",""", "&" ),
|
---|
| 438 | array( ">" , "<" , '"' , "&" ),
|
---|
| 439 | $data
|
---|
| 440 | )
|
---|
| 441 | );
|
---|
| 442 | }
|
---|
| 443 | }
|
---|
| 444 |
|
---|
| 445 | function _addElement( &$name, &$start, $add = array() )
|
---|
| 446 | {
|
---|
| 447 | if ( ( sizeof( $add ) == 0 && is_array( $add ) ) || ! $add )
|
---|
| 448 | {
|
---|
| 449 | if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
|
---|
| 450 | $add = '';
|
---|
| 451 | }
|
---|
| 452 |
|
---|
| 453 | $update = &$start[ $name ];
|
---|
| 454 |
|
---|
| 455 | if ( is_array( $add) &&
|
---|
| 456 | is_array( $update ) ) $update += $add;
|
---|
| 457 | elseif ( is_array( $update ) ) return;
|
---|
| 458 | elseif ( is_array( $add ) ) $update = $add;
|
---|
| 459 | elseif ( $add ) $update .= $add;
|
---|
| 460 | }
|
---|
| 461 | }
|
---|
| 462 | ?>
|
---|