[1] | 1 | <?php
|
---|
| 2 |
|
---|
[20] | 3 | $us3bin = exec( "ls -d ~us3/lims/bin" );
|
---|
[6] | 4 | include "$us3bin/listen-config.php";
|
---|
[1] | 5 |
|
---|
[26] | 6 | if ( ! preg_match( "/_local/", $class_dir ) )
|
---|
[25] | 7 | {
|
---|
| 8 | $xml = get_data();
|
---|
[1] | 9 |
|
---|
[25] | 10 | if ( $xml != "" )
|
---|
| 11 | parse( $xml );
|
---|
| 12 | }
|
---|
[1] | 13 |
|
---|
| 14 | $data = array();
|
---|
| 15 |
|
---|
[6] | 16 | local_status();
|
---|
| 17 |
|
---|
[1] | 18 | foreach ( $data as $item )
|
---|
| 19 | {
|
---|
[26] | 20 | if ( ! preg_match( "/error/", $item[ 'running' ] ) )
|
---|
| 21 | update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
|
---|
[1] | 22 | }
|
---|
| 23 |
|
---|
[35] | 24 | //exit();
|
---|
| 25 | exit(0);
|
---|
[1] | 26 |
|
---|
| 27 | // Get the cluster status
|
---|
| 28 |
|
---|
| 29 | function get_data()
|
---|
| 30 | {
|
---|
| 31 | global $self;
|
---|
| 32 | $url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
|
---|
| 33 |
|
---|
| 34 | try
|
---|
| 35 | {
|
---|
| 36 | $post = new HttpRequest( $url, HttpRequest::METH_GET );
|
---|
| 37 | $http = $post->send();
|
---|
| 38 | $xml = $post->getResponseBody();
|
---|
| 39 | }
|
---|
| 40 | catch ( HttpException $e )
|
---|
| 41 | {
|
---|
[6] | 42 | // write_log( "$self: Cluster Status not available" );
|
---|
[1] | 43 | return "";
|
---|
| 44 | }
|
---|
| 45 |
|
---|
| 46 | return $xml;
|
---|
| 47 | }
|
---|
| 48 |
|
---|
| 49 | // Parse the xml
|
---|
| 50 |
|
---|
| 51 | function parse( $xml )
|
---|
| 52 | {
|
---|
| 53 | global $data;
|
---|
| 54 |
|
---|
| 55 | $data = array();
|
---|
| 56 |
|
---|
| 57 | $x = new XML_Array( $xml );
|
---|
| 58 | $d = $x->ReturnArray();
|
---|
| 59 |
|
---|
| 60 | if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
|
---|
| 61 |
|
---|
| 62 | foreach ( $d[ 'summaries' ] as $item )
|
---|
| 63 | {
|
---|
| 64 | $a = Array();
|
---|
| 65 |
|
---|
| 66 |
|
---|
| 67 | $a[ 'queued' ] = $item[ 'waitingJobs' ];
|
---|
| 68 | $a[ 'running' ] = $item[ 'runningJobs' ];
|
---|
| 69 |
|
---|
| 70 | if ( $a[ 'queued' ] == "" || $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
|
---|
| 71 | if ( $a[ 'running' ] == "" || $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
|
---|
| 72 |
|
---|
| 73 | $clusterParts = explode( ".", $item[ 'resourceId' ] );
|
---|
| 74 | $cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
|
---|
| 75 |
|
---|
| 76 | if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
|
---|
| 77 | if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
|
---|
| 78 |
|
---|
| 79 | $a[ 'cluster' ] = $cluster;
|
---|
| 80 |
|
---|
| 81 | switch ( $item[ 'resourceStatus' ] )
|
---|
| 82 | {
|
---|
| 83 | case 'UP' :
|
---|
| 84 | $status = "up";
|
---|
| 85 | break;
|
---|
| 86 |
|
---|
| 87 | case 'DOWN' :
|
---|
| 88 | $status = "down";
|
---|
| 89 | break;
|
---|
| 90 |
|
---|
| 91 | case 'WARN' :
|
---|
| 92 | $status = "warn";
|
---|
| 93 | break;
|
---|
| 94 |
|
---|
| 95 | case 'FAILED' :
|
---|
| 96 | default :
|
---|
| 97 | $status = "unknown";
|
---|
| 98 | break;
|
---|
| 99 | }
|
---|
| 100 |
|
---|
| 101 | $a[ 'status' ] = $status;
|
---|
| 102 |
|
---|
| 103 | $data[] = $a;
|
---|
| 104 | }
|
---|
| 105 | }
|
---|
| 106 |
|
---|
| 107 | // Put it in the DB
|
---|
| 108 |
|
---|
| 109 | function update( $cluster, $queued, $status, $running )
|
---|
| 110 | {
|
---|
| 111 | global $dbhost;
|
---|
| 112 | global $guser;
|
---|
| 113 | global $gpasswd;
|
---|
| 114 | global $gDB;
|
---|
| 115 | global $self;
|
---|
[23] | 116 | //echo " $cluster $queued, $status, $running\n";
|
---|
[1] | 117 |
|
---|
[35] | 118 | $gfac_link = mysqli_connect( $dbhost, $guser, $gpasswd, $gDB );
|
---|
[1] | 119 |
|
---|
[35] | 120 | if ( ! $gfac_link )
|
---|
[1] | 121 | {
|
---|
| 122 | write_log( "$self: Could not connect to DB $gDB" );
|
---|
| 123 | echo "Could not connect to DB $gDB.\n";
|
---|
| 124 | exit();
|
---|
| 125 | }
|
---|
| 126 |
|
---|
| 127 | $query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
|
---|
[35] | 128 | $result = mysqli_query( $gfac_link, $query );
|
---|
[1] | 129 |
|
---|
| 130 | if ( ! $result )
|
---|
| 131 | {
|
---|
[35] | 132 | write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
|
---|
| 133 | echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
|
---|
[1] | 134 | exit();
|
---|
| 135 | }
|
---|
| 136 |
|
---|
[35] | 137 | $rows = mysqli_num_rows( $result );
|
---|
[1] | 138 |
|
---|
| 139 | if ( $rows == 0 ) // INSERT
|
---|
| 140 | {
|
---|
| 141 | $query = "INSERT INTO cluster_status SET " .
|
---|
| 142 | "cluster='$cluster', " .
|
---|
| 143 | "queued=$queued, " .
|
---|
| 144 | "running=$running, " .
|
---|
| 145 | "status='$status'";
|
---|
| 146 | }
|
---|
| 147 | else // UPDATE
|
---|
| 148 | {
|
---|
| 149 | $query = "UPDATE cluster_status SET " .
|
---|
| 150 | "queued=$queued, " .
|
---|
| 151 | "running=$running, " .
|
---|
| 152 | "status='$status' " .
|
---|
| 153 | "WHERE cluster='$cluster'";
|
---|
| 154 | }
|
---|
| 155 |
|
---|
[35] | 156 | $result = mysqli_query( $gfac_link, $query );
|
---|
[1] | 157 |
|
---|
| 158 | if ( ! $result )
|
---|
| 159 | {
|
---|
[35] | 160 | write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
|
---|
| 161 | echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
|
---|
[1] | 162 | }
|
---|
| 163 | }
|
---|
| 164 |
|
---|
[6] | 165 | // Get local cluster status
|
---|
| 166 |
|
---|
| 167 | function local_status()
|
---|
| 168 | {
|
---|
| 169 | global $self;
|
---|
| 170 | global $data;
|
---|
[25] | 171 | global $dbhost;
|
---|
| 172 | global $org_domain;
|
---|
[26] | 173 | global $class_dir;
|
---|
[6] | 174 |
|
---|
[26] | 175 | if ( preg_match( "/_local/", $class_dir ) )
|
---|
[25] | 176 | {
|
---|
| 177 | if ( preg_match( "/attlocal/", $org_domain ) )
|
---|
[38] | 178 | $clusters = array( "us3iab-devel", "alamo-local", "taito-local" );
|
---|
[25] | 179 | else
|
---|
[39] | 180 | $clusters = array( "us3iab-node0", "taito-local" );
|
---|
[25] | 181 | }
|
---|
| 182 | else
|
---|
| 183 | {
|
---|
[37] | 184 | $clusters = array( "lonestar5", "comet",
|
---|
[35] | 185 | "stampede2", "jetstream" );
|
---|
[25] | 186 | }
|
---|
| 187 |
|
---|
[6] | 188 | foreach ( $clusters as $clname )
|
---|
| 189 | {
|
---|
| 190 | $a = Array();
|
---|
[25] | 191 | //echo "$self: clname=$clname\n";
|
---|
| 192 |
|
---|
[11] | 193 | switch( $clname )
|
---|
| 194 | {
|
---|
[25] | 195 | case 'us3iab-node0':
|
---|
| 196 | case 'us3iab-node1':
|
---|
| 197 | case 'us3iab-devel':
|
---|
| 198 | {
|
---|
| 199 | $qstat = `/usr/bin/qstat -B 2>&1|tail -1`;
|
---|
| 200 |
|
---|
| 201 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 202 | $que = $sparts[ 3 ];
|
---|
| 203 | $run = $sparts[ 4 ];
|
---|
| 204 | $sta = $sparts[ 10 ];
|
---|
| 205 | if ( $sta == "Active" )
|
---|
| 206 | $sta = "up";
|
---|
| 207 | else
|
---|
| 208 | $sta = "down";
|
---|
| 209 | break;
|
---|
| 210 | }
|
---|
[28] | 211 | case 'alamo-local':
|
---|
[11] | 212 | case 'alamo':
|
---|
| 213 | {
|
---|
| 214 | $host = "us3@alamo.uthscsa.edu";
|
---|
| 215 | $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
|
---|
| 216 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 217 | $que = $sparts[ 3 ];
|
---|
| 218 | $run = $sparts[ 4 ];
|
---|
| 219 | $sta = $sparts[ 10 ];
|
---|
| 220 | if ( $sta == "Active" )
|
---|
| 221 | $sta = "up";
|
---|
| 222 | else
|
---|
| 223 | $sta = "down";
|
---|
| 224 | break;
|
---|
| 225 | }
|
---|
[17] | 226 | case 'jacinto':
|
---|
| 227 | {
|
---|
| 228 | $host = "us3@jacinto.uthscsa.edu";
|
---|
| 229 | $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
|
---|
| 230 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 231 | $que = $sparts[ 3 ];
|
---|
| 232 | $run = $sparts[ 4 ];
|
---|
| 233 | $sta = $sparts[ 9 ];
|
---|
| 234 | if ( $sta == "Active" )
|
---|
| 235 | $sta = "up";
|
---|
| 236 | else
|
---|
| 237 | $sta = "down";
|
---|
| 238 | break;
|
---|
| 239 | }
|
---|
[31] | 240 | case 'stampede2':
|
---|
| 241 | {
|
---|
| 242 | $host = "us3@stampede2.tacc.utexas.edu";
|
---|
[39] | 243 | $qstat = `ssh $host '~us3/scripts/clusstat skx-normal 2>/dev/null'`;
|
---|
[31] | 244 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 245 | $tot = $sparts[ 2 ];
|
---|
| 246 | $run = $sparts[ 5 ];
|
---|
| 247 | $que = $sparts[ 8 ];
|
---|
| 248 | $sta = "up";
|
---|
| 249 | if ( $tot == '' || $tot == '0' )
|
---|
| 250 | $sta = "down";
|
---|
| 251 | break;
|
---|
| 252 | }
|
---|
[17] | 253 | case 'lonestar5':
|
---|
| 254 | {
|
---|
| 255 | $host = "us3@ls5.tacc.utexas.edu";
|
---|
[39] | 256 | $qstat = `ssh $host '/opt/apps/tacc/bin/showq 2>/dev/null|grep "Total Jobs"'`;
|
---|
[17] | 257 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 258 | $tot = $sparts[ 2 ];
|
---|
| 259 | $run = '0';
|
---|
| 260 | $que = '0';
|
---|
| 261 | $sta = "up";
|
---|
| 262 | if ( $tot == '' || $tot == '0' )
|
---|
| 263 | {
|
---|
| 264 | $sta = "down";
|
---|
| 265 | }
|
---|
| 266 | else
|
---|
| 267 | {
|
---|
| 268 | $run = $sparts[ 5 ];
|
---|
[35] | 269 | $que = $sparts[ 8 ];
|
---|
| 270 | // $que = $sparts[ 11 ];
|
---|
[17] | 271 | }
|
---|
| 272 | break;
|
---|
| 273 | }
|
---|
[11] | 274 | case 'comet':
|
---|
| 275 | {
|
---|
| 276 | $host = "us3@comet.sdsc.edu";
|
---|
[35] | 277 | //$qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" |tail -1'`;
|
---|
| 278 | $qstat = `ssh $host '/home/us3/scripts/cstat 2>&1'`;
|
---|
[11] | 279 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
[35] | 280 | $tot = $sparts[ 1 ];
|
---|
| 281 | $run = '0';
|
---|
| 282 | $que = '0';
|
---|
| 283 | $sta = "up";
|
---|
| 284 | if ( $tot == '' || $tot == '0' )
|
---|
| 285 | {
|
---|
[15] | 286 | $sta = "down";
|
---|
[35] | 287 | }
|
---|
[11] | 288 | else
|
---|
[35] | 289 | {
|
---|
| 290 | $run = $sparts[ 3 ];
|
---|
| 291 | $que = $sparts[ 5 ];
|
---|
[39] | 292 | if ( $run == '0' && $que == '0' )
|
---|
| 293 | {
|
---|
| 294 | $sta = "down";
|
---|
| 295 | }
|
---|
[35] | 296 | }
|
---|
[11] | 297 | break;
|
---|
| 298 | }
|
---|
| 299 | case 'jureca':
|
---|
| 300 | {
|
---|
[15] | 301 | $host = "swus1@jureca.fz-juelich.de";
|
---|
[17] | 302 | $qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
|
---|
[11] | 303 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 304 | $sta = $sparts[ 0 ];
|
---|
[17] | 305 | $run = $sparts[ 1 ];
|
---|
| 306 | $que = $sparts[ 2 ];
|
---|
[11] | 307 | break;
|
---|
| 308 | }
|
---|
[31] | 309 | case 'jetstream-local':
|
---|
| 310 | case 'jetstream':
|
---|
| 311 | {
|
---|
[32] | 312 | $host = "us3@js-169-137.jetstream-cloud.org";
|
---|
| 313 | // $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" |tail -1'`;
|
---|
| 314 | $qstat = `ssh $host '/home/us3/bin/clusstat |tail -n 1'`;
|
---|
[31] | 315 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 316 | $sta = $sparts[ 0 ];
|
---|
| 317 | $knts = $sparts[ 1 ];
|
---|
| 318 | $sparts = preg_split( '/\//', $knts );
|
---|
| 319 | $run = $sparts[ 0 ];
|
---|
[32] | 320 | $que = $sparts[ 2 ];
|
---|
| 321 | $tot = $sparts[ 3 ];
|
---|
[31] | 322 | if ( $sta == "" )
|
---|
| 323 | $sta = "down";
|
---|
| 324 | break;
|
---|
| 325 | }
|
---|
[39] | 326 | case 'taito-local':
|
---|
| 327 | {
|
---|
| 328 | $host = "rb_2001068_taito01@taito.csc.fi";
|
---|
| 329 | $qstat = `ssh -i /home/us3/.ssh/id_rsa_taito_robot $host '/homeappl/home/rb_2001068_taito01/scripts/cstat 2>&1'`;
|
---|
| 330 | $sparts = preg_split( '/\s+/', $qstat );
|
---|
| 331 | $tot = $sparts[ 1 ];
|
---|
| 332 | $run = '0';
|
---|
| 333 | $que = '0';
|
---|
| 334 | $sta = "up";
|
---|
| 335 | if ( $tot == '' || $tot == '0' )
|
---|
| 336 | {
|
---|
| 337 | $sta = "down";
|
---|
| 338 | }
|
---|
| 339 | else
|
---|
| 340 | {
|
---|
| 341 | $run = $sparts[ 3 ];
|
---|
| 342 | $que = $sparts[ 5 ];
|
---|
| 343 | }
|
---|
| 344 | break;
|
---|
| 345 | }
|
---|
| 346 |
|
---|
[11] | 347 | }
|
---|
[6] | 348 |
|
---|
[17] | 349 | if ( $sta == "" )
|
---|
| 350 | $sta = "down";
|
---|
| 351 |
|
---|
[11] | 352 | if ( $sta == "down" )
|
---|
| 353 | {
|
---|
| 354 | $que = "0";
|
---|
| 355 | $run = "0";
|
---|
| 356 | }
|
---|
| 357 |
|
---|
[36] | 358 | // Insure queued,running counts are numeric
|
---|
| 359 | $que_s = $que;
|
---|
| 360 | $run_s = $run;
|
---|
| 361 | $que_e = preg_replace( '/=/', "", $que_s );
|
---|
| 362 | $run_e = preg_replace( '/=/', "", $run_s );
|
---|
| 363 | $que = intval( $que_e );
|
---|
| 364 | $run = intval( $run_e );
|
---|
| 365 | if($que!=$que_s || $run!=$run_s)
|
---|
| 366 | echo "$self: *** que s,e,i $que_s $que_e $que run s,e,i $run_s $run_e $run\n";
|
---|
| 367 |
|
---|
| 368 | // Save cluster status values
|
---|
[6] | 369 | $a[ 'cluster' ] = $clname;
|
---|
| 370 | $a[ 'queued' ] = $que;
|
---|
| 371 | $a[ 'running' ] = $run;
|
---|
| 372 | $a[ 'status' ] = $sta;
|
---|
[32] | 373 | echo "$self: $clname $que $run $sta\n";
|
---|
[6] | 374 |
|
---|
| 375 | $data[] = $a;
|
---|
[11] | 376 |
|
---|
[31] | 377 | if ( $clname == 'alamo' ||
|
---|
| 378 | $clname == 'jacinto' ||
|
---|
| 379 | $clname == 'jetstream' )
|
---|
[11] | 380 | {
|
---|
| 381 | $a[ 'cluster' ] = $clname . "-local";
|
---|
| 382 | $data[] = $a;
|
---|
| 383 | }
|
---|
[6] | 384 | }
|
---|
| 385 | }
|
---|
| 386 |
|
---|
[1] | 387 | class XML_Array
|
---|
| 388 | {
|
---|
| 389 | var $_data = Array();
|
---|
| 390 | var $_name = Array();
|
---|
| 391 | var $_rep = Array();
|
---|
| 392 | var $_parser = 0;
|
---|
| 393 | var $_level = 0;
|
---|
| 394 | var $_index = 0;
|
---|
| 395 |
|
---|
| 396 | function XML_Array( &$data )
|
---|
| 397 | {
|
---|
| 398 | $this->_parser = xml_parser_create();
|
---|
| 399 |
|
---|
| 400 | xml_set_object ( $this->_parser, $this );
|
---|
| 401 | xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
|
---|
| 402 | xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
|
---|
| 403 | xml_set_character_data_handler( $this->_parser, "_cdata" );
|
---|
| 404 |
|
---|
| 405 | $this->_data = array();
|
---|
| 406 | $this->_level = 0;
|
---|
| 407 |
|
---|
| 408 | if ( ! xml_parse( $this->_parser, $data, true ) )
|
---|
| 409 | return false;
|
---|
| 410 |
|
---|
| 411 | xml_parser_free( $this->_parser );
|
---|
| 412 | }
|
---|
| 413 |
|
---|
| 414 | function & ReturnArray()
|
---|
| 415 | {
|
---|
| 416 | return $this->_data[ 0 ];
|
---|
| 417 | }
|
---|
| 418 |
|
---|
| 419 | function _startElement( $parser, $name, $attrs )
|
---|
| 420 | {
|
---|
| 421 | if ( $name == "resourceHealth" )
|
---|
| 422 | {
|
---|
[6] | 423 | ## $name .= $this->_index;
|
---|
[1] | 424 | $this->_index++;
|
---|
| 425 | }
|
---|
| 426 |
|
---|
| 427 | if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
|
---|
| 428 |
|
---|
| 429 | $this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
|
---|
| 430 | $this->_name[ $this->_level ] = $name;
|
---|
| 431 | $this->_level++;
|
---|
| 432 | }
|
---|
| 433 |
|
---|
| 434 | function _endElement( $parser, $name )
|
---|
| 435 | {
|
---|
| 436 | if ( isset( $this->_data[ $this->_level ] ) )
|
---|
| 437 | {
|
---|
| 438 | $this->_addElement( $this->_name[ $this->_level - 1 ],
|
---|
| 439 | $this->_data[ $this->_level - 1 ],
|
---|
| 440 | $this->_data[ $this->_level ]
|
---|
| 441 | );
|
---|
| 442 | }
|
---|
| 443 |
|
---|
| 444 | unset( $this->_data[ $this->_level ] );
|
---|
| 445 | $this->_level--;
|
---|
| 446 | $this->_rep[ $name ]++;
|
---|
| 447 | }
|
---|
| 448 |
|
---|
| 449 | function _cdata( $parser, $data )
|
---|
| 450 | {
|
---|
| 451 | if ( $this->_name[ $this->_level - 1 ] )
|
---|
| 452 | {
|
---|
| 453 | $this->_addElement( $this->_name[ $this->_level - 1 ],
|
---|
| 454 | $this->_data[ $this->_level - 1 ],
|
---|
| 455 | str_replace( array( ">", "<",""", "&" ),
|
---|
| 456 | array( ">" , "<" , '"' , "&" ),
|
---|
| 457 | $data
|
---|
| 458 | )
|
---|
| 459 | );
|
---|
| 460 | }
|
---|
| 461 | }
|
---|
| 462 |
|
---|
| 463 | function _addElement( &$name, &$start, $add = array() )
|
---|
| 464 | {
|
---|
| 465 | if ( ( sizeof( $add ) == 0 && is_array( $add ) ) || ! $add )
|
---|
| 466 | {
|
---|
| 467 | if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
|
---|
| 468 | $add = '';
|
---|
| 469 | }
|
---|
| 470 |
|
---|
| 471 | $update = &$start[ $name ];
|
---|
| 472 |
|
---|
| 473 | if ( is_array( $add) &&
|
---|
| 474 | is_array( $update ) ) $update += $add;
|
---|
| 475 | elseif ( is_array( $update ) ) return;
|
---|
| 476 | elseif ( is_array( $add ) ) $update = $add;
|
---|
| 477 | elseif ( $add ) $update .= $add;
|
---|
| 478 | }
|
---|
| 479 | }
|
---|
| 480 | ?>
|
---|