source: trunk/cluster_status.php

Last change on this file was 41, checked in by gegorbet, 6 years ago

gridctl mods for uslims.uleth.ca demeler3 cluster
File size: 13.4 KB
Line 
1<?php
2
3$us3bin = exec( "ls -d ~us3/lims/bin" );
4include "$us3bin/listen-config.php";
5
6if ( ! preg_match( "/_local/", $class_dir ) )
7{
8 $xml = get_data();
9
10 if ( $xml != "" )
11 parse( $xml );
12}
13
14$data = array();
15
16local_status();
17
18foreach ( $data as $item )
19{
20 if ( ! preg_match( "/error/", $item[ 'running' ] ) )
21 update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
22}
23
24//exit();
25exit(0);
26
27// Get the cluster status
28
29function get_data()
30{
31 global $self;
32 $url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
33
34 try
35 {
36 $post = new HttpRequest( $url, HttpRequest::METH_GET );
37 $http = $post->send();
38 $xml = $post->getResponseBody();
39 }
40 catch ( HttpException $e )
41 {
42// write_log( "$self: Cluster Status not available" );
43 return "";
44 }
45
46 return $xml;
47}
48
49// Parse the xml
50
51function parse( $xml )
52{
53 global $data;
54
55 $data = array();
56
57 $x = new XML_Array( $xml );
58 $d = $x->ReturnArray();
59
60 if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
61
62 foreach ( $d[ 'summaries' ] as $item )
63 {
64 $a = Array();
65
66
67 $a[ 'queued' ] = $item[ 'waitingJobs' ];
68 $a[ 'running' ] = $item[ 'runningJobs' ];
69
70 if ( $a[ 'queued' ] == "" || $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
71 if ( $a[ 'running' ] == "" || $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
72
73 $clusterParts = explode( ".", $item[ 'resourceId' ] );
74 $cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
75
76 if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
77 if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
78
79 $a[ 'cluster' ] = $cluster;
80
81 switch ( $item[ 'resourceStatus' ] )
82 {
83 case 'UP' :
84 $status = "up";
85 break;
86
87 case 'DOWN' :
88 $status = "down";
89 break;
90
91 case 'WARN' :
92 $status = "warn";
93 break;
94
95 case 'FAILED' :
96 default :
97 $status = "unknown";
98 break;
99 }
100
101 $a[ 'status' ] = $status;
102
103 $data[] = $a;
104 }
105}
106
107// Put it in the DB
108
109function update( $cluster, $queued, $status, $running )
110{
111 global $dbhost;
112 global $guser;
113 global $gpasswd;
114 global $gDB;
115 global $self;
116//echo " $cluster $queued, $status, $running\n";
117
118 $gfac_link = mysqli_connect( $dbhost, $guser, $gpasswd, $gDB );
119
120 if ( ! $gfac_link )
121 {
122 write_log( "$self: Could not connect to DB $gDB" );
123 echo "Could not connect to DB $gDB.\n";
124 exit();
125 }
126
127 $query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
128 $result = mysqli_query( $gfac_link, $query );
129
130 if ( ! $result )
131 {
132 write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
133 echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
134 exit();
135 }
136
137 $rows = mysqli_num_rows( $result );
138
139 if ( $rows == 0 ) // INSERT
140 {
141 $query = "INSERT INTO cluster_status SET " .
142 "cluster='$cluster', " .
143 "queued=$queued, " .
144 "running=$running, " .
145 "status='$status'";
146 }
147 else // UPDATE
148 {
149 $query = "UPDATE cluster_status SET " .
150 "queued=$queued, " .
151 "running=$running, " .
152 "status='$status' " .
153 "WHERE cluster='$cluster'";
154 }
155
156 $result = mysqli_query( $gfac_link, $query );
157
158 if ( ! $result )
159 {
160 write_log( "$self: Query failed $query - " . mysqli_error( $gfac_link ) );
161 echo "$self: Query failed $query - " . mysqli_error( $gfac_link ) . "\n";
162 }
163}
164
165// Get local cluster status
166
167function local_status()
168{
169 global $self;
170 global $data;
171 global $dbhost;
172 global $org_domain;
173 global $class_dir;
174
175 if ( preg_match( "/_local/", $class_dir ) )
176 {
177 if ( preg_match( "/attlocal/", $org_domain ) )
178 $clusters = array( "us3iab-devel" );
179 else
180 $clusters = array( "us3iab-node0", "demeler3-local", "taito-local" );
181 }
182 else
183 {
184 $clusters = array( "lonestar5", "comet",
185 "stampede2", "jetstream" );
186 }
187
188 foreach ( $clusters as $clname )
189 {
190 $a = Array();
191//echo "$self: clname=$clname\n";
192
193 switch( $clname )
194 {
195 case 'us3iab-node0':
196 case 'us3iab-node1':
197 case 'us3iab-devel':
198 {
199 $qstat = `/usr/bin/qstat -B 2>&1|tail -1`;
200
201 $sparts = preg_split( '/\s+/', $qstat );
202 $que = $sparts[ 3 ];
203 $run = $sparts[ 4 ];
204 $sta = $sparts[ 10 ];
205 if ( $sta == "Active" )
206 $sta = "up";
207 else
208 $sta = "down";
209 break;
210 }
211 case 'alamo-local':
212 case 'alamo':
213 {
214 $host = "us3@alamo.uthscsa.edu";
215 $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
216 $sparts = preg_split( '/\s+/', $qstat );
217 $que = $sparts[ 3 ];
218 $run = $sparts[ 4 ];
219 $sta = $sparts[ 10 ];
220 if ( $sta == "Active" )
221 $sta = "up";
222 else
223 $sta = "down";
224 break;
225 }
226
227 case 'demeler3-local':
228 {
229 $host = "us3@demeler3.uleth.ca";
230 $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
231 $sparts = preg_split( '/\s+/', $qstat );
232 $que = $sparts[ 3 ];
233 $run = $sparts[ 4 ];
234 $sta = $sparts[ 10 ];
235 if ( $sta == "Active" )
236 $sta = "up";
237 else
238 $sta = "down";
239 break;
240 }
241
242 case 'jacinto':
243 {
244 $host = "us3@jacinto.uthscsa.edu";
245 $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
246 $sparts = preg_split( '/\s+/', $qstat );
247 $que = $sparts[ 3 ];
248 $run = $sparts[ 4 ];
249 $sta = $sparts[ 9 ];
250 if ( $sta == "Active" )
251 $sta = "up";
252 else
253 $sta = "down";
254 break;
255 }
256 case 'stampede2':
257 {
258 $host = "us3@stampede2.tacc.utexas.edu";
259 $qstat = `ssh $host '~us3/scripts/clusstat skx-normal 2>/dev/null'`;
260 $sparts = preg_split( '/\s+/', $qstat );
261 $tot = $sparts[ 2 ];
262 $run = $sparts[ 5 ];
263 $que = $sparts[ 8 ];
264 $sta = "up";
265 if ( $tot == '' || $tot == '0' )
266 $sta = "down";
267 break;
268 }
269 case 'lonestar5':
270 {
271 $host = "us3@ls5.tacc.utexas.edu";
272 $qstat = `ssh $host '/opt/apps/tacc/bin/showq 2>/dev/null|grep "Total Jobs"'`;
273 $sparts = preg_split( '/\s+/', $qstat );
274 $tot = $sparts[ 2 ];
275 $run = '0';
276 $que = '0';
277 $sta = "up";
278 if ( $tot == '' || $tot == '0' )
279 {
280 $sta = "down";
281 }
282 else
283 {
284 $run = $sparts[ 5 ];
285 $que = $sparts[ 8 ];
286// $que = $sparts[ 11 ];
287 }
288 break;
289 }
290 case 'comet':
291 {
292 $host = "us3@comet.sdsc.edu";
293 //$qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" |tail -1'`;
294 $qstat = `ssh $host '/home/us3/scripts/cstat 2>&1'`;
295 $sparts = preg_split( '/\s+/', $qstat );
296 $tot = $sparts[ 1 ];
297 $run = '0';
298 $que = '0';
299 $sta = "up";
300 if ( $tot == '' || $tot == '0' )
301 {
302 $sta = "down";
303 }
304 else
305 {
306 $run = $sparts[ 3 ];
307 $que = $sparts[ 5 ];
308 if ( $run == '0' && $que == '0' )
309 {
310 $sta = "down";
311 }
312 }
313 break;
314 }
315 case 'jureca':
316 {
317 $host = "swus1@jureca.fz-juelich.de";
318 $qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
319 $sparts = preg_split( '/\s+/', $qstat );
320 $sta = $sparts[ 0 ];
321 $run = $sparts[ 1 ];
322 $que = $sparts[ 2 ];
323 break;
324 }
325 case 'jetstream-local':
326 case 'jetstream':
327 {
328 $host = "us3@js-169-137.jetstream-cloud.org";
329// $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" |tail -1'`;
330 $qstat = `ssh $host '/home/us3/bin/clusstat |tail -n 1'`;
331 $sparts = preg_split( '/\s+/', $qstat );
332 $sta = $sparts[ 0 ];
333 $knts = $sparts[ 1 ];
334 $sparts = preg_split( '/\//', $knts );
335 $run = $sparts[ 0 ];
336 $que = $sparts[ 2 ];
337 $tot = $sparts[ 3 ];
338 if ( $sta == "" )
339 $sta = "down";
340 break;
341 }
342 case 'taito-local':
343 {
344 $host = "rb_2001068_taito01@taito.csc.fi";
345 $qstat = `ssh -i /home/us3/.ssh/id_rsa_taito_robot $host '/homeappl/home/rb_2001068_taito01/scripts/cstat 2>&1'`;
346 $sparts = preg_split( '/\s+/', $qstat );
347 $tot = $sparts[ 1 ];
348 $run = '0';
349 $que = '0';
350 $sta = "up";
351 if ( $tot == '' || $tot == '0' )
352 {
353 $sta = "down";
354 }
355 else
356 {
357 $run = $sparts[ 3 ];
358 $que = $sparts[ 5 ];
359 }
360 break;
361 }
362
363 }
364
365 if ( $sta == "" )
366 $sta = "down";
367
368 if ( $sta == "down" )
369 {
370 $que = "0";
371 $run = "0";
372 }
373
374 // Insure queued,running counts are numeric
375 $que_s = $que;
376 $run_s = $run;
377 $que_e = preg_replace( '/=/', "", $que_s );
378 $run_e = preg_replace( '/=/', "", $run_s );
379 $que = intval( $que_e );
380 $run = intval( $run_e );
381if($que!=$que_s || $run!=$run_s)
382 echo "$self: *** que s,e,i $que_s $que_e $que run s,e,i $run_s $run_e $run\n";
383
384 // Save cluster status values
385 $a[ 'cluster' ] = $clname;
386 $a[ 'queued' ] = $que;
387 $a[ 'running' ] = $run;
388 $a[ 'status' ] = $sta;
389echo "$self: $clname $que $run $sta\n";
390
391 $data[] = $a;
392
393 if ( $clname == 'alamo' ||
394 $clname == 'jacinto' ||
395 $clname == 'jetstream' )
396 {
397 $a[ 'cluster' ] = $clname . "-local";
398 $data[] = $a;
399 }
400 }
401}
402
403class XML_Array
404{
405 var $_data = Array();
406 var $_name = Array();
407 var $_rep = Array();
408 var $_parser = 0;
409 var $_level = 0;
410 var $_index = 0;
411
412 function XML_Array( &$data )
413 {
414 $this->_parser = xml_parser_create();
415
416 xml_set_object ( $this->_parser, $this );
417 xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
418 xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
419 xml_set_character_data_handler( $this->_parser, "_cdata" );
420
421 $this->_data = array();
422 $this->_level = 0;
423
424 if ( ! xml_parse( $this->_parser, $data, true ) )
425 return false;
426
427 xml_parser_free( $this->_parser );
428 }
429
430 function & ReturnArray()
431 {
432 return $this->_data[ 0 ];
433 }
434
435 function _startElement( $parser, $name, $attrs )
436 {
437 if ( $name == "resourceHealth" )
438 {
439## $name .= $this->_index;
440 $this->_index++;
441 }
442
443 if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
444
445 $this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
446 $this->_name[ $this->_level ] = $name;
447 $this->_level++;
448 }
449
450 function _endElement( $parser, $name )
451 {
452 if ( isset( $this->_data[ $this->_level ] ) )
453 {
454 $this->_addElement( $this->_name[ $this->_level - 1 ],
455 $this->_data[ $this->_level - 1 ],
456 $this->_data[ $this->_level ]
457 );
458 }
459
460 unset( $this->_data[ $this->_level ] );
461 $this->_level--;
462 $this->_rep[ $name ]++;
463 }
464
465 function _cdata( $parser, $data )
466 {
467 if ( $this->_name[ $this->_level - 1 ] )
468 {
469 $this->_addElement( $this->_name[ $this->_level - 1 ],
470 $this->_data[ $this->_level - 1 ],
471 str_replace( array( "&gt;", "&lt;","&quot;", "&amp;" ),
472 array( ">" , "<" , '"' , "&" ),
473 $data
474 )
475 );
476 }
477 }
478
479 function _addElement( &$name, &$start, $add = array() )
480 {
481 if ( ( sizeof( $add ) == 0 && is_array( $add ) ) || ! $add )
482 {
483 if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
484 $add = '';
485 }
486
487 $update = &$start[ $name ];
488
489 if ( is_array( $add) &&
490 is_array( $update ) ) $update += $add;
491 elseif ( is_array( $update ) ) return;
492 elseif ( is_array( $add ) ) $update = $add;
493 elseif ( $add ) $update .= $add;
494 }
495}
496?>
Note: See TracBrowser for help on using the repository browser.