source: trunk/cluster_status.php@ 32

Last change on this file since 32 was 32, checked in by gegorbet, 6 years ago

gridctl updates, mostly for jetstream and 4.0
File size: 12.5 KB
Line 
1<?php
2
3$us3bin = exec( "ls -d ~us3/lims/bin" );
4include "$us3bin/listen-config.php";
5
6if ( ! preg_match( "/_local/", $class_dir ) )
7{
8 $xml = get_data();
9
10 if ( $xml != "" )
11 parse( $xml );
12}
13
14$data = array();
15
16local_status();
17
18foreach ( $data as $item )
19{
20 if ( ! preg_match( "/error/", $item[ 'running' ] ) )
21 update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
22}
23
24exit();
25
26// Get the cluster status
27
28function get_data()
29{
30 global $self;
31 $url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
32
33 try
34 {
35 $post = new HttpRequest( $url, HttpRequest::METH_GET );
36 $http = $post->send();
37 $xml = $post->getResponseBody();
38 }
39 catch ( HttpException $e )
40 {
41// write_log( "$self: Cluster Status not available" );
42 return "";
43 }
44
45 return $xml;
46}
47
48// Parse the xml
49
50function parse( $xml )
51{
52 global $data;
53
54 $data = array();
55
56 $x = new XML_Array( $xml );
57 $d = $x->ReturnArray();
58
59 if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
60
61 foreach ( $d[ 'summaries' ] as $item )
62 {
63 $a = Array();
64
65
66 $a[ 'queued' ] = $item[ 'waitingJobs' ];
67 $a[ 'running' ] = $item[ 'runningJobs' ];
68
69 if ( $a[ 'queued' ] == "" || $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
70 if ( $a[ 'running' ] == "" || $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
71
72 $clusterParts = explode( ".", $item[ 'resourceId' ] );
73 $cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
74
75 if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
76 if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
77
78 $a[ 'cluster' ] = $cluster;
79
80 switch ( $item[ 'resourceStatus' ] )
81 {
82 case 'UP' :
83 $status = "up";
84 break;
85
86 case 'DOWN' :
87 $status = "down";
88 break;
89
90 case 'WARN' :
91 $status = "warn";
92 break;
93
94 case 'FAILED' :
95 default :
96 $status = "unknown";
97 break;
98 }
99
100 $a[ 'status' ] = $status;
101
102 $data[] = $a;
103 }
104}
105
106// Put it in the DB
107
108function update( $cluster, $queued, $status, $running )
109{
110 global $dbhost;
111 global $guser;
112 global $gpasswd;
113 global $gDB;
114 global $self;
115//echo " $cluster $queued, $status, $running\n";
116
117 $gfac_link = mysql_connect( $dbhost, $guser, $gpasswd );
118 $result = mysql_select_db( $gDB, $gfac_link );
119
120 if ( ! $result )
121 {
122 write_log( "$self: Could not connect to DB $gDB" );
123 echo "Could not connect to DB $gDB.\n";
124 exit();
125 }
126
127 $query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
128 $result = mysql_query( $query, $gfac_link );
129
130 if ( ! $result )
131 {
132 write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
133 echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
134 exit();
135 }
136
137 $rows = mysql_num_rows( $result );
138
139 if ( $rows == 0 ) // INSERT
140 {
141 $query = "INSERT INTO cluster_status SET " .
142 "cluster='$cluster', " .
143 "queued=$queued, " .
144 "running=$running, " .
145 "status='$status'";
146 }
147 else // UPDATE
148 {
149 $query = "UPDATE cluster_status SET " .
150 "queued=$queued, " .
151 "running=$running, " .
152 "status='$status' " .
153 "WHERE cluster='$cluster'";
154 }
155
156 $result = mysql_query( $query, $gfac_link );
157
158 if ( ! $result )
159 {
160 write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
161 echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
162 }
163}
164
165// Get local cluster status
166
167function local_status()
168{
169 global $self;
170 global $data;
171 global $dbhost;
172 global $org_domain;
173 global $class_dir;
174
175 if ( preg_match( "/_local/", $class_dir ) )
176 {
177 if ( preg_match( "/attlocal/", $org_domain ) )
178 $clusters = array( "us3iab-devel", "alamo-local" );
179 else
180 $clusters = array( "us3iab-node0" );
181 }
182 else
183 {
184 $clusters = array( "alamo", "lonestar5", "stampede", "comet",
185 "stampede2-b", "jetstream", "jureca", "jacinto-b" );
186 }
187
188 foreach ( $clusters as $clname )
189 {
190 $a = Array();
191//echo "$self: clname=$clname\n";
192
193 switch( $clname )
194 {
195 case 'us3iab-node0':
196 case 'us3iab-node1':
197 case 'us3iab-devel':
198 {
199 $qstat = `/usr/bin/qstat -B 2>&1|tail -1`;
200
201 $sparts = preg_split( '/\s+/', $qstat );
202 $que = $sparts[ 3 ];
203 $run = $sparts[ 4 ];
204 $sta = $sparts[ 10 ];
205 if ( $sta == "Active" )
206 $sta = "up";
207 else
208 $sta = "down";
209 break;
210 }
211 case 'alamo-local':
212 case 'alamo':
213 {
214 $host = "us3@alamo.uthscsa.edu";
215 $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
216 $sparts = preg_split( '/\s+/', $qstat );
217 $que = $sparts[ 3 ];
218 $run = $sparts[ 4 ];
219 $sta = $sparts[ 10 ];
220 if ( $sta == "Active" )
221 $sta = "up";
222 else
223 $sta = "down";
224 break;
225 }
226 case 'jacinto':
227 {
228 $host = "us3@jacinto.uthscsa.edu";
229 $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
230 $sparts = preg_split( '/\s+/', $qstat );
231 $que = $sparts[ 3 ];
232 $run = $sparts[ 4 ];
233 $sta = $sparts[ 9 ];
234 if ( $sta == "Active" )
235 $sta = "up";
236 else
237 $sta = "down";
238 break;
239 }
240 case 'stampede':
241 {
242 $host = "us3@stampede.tacc.utexas.edu";
243 $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
244 $sparts = preg_split( '/\s+/', $qstat );
245 $tot = $sparts[ 2 ];
246 $run = $sparts[ 5 ];
247 $que = $sparts[ 8 ];
248 $sta = "up";
249 if ( $tot == '' || $tot == '0' )
250 $sta = "down";
251 break;
252 }
253 case 'stampede2':
254 {
255 $host = "us3@stampede2.tacc.utexas.edu";
256 $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
257 $sparts = preg_split( '/\s+/', $qstat );
258 $tot = $sparts[ 2 ];
259 $run = $sparts[ 5 ];
260 $que = $sparts[ 8 ];
261 $sta = "up";
262 if ( $tot == '' || $tot == '0' )
263 $sta = "down";
264 break;
265 }
266 case 'lonestar5':
267 {
268 $host = "us3@ls5.tacc.utexas.edu";
269 $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
270 $sparts = preg_split( '/\s+/', $qstat );
271 $tot = $sparts[ 2 ];
272 $run = '0';
273 $que = '0';
274 $sta = "up";
275 if ( $tot == '' || $tot == '0' )
276 {
277 $sta = "down";
278 }
279 else
280 {
281 $run = $sparts[ 5 ];
282// $que = $sparts[ 8 ];
283 $que = $sparts[ 11 ];
284 }
285 break;
286 }
287 case 'comet':
288 {
289 $host = "us3@comet.sdsc.edu";
290 $qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" |tail -1'`;
291 $sparts = preg_split( '/\s+/', $qstat );
292 $sta = $sparts[ 0 ];
293 $knts = $sparts[ 1 ];
294 $sparts = preg_split( '/\//', $knts );
295 $run = $sparts[ 0 ];
296 $que = $sparts[ 1 ];
297 if ( $sta == "" )
298 $sta = "down";
299 break;
300 }
301 case 'gordon':
302 {
303 $host = "us3@gordon.sdsc.edu";
304 $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
305 $sparts = preg_split( '/\s+/', $qstat );
306 $que = $sparts[ 3 ];
307 $run = $sparts[ 4 ];
308 $sta = $sparts[ 10 ];
309 if ( $sta == "Active" )
310 $sta = "up";
311 else
312 $sta = "down";
313 break;
314 }
315 case 'jureca':
316 {
317 $host = "swus1@jureca.fz-juelich.de";
318 $qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
319 $sparts = preg_split( '/\s+/', $qstat );
320 $sta = $sparts[ 0 ];
321 $run = $sparts[ 1 ];
322 $que = $sparts[ 2 ];
323 break;
324 }
325 case 'jetstream-local':
326 case 'jetstream':
327 {
328 $host = "us3@js-169-137.jetstream-cloud.org";
329// $qstat = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" |tail -1'`;
330 $qstat = `ssh $host '/home/us3/bin/clusstat |tail -n 1'`;
331 $sparts = preg_split( '/\s+/', $qstat );
332 $sta = $sparts[ 0 ];
333 $knts = $sparts[ 1 ];
334 $sparts = preg_split( '/\//', $knts );
335 $run = $sparts[ 0 ];
336 $que = $sparts[ 2 ];
337 $tot = $sparts[ 3 ];
338 if ( $sta == "" )
339 $sta = "down";
340 break;
341 }
342 }
343
344 if ( $sta == "" )
345 $sta = "down";
346
347 if ( $sta == "down" )
348 {
349 $que = "0";
350 $run = "0";
351 }
352
353 $a[ 'cluster' ] = $clname;
354 $a[ 'queued' ] = $que;
355 $a[ 'running' ] = $run;
356 $a[ 'status' ] = $sta;
357echo "$self: $clname $que $run $sta\n";
358
359 $data[] = $a;
360
361 if ( $clname == 'alamo' ||
362 $clname == 'jacinto' ||
363 $clname == 'jetstream' )
364 {
365 $a[ 'cluster' ] = $clname . "-local";
366 $data[] = $a;
367 }
368 }
369}
370
371class XML_Array
372{
373 var $_data = Array();
374 var $_name = Array();
375 var $_rep = Array();
376 var $_parser = 0;
377 var $_level = 0;
378 var $_index = 0;
379
380 function XML_Array( &$data )
381 {
382 $this->_parser = xml_parser_create();
383
384 xml_set_object ( $this->_parser, $this );
385 xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
386 xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
387 xml_set_character_data_handler( $this->_parser, "_cdata" );
388
389 $this->_data = array();
390 $this->_level = 0;
391
392 if ( ! xml_parse( $this->_parser, $data, true ) )
393 return false;
394
395 xml_parser_free( $this->_parser );
396 }
397
398 function & ReturnArray()
399 {
400 return $this->_data[ 0 ];
401 }
402
403 function _startElement( $parser, $name, $attrs )
404 {
405 if ( $name == "resourceHealth" )
406 {
407## $name .= $this->_index;
408 $this->_index++;
409 }
410
411 if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
412
413 $this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
414 $this->_name[ $this->_level ] = $name;
415 $this->_level++;
416 }
417
418 function _endElement( $parser, $name )
419 {
420 if ( isset( $this->_data[ $this->_level ] ) )
421 {
422 $this->_addElement( $this->_name[ $this->_level - 1 ],
423 $this->_data[ $this->_level - 1 ],
424 $this->_data[ $this->_level ]
425 );
426 }
427
428 unset( $this->_data[ $this->_level ] );
429 $this->_level--;
430 $this->_rep[ $name ]++;
431 }
432
433 function _cdata( $parser, $data )
434 {
435 if ( $this->_name[ $this->_level - 1 ] )
436 {
437 $this->_addElement( $this->_name[ $this->_level - 1 ],
438 $this->_data[ $this->_level - 1 ],
439 str_replace( array( "&gt;", "&lt;","&quot;", "&amp;" ),
440 array( ">" , "<" , '"' , "&" ),
441 $data
442 )
443 );
444 }
445 }
446
447 function _addElement( &$name, &$start, $add = array() )
448 {
449 if ( ( sizeof( $add ) == 0 && is_array( $add ) ) || ! $add )
450 {
451 if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
452 $add = '';
453 }
454
455 $update = &$start[ $name ];
456
457 if ( is_array( $add) &&
458 is_array( $update ) ) $update += $add;
459 elseif ( is_array( $update ) ) return;
460 elseif ( is_array( $add ) ) $update = $add;
461 elseif ( $add ) $update .= $add;
462 }
463}
464?>
Note: See TracBrowser for help on using the repository browser.