source: trunk/cluster_status.php@ 26

Last change on this file since 26 was 26, checked in by gegorbet, 7 years ago

various minor fixes/mods to gridctl
File size: 11.2 KB
Line 
1<?php
2
3$us3bin = exec( "ls -d ~us3/lims/bin" );
4include "$us3bin/listen-config.php";
5
6if ( ! preg_match( "/_local/", $class_dir ) )
7{
8 $xml = get_data();
9
10 if ( $xml != "" )
11 parse( $xml );
12}
13
14$data = array();
15
16local_status();
17
18foreach ( $data as $item )
19{
20 if ( ! preg_match( "/error/", $item[ 'running' ] ) )
21 update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
22}
23
24exit();
25
26// Get the cluster status
27
28function get_data()
29{
30 global $self;
31 $url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
32
33 try
34 {
35 $post = new HttpRequest( $url, HttpRequest::METH_GET );
36 $http = $post->send();
37 $xml = $post->getResponseBody();
38 }
39 catch ( HttpException $e )
40 {
41// write_log( "$self: Cluster Status not available" );
42 return "";
43 }
44
45 return $xml;
46}
47
48// Parse the xml
49
50function parse( $xml )
51{
52 global $data;
53
54 $data = array();
55
56 $x = new XML_Array( $xml );
57 $d = $x->ReturnArray();
58
59 if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
60
61 foreach ( $d[ 'summaries' ] as $item )
62 {
63 $a = Array();
64
65
66 $a[ 'queued' ] = $item[ 'waitingJobs' ];
67 $a[ 'running' ] = $item[ 'runningJobs' ];
68
69 if ( $a[ 'queued' ] == "" || $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
70 if ( $a[ 'running' ] == "" || $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
71
72 $clusterParts = explode( ".", $item[ 'resourceId' ] );
73 $cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
74
75 if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
76 if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
77
78 $a[ 'cluster' ] = $cluster;
79
80 switch ( $item[ 'resourceStatus' ] )
81 {
82 case 'UP' :
83 $status = "up";
84 break;
85
86 case 'DOWN' :
87 $status = "down";
88 break;
89
90 case 'WARN' :
91 $status = "warn";
92 break;
93
94 case 'FAILED' :
95 default :
96 $status = "unknown";
97 break;
98 }
99
100 $a[ 'status' ] = $status;
101
102 $data[] = $a;
103 }
104}
105
106// Put it in the DB
107
108function update( $cluster, $queued, $status, $running )
109{
110 global $dbhost;
111 global $guser;
112 global $gpasswd;
113 global $gDB;
114 global $self;
115//echo " $cluster $queued, $status, $running\n";
116
117 $gfac_link = mysql_connect( $dbhost, $guser, $gpasswd );
118 $result = mysql_select_db( $gDB, $gfac_link );
119
120 if ( ! $result )
121 {
122 write_log( "$self: Could not connect to DB $gDB" );
123 echo "Could not connect to DB $gDB.\n";
124 exit();
125 }
126
127 $query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
128 $result = mysql_query( $query, $gfac_link );
129
130 if ( ! $result )
131 {
132 write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
133 echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
134 exit();
135 }
136
137 $rows = mysql_num_rows( $result );
138
139 if ( $rows == 0 ) // INSERT
140 {
141 $query = "INSERT INTO cluster_status SET " .
142 "cluster='$cluster', " .
143 "queued=$queued, " .
144 "running=$running, " .
145 "status='$status'";
146 }
147 else // UPDATE
148 {
149 $query = "UPDATE cluster_status SET " .
150 "queued=$queued, " .
151 "running=$running, " .
152 "status='$status' " .
153 "WHERE cluster='$cluster'";
154 }
155
156 $result = mysql_query( $query, $gfac_link );
157
158 if ( ! $result )
159 {
160 write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
161 echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
162 }
163}
164
165// Get local cluster status
166
167function local_status()
168{
169 global $self;
170 global $data;
171 global $dbhost;
172 global $org_domain;
173 global $class_dir;
174
175 if ( preg_match( "/_local/", $class_dir ) )
176 {
177 if ( preg_match( "/attlocal/", $org_domain ) )
178 $clusters = array( "us3iab-devel" );
179 else
180 $clusters = array( "us3iab-node0" );
181 }
182 else
183 {
184 $clusters = array( "alamo", "lonestar5", "stampede", "comet",
185 "gordon", "jureca", "jacinto" );
186 }
187
188 foreach ( $clusters as $clname )
189 {
190 $a = Array();
191//echo "$self: clname=$clname\n";
192
193 switch( $clname )
194 {
195 case 'us3iab-node0':
196 case 'us3iab-node1':
197 case 'us3iab-devel':
198 {
199 $qstat = `/usr/bin/qstat -B 2>&1|tail -1`;
200
201 $sparts = preg_split( '/\s+/', $qstat );
202 $que = $sparts[ 3 ];
203 $run = $sparts[ 4 ];
204 $sta = $sparts[ 10 ];
205 if ( $sta == "Active" )
206 $sta = "up";
207 else
208 $sta = "down";
209 break;
210 }
211 case 'alamo':
212 {
213 $host = "us3@alamo.uthscsa.edu";
214 $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
215 $sparts = preg_split( '/\s+/', $qstat );
216 $que = $sparts[ 3 ];
217 $run = $sparts[ 4 ];
218 $sta = $sparts[ 10 ];
219 if ( $sta == "Active" )
220 $sta = "up";
221 else
222 $sta = "down";
223 break;
224 }
225 case 'jacinto':
226 {
227 $host = "us3@jacinto.uthscsa.edu";
228 $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
229 $sparts = preg_split( '/\s+/', $qstat );
230 $que = $sparts[ 3 ];
231 $run = $sparts[ 4 ];
232 $sta = $sparts[ 9 ];
233 if ( $sta == "Active" )
234 $sta = "up";
235 else
236 $sta = "down";
237 break;
238 }
239 case 'stampede':
240 {
241 $host = "us3@stampede.tacc.utexas.edu";
242 $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
243 $sparts = preg_split( '/\s+/', $qstat );
244 $tot = $sparts[ 2 ];
245 $run = $sparts[ 5 ];
246 $que = $sparts[ 8 ];
247 $sta = "up";
248 if ( $tot == '' || $tot == '0' )
249 $sta = "down";
250 break;
251 }
252 case 'lonestar5':
253 {
254 $host = "us3@ls5.tacc.utexas.edu";
255 $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
256 $sparts = preg_split( '/\s+/', $qstat );
257 $tot = $sparts[ 2 ];
258 $run = '0';
259 $que = '0';
260 $sta = "up";
261 if ( $tot == '' || $tot == '0' )
262 {
263 $sta = "down";
264 }
265 else
266 {
267 $run = $sparts[ 5 ];
268 $que = $sparts[ 8 ];
269 }
270 break;
271 }
272 case 'comet':
273 {
274 $host = "us3@comet.sdsc.edu";
275 $qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" |tail -1'`;
276 $sparts = preg_split( '/\s+/', $qstat );
277 $sta = $sparts[ 0 ];
278 $knts = $sparts[ 1 ];
279 $sparts = preg_split( '/\//', $knts );
280 $run = $sparts[ 0 ];
281 $que = $sparts[ 1 ];
282 if ( $sta == "" )
283 $sta = "down";
284 break;
285 }
286 case 'gordon':
287 {
288 $host = "us3@gordon.sdsc.edu";
289 $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
290 $sparts = preg_split( '/\s+/', $qstat );
291 $que = $sparts[ 3 ];
292 $run = $sparts[ 4 ];
293 $sta = $sparts[ 10 ];
294 if ( $sta == "Active" )
295 $sta = "up";
296 else
297 $sta = "down";
298 break;
299 }
300 case 'jureca':
301 {
302 $host = "swus1@jureca.fz-juelich.de";
303 $qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
304 $sparts = preg_split( '/\s+/', $qstat );
305 $sta = $sparts[ 0 ];
306 $run = $sparts[ 1 ];
307 $que = $sparts[ 2 ];
308 break;
309 }
310 }
311
312 if ( $sta == "" )
313 $sta = "down";
314
315 if ( $sta == "down" )
316 {
317 $que = "0";
318 $run = "0";
319 }
320
321 $a[ 'cluster' ] = $clname;
322 $a[ 'queued' ] = $que;
323 $a[ 'running' ] = $run;
324 $a[ 'status' ] = $sta;
325//echo "$self: $clname $que $run $sta\n";
326
327 $data[] = $a;
328
329 if ( $clname == 'alamo' || $clname == 'jacinto' )
330 {
331 $a[ 'cluster' ] = $clname . "-local";
332 $data[] = $a;
333 }
334 }
335}
336
337class XML_Array
338{
339 var $_data = Array();
340 var $_name = Array();
341 var $_rep = Array();
342 var $_parser = 0;
343 var $_level = 0;
344 var $_index = 0;
345
346 function XML_Array( &$data )
347 {
348 $this->_parser = xml_parser_create();
349
350 xml_set_object ( $this->_parser, $this );
351 xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
352 xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
353 xml_set_character_data_handler( $this->_parser, "_cdata" );
354
355 $this->_data = array();
356 $this->_level = 0;
357
358 if ( ! xml_parse( $this->_parser, $data, true ) )
359 return false;
360
361 xml_parser_free( $this->_parser );
362 }
363
364 function & ReturnArray()
365 {
366 return $this->_data[ 0 ];
367 }
368
369 function _startElement( $parser, $name, $attrs )
370 {
371 if ( $name == "resourceHealth" )
372 {
373## $name .= $this->_index;
374 $this->_index++;
375 }
376
377 if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
378
379 $this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
380 $this->_name[ $this->_level ] = $name;
381 $this->_level++;
382 }
383
384 function _endElement( $parser, $name )
385 {
386 if ( isset( $this->_data[ $this->_level ] ) )
387 {
388 $this->_addElement( $this->_name[ $this->_level - 1 ],
389 $this->_data[ $this->_level - 1 ],
390 $this->_data[ $this->_level ]
391 );
392 }
393
394 unset( $this->_data[ $this->_level ] );
395 $this->_level--;
396 $this->_rep[ $name ]++;
397 }
398
399 function _cdata( $parser, $data )
400 {
401 if ( $this->_name[ $this->_level - 1 ] )
402 {
403 $this->_addElement( $this->_name[ $this->_level - 1 ],
404 $this->_data[ $this->_level - 1 ],
405 str_replace( array( "&gt;", "&lt;","&quot;", "&amp;" ),
406 array( ">" , "<" , '"' , "&" ),
407 $data
408 )
409 );
410 }
411 }
412
413 function _addElement( &$name, &$start, $add = array() )
414 {
415 if ( ( sizeof( $add ) == 0 && is_array( $add ) ) || ! $add )
416 {
417 if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
418 $add = '';
419 }
420
421 $update = &$start[ $name ];
422
423 if ( is_array( $add) &&
424 is_array( $update ) ) $update += $add;
425 elseif ( is_array( $update ) ) return;
426 elseif ( is_array( $add ) ) $update = $add;
427 elseif ( $add ) $update .= $add;
428 }
429}
430?>
Note: See TracBrowser for help on using the repository browser.