source: trunk/cluster_status.php@ 28

Last change on this file since 28 was 28, checked in by gegorbet, 7 years ago

fixes for alamo-local and airavata EXECUTING status
File size: 11.3 KB
Line 
1<?php
2
3$us3bin = exec( "ls -d ~us3/lims/bin" );
4include "$us3bin/listen-config.php";
5
6if ( ! preg_match( "/_local/", $class_dir ) )
7{
8 $xml = get_data();
9
10 if ( $xml != "" )
11 parse( $xml );
12}
13
14$data = array();
15
16local_status();
17
18foreach ( $data as $item )
19{
20 if ( ! preg_match( "/error/", $item[ 'running' ] ) )
21 update( $item[ 'cluster' ], $item[ 'queued' ], $item[ 'status' ], $item[ 'running' ] );
22}
23
24exit();
25
26// Get the cluster status
27
28function get_data()
29{
30 global $self;
31 $url = "http://community.ucs.indiana.edu:19444/orps-service/XML/gateway/ultrascan";
32
33 try
34 {
35 $post = new HttpRequest( $url, HttpRequest::METH_GET );
36 $http = $post->send();
37 $xml = $post->getResponseBody();
38 }
39 catch ( HttpException $e )
40 {
41// write_log( "$self: Cluster Status not available" );
42 return "";
43 }
44
45 return $xml;
46}
47
48// Parse the xml
49
50function parse( $xml )
51{
52 global $data;
53
54 $data = array();
55
56 $x = new XML_Array( $xml );
57 $d = $x->ReturnArray();
58
59 if ( ! isset( $d[ 'summaries' ] ) ) exit(); // Bad input
60
61 foreach ( $d[ 'summaries' ] as $item )
62 {
63 $a = Array();
64
65
66 $a[ 'queued' ] = $item[ 'waitingJobs' ];
67 $a[ 'running' ] = $item[ 'runningJobs' ];
68
69 if ( $a[ 'queued' ] == "" || $a[ 'queued' ] < 0 ) $a[ 'queued' ] = 0;
70 if ( $a[ 'running' ] == "" || $a[ 'running' ] < 0 ) $a[ 'running' ] = 0;
71
72 $clusterParts = explode( ".", $item[ 'resourceId' ] );
73 $cluster = preg_replace( '/\d+$/', "", $clusterParts[ 0 ] );
74
75 if ( $cluster == 'uthscsa-bcf' ) $cluster = 'bcf';
76 if ( $cluster == 'uthscsa-alamo' ) $cluster = 'alamo';
77
78 $a[ 'cluster' ] = $cluster;
79
80 switch ( $item[ 'resourceStatus' ] )
81 {
82 case 'UP' :
83 $status = "up";
84 break;
85
86 case 'DOWN' :
87 $status = "down";
88 break;
89
90 case 'WARN' :
91 $status = "warn";
92 break;
93
94 case 'FAILED' :
95 default :
96 $status = "unknown";
97 break;
98 }
99
100 $a[ 'status' ] = $status;
101
102 $data[] = $a;
103 }
104}
105
106// Put it in the DB
107
108function update( $cluster, $queued, $status, $running )
109{
110 global $dbhost;
111 global $guser;
112 global $gpasswd;
113 global $gDB;
114 global $self;
115//echo " $cluster $queued, $status, $running\n";
116
117 $gfac_link = mysql_connect( $dbhost, $guser, $gpasswd );
118 $result = mysql_select_db( $gDB, $gfac_link );
119
120 if ( ! $result )
121 {
122 write_log( "$self: Could not connect to DB $gDB" );
123 echo "Could not connect to DB $gDB.\n";
124 exit();
125 }
126
127 $query = "SELECT * FROM cluster_status WHERE cluster='$cluster'";
128 $result = mysql_query( $query, $gfac_link );
129
130 if ( ! $result )
131 {
132 write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
133 echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
134 exit();
135 }
136
137 $rows = mysql_num_rows( $result );
138
139 if ( $rows == 0 ) // INSERT
140 {
141 $query = "INSERT INTO cluster_status SET " .
142 "cluster='$cluster', " .
143 "queued=$queued, " .
144 "running=$running, " .
145 "status='$status'";
146 }
147 else // UPDATE
148 {
149 $query = "UPDATE cluster_status SET " .
150 "queued=$queued, " .
151 "running=$running, " .
152 "status='$status' " .
153 "WHERE cluster='$cluster'";
154 }
155
156 $result = mysql_query( $query, $gfac_link );
157
158 if ( ! $result )
159 {
160 write_log( "$self: Query failed $query - " . mysql_error( $gfac_link ) );
161 echo "$self: Query failed $query - " . mysql_error( $gfac_link ) . "\n";
162 }
163}
164
165// Get local cluster status
166
167function local_status()
168{
169 global $self;
170 global $data;
171 global $dbhost;
172 global $org_domain;
173 global $class_dir;
174
175 if ( preg_match( "/_local/", $class_dir ) )
176 {
177 if ( preg_match( "/attlocal/", $org_domain ) )
178 $clusters = array( "us3iab-devel", "alamo-local" );
179 else
180 $clusters = array( "us3iab-node0" );
181 }
182 else
183 {
184 $clusters = array( "alamo", "lonestar5", "stampede", "comet",
185 "gordon", "jureca", "jacinto" );
186 }
187
188 foreach ( $clusters as $clname )
189 {
190 $a = Array();
191//echo "$self: clname=$clname\n";
192
193 switch( $clname )
194 {
195 case 'us3iab-node0':
196 case 'us3iab-node1':
197 case 'us3iab-devel':
198 {
199 $qstat = `/usr/bin/qstat -B 2>&1|tail -1`;
200
201 $sparts = preg_split( '/\s+/', $qstat );
202 $que = $sparts[ 3 ];
203 $run = $sparts[ 4 ];
204 $sta = $sparts[ 10 ];
205 if ( $sta == "Active" )
206 $sta = "up";
207 else
208 $sta = "down";
209 break;
210 }
211 case 'alamo-local':
212 case 'alamo':
213 {
214 $host = "us3@alamo.uthscsa.edu";
215 $qstat = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
216 $sparts = preg_split( '/\s+/', $qstat );
217 $que = $sparts[ 3 ];
218 $run = $sparts[ 4 ];
219 $sta = $sparts[ 10 ];
220 if ( $sta == "Active" )
221 $sta = "up";
222 else
223 $sta = "down";
224 break;
225 }
226 case 'jacinto':
227 {
228 $host = "us3@jacinto.uthscsa.edu";
229 $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
230 $sparts = preg_split( '/\s+/', $qstat );
231 $que = $sparts[ 3 ];
232 $run = $sparts[ 4 ];
233 $sta = $sparts[ 9 ];
234 if ( $sta == "Active" )
235 $sta = "up";
236 else
237 $sta = "down";
238 break;
239 }
240 case 'stampede':
241 {
242 $host = "us3@stampede.tacc.utexas.edu";
243 $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
244 $sparts = preg_split( '/\s+/', $qstat );
245 $tot = $sparts[ 2 ];
246 $run = $sparts[ 5 ];
247 $que = $sparts[ 8 ];
248 $sta = "up";
249 if ( $tot == '' || $tot == '0' )
250 $sta = "down";
251 break;
252 }
253 case 'lonestar5':
254 {
255 $host = "us3@ls5.tacc.utexas.edu";
256 $qstat = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
257 $sparts = preg_split( '/\s+/', $qstat );
258 $tot = $sparts[ 2 ];
259 $run = '0';
260 $que = '0';
261 $sta = "up";
262 if ( $tot == '' || $tot == '0' )
263 {
264 $sta = "down";
265 }
266 else
267 {
268 $run = $sparts[ 5 ];
269 $que = $sparts[ 8 ];
270 }
271 break;
272 }
273 case 'comet':
274 {
275 $host = "us3@comet.sdsc.edu";
276 $qstat = `ssh $host '/usr/bin/sinfo -s -p compute -o "%a %F" |tail -1'`;
277 $sparts = preg_split( '/\s+/', $qstat );
278 $sta = $sparts[ 0 ];
279 $knts = $sparts[ 1 ];
280 $sparts = preg_split( '/\//', $knts );
281 $run = $sparts[ 0 ];
282 $que = $sparts[ 1 ];
283 if ( $sta == "" )
284 $sta = "down";
285 break;
286 }
287 case 'gordon':
288 {
289 $host = "us3@gordon.sdsc.edu";
290 $qstat = `ssh $host '/opt/torque/bin/qstat -B 2>&1|tail -1'`;
291 $sparts = preg_split( '/\s+/', $qstat );
292 $que = $sparts[ 3 ];
293 $run = $sparts[ 4 ];
294 $sta = $sparts[ 10 ];
295 if ( $sta == "Active" )
296 $sta = "up";
297 else
298 $sta = "down";
299 break;
300 }
301 case 'jureca':
302 {
303 $host = "swus1@jureca.fz-juelich.de";
304 $qstat = `ssh $host '~swus1/scripts/qstat-jureca 2>&1'`;
305 $sparts = preg_split( '/\s+/', $qstat );
306 $sta = $sparts[ 0 ];
307 $run = $sparts[ 1 ];
308 $que = $sparts[ 2 ];
309 break;
310 }
311 }
312
313 if ( $sta == "" )
314 $sta = "down";
315
316 if ( $sta == "down" )
317 {
318 $que = "0";
319 $run = "0";
320 }
321
322 $a[ 'cluster' ] = $clname;
323 $a[ 'queued' ] = $que;
324 $a[ 'running' ] = $run;
325 $a[ 'status' ] = $sta;
326//echo "$self: $clname $que $run $sta\n";
327
328 $data[] = $a;
329
330 if ( $clname == 'alamo' || $clname == 'jacinto' )
331 {
332 $a[ 'cluster' ] = $clname . "-local";
333 $data[] = $a;
334 }
335 }
336}
337
338class XML_Array
339{
340 var $_data = Array();
341 var $_name = Array();
342 var $_rep = Array();
343 var $_parser = 0;
344 var $_level = 0;
345 var $_index = 0;
346
347 function XML_Array( &$data )
348 {
349 $this->_parser = xml_parser_create();
350
351 xml_set_object ( $this->_parser, $this );
352 xml_parser_set_option ( $this->_parser, XML_OPTION_CASE_FOLDING, false );
353 xml_set_element_handler ( $this->_parser, "_startElement", "_endElement" );
354 xml_set_character_data_handler( $this->_parser, "_cdata" );
355
356 $this->_data = array();
357 $this->_level = 0;
358
359 if ( ! xml_parse( $this->_parser, $data, true ) )
360 return false;
361
362 xml_parser_free( $this->_parser );
363 }
364
365 function & ReturnArray()
366 {
367 return $this->_data[ 0 ];
368 }
369
370 function _startElement( $parser, $name, $attrs )
371 {
372 if ( $name == "resourceHealth" )
373 {
374## $name .= $this->_index;
375 $this->_index++;
376 }
377
378 if ( ! isset( $this->_rep[ $name ] ) ) $this->_rep[ $name ] = 0;
379
380 $this->_addElement( $name, $this->_data[ $this->_level ], $attrs );
381 $this->_name[ $this->_level ] = $name;
382 $this->_level++;
383 }
384
385 function _endElement( $parser, $name )
386 {
387 if ( isset( $this->_data[ $this->_level ] ) )
388 {
389 $this->_addElement( $this->_name[ $this->_level - 1 ],
390 $this->_data[ $this->_level - 1 ],
391 $this->_data[ $this->_level ]
392 );
393 }
394
395 unset( $this->_data[ $this->_level ] );
396 $this->_level--;
397 $this->_rep[ $name ]++;
398 }
399
400 function _cdata( $parser, $data )
401 {
402 if ( $this->_name[ $this->_level - 1 ] )
403 {
404 $this->_addElement( $this->_name[ $this->_level - 1 ],
405 $this->_data[ $this->_level - 1 ],
406 str_replace( array( "&gt;", "&lt;","&quot;", "&amp;" ),
407 array( ">" , "<" , '"' , "&" ),
408 $data
409 )
410 );
411 }
412 }
413
414 function _addElement( &$name, &$start, $add = array() )
415 {
416 if ( ( sizeof( $add ) == 0 && is_array( $add ) ) || ! $add )
417 {
418 if ( ! isset( $start[ $name ] ) ) $start[ $name ] = '';
419 $add = '';
420 }
421
422 $update = &$start[ $name ];
423
424 if ( is_array( $add) &&
425 is_array( $update ) ) $update += $add;
426 elseif ( is_array( $update ) ) return;
427 elseif ( is_array( $add ) ) $update = $add;
428 elseif ( $add ) $update .= $add;
429 }
430}
431?>
Note: See TracBrowser for help on using the repository browser.