Changeset 31 for trunk/gridctl.php

Timestamp:

Oct 13, 2017, 9:37:16 PM (7 years ago)

Author:

gegorbet

Message:

global-fit and jetstream,stampede2 mods

File:

• trunk/gridctl.php (modified) (10 diffs)

trunk/gridctl.php

@@ -70 +70 @@
    // Checking we need to do for each entry
 echo "us3db=$us3_db  gfid=$gfacID\n";
+//write_log( " us3db=$us3_db  gfid=$gfacID" );
    switch ( $us3_db )
    {
@@ -99 +100 @@
       $status     = aira_status( $gfacID, $status_in );
 if($status != $status_in )
-write_log( "$loghdr Set to $status from $status_in" );
+ write_log( "$loghdr Set to $status from $status_in" );
+//write_log( "$loghdr    aira status=$status" );
    }
    else if ( is_gfac_job( $gfacID ) )
@@ -112 +114 @@
    else
    {
+//write_log( "$loghdr Local gfacID=$gfacID" );
       $status_gw  = $status;
       $status     = get_local_status( $gfacID );
@@ -136 +139 @@
 
 //echo "  st=$status\n";
+//write_log( "$loghdr switch status=$status" );
    switch ( $status )
    {
@@ -174 +178 @@
       case "COMPLETED":
       case "COMPLETE":
-write_log( "$loghdr   COMPLETE gfacID=$gfacID" );
+//write_log( "$loghdr   COMPLETE gfacID=$gfacID" );
          complete();
          break;
@@ -833 +837 @@
    global $self;
 
-   $cmd    = "/usr/bin/qstat -a $gfacID 2>&1|tail -n 1";
+   $is_jetstr = preg_match( "/jetstream/", $cluster );
+   if ( $is_jetstr )
+      $cmd    = "squeue -a $gfacID 2>&1|tail -n 1";
+   else
+      $cmd    = "/usr/bin/qstat -a $gfacID 2>&1|tail -n 1";
 //write_log( "$self cmd: $cmd" );
 //write_log( "$self cluster: $cluster" );
 //write_log( "$self gfacID: $gfacID" );
+
    if ( ! preg_match( "/us3iab/", $cluster ) )
    {
       $system = "$cluster.uthscsa.edu";
+      if ( $is_jetstr )
+         $system = "$cluster";
       $system = preg_replace( "/\-local/", "", $system );
-//write_log( "$self system: $system" );
       $cmd    = "/usr/bin/ssh -x us3@$system " . $cmd;
 //write_log( "$self  cmd: $cmd" );
@@ -849 +859 @@
 //write_log( "$self  result: $result" );
 
-   if ( $result == ""  ||  preg_match( "/^qstat: Unknown/", $result ) )
+///////////////////////////////////////////////////////////////////
+   $secwait    = 2;
+   $num_try    = 0;
+   // Sleep and retry up to 3 times if ssh has "ssh_exchange_identification" error
+   while ( preg_match( "/ssh_exchange_id/", $result )  &&  $num_try < 3 )
+   {
+      sleep( $secwait );
+      $num_try++;
+      $secwait   *= 2;
+write_log( "$me:   num_try=$num_try  secwait=$secwait" );
+   }
+///////////////////////////////////////////////////////////////////
+   if ( $result == ""  ||
+        preg_match( "/^qstat: Unknown/", $result )  ||
+        preg_match( "/ssh_exchange_id/", $result ) )
    {
       write_log( "$self get_local_status: Local job $gfacID unknown" );
@@ -857 +881 @@
 
    $values = preg_split( "/\s+/", $result );
-//write_log( "$self: get_local_status: job status = /{$values[9]}/");
-   switch ( $values[ 9 ] )
+   $jstat   = ( $is_jetstr == 0 ) ? $values[ 9 ] : $values[ 4 ];
+//write_log( "$self: get_local_status: job status = /$jstat/");
+   switch ( $jstat )
    {
       case "W" :                      // Waiting for execution time to be reached
@@ -873 +898 @@
       case "H" :                      // Held
       case "Q" :                      // Queued
+      case "PD" :                     // Queued
         $status = 'SUBMITTED';
         break;
@@ -1079 +1105 @@
    if ( preg_match( "/US3-A/i", $gfacID )  &&  $devmatch )
    {
+//write_log( "$loghdr status_in=$status_in status=$status gfacID=$gfacID" );
       $status_ex = getExperimentStatus( $gfacID );
+//write_log( "$loghdr   status_ex=$status_ex" );
 
       if ( $status_ex == 'COMPLETED' )