Index: /trunk/cleanup.php
===================================================================
--- /trunk/cleanup.php	(revision 30)
+++ /trunk/cleanup.php	(revision 31)
@@ -225,4 +225,6 @@
    if ( file_exists( $fn_stdout  ) ) $stdout   = file_get_contents( $fn_stdout  );
    if ( file_exists( $fn_tarfile ) ) $tarfile  = file_get_contents( $fn_tarfile );
+write_log( "$me(0):  length contents stderr,stdout,tarfile -- "
+ . strlen($stderr) . "," . strlen($stdout) . "," . strlen($tarfile) );
    // If stdout,stderr have no content, retry after delay
    if ( strlen( $stdout ) == 0  ||  strlen( $stderr ) == 0 )
@@ -870,4 +872,5 @@
    global $status;
    $is_us3iab  = preg_match( "/us3iab/", $cluster );
+   $is_jetstr  = preg_match( "/jetstream/", $cluster );
 
    // Figure out remote directory
@@ -881,5 +884,12 @@
    {
       // For "-local", recompute remote work directory
-      $cmd = "ssh us3@$cluster.uthscsa.edu 'ls -d ~us3/lims/work/local' 2/dev/null";
+      $clushost = "$cluster.uthscsa.edu";
+      $lworkdir = "~us3/lims/work/local";
+      if ( $is_jetstr )
+      {
+         $clushost = "js-157-184.jetstream-cloud.org";
+         $lworkdir = "/N/us3_cluster/work/local";
+      }
+      $cmd         = "ssh us3@$clushost 'ls -d $lworkdir' 2/dev/null";
       exec( $cmd, $output, $stat );
       $work_remote = $output[ 0 ];
@@ -891,5 +901,5 @@
       $pwd = chdir( "$work/$gfacID" );
 
-      $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
+      $cmd = "scp us3@$clushost:$remoteDir/output/analysis-results.tar . 2>&1";
 
       exec( $cmd, $output, $stat );
@@ -897,5 +907,5 @@
          write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
 
-      $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
+      $cmd = "scp us3@$clushost:$remoteDir/stdout . 2>&1";
 
       exec( $cmd, $output, $stat );
@@ -910,5 +920,5 @@
       }
 
-      $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
+      $cmd = "scp us3@$clushost:$remoteDir/stderr . 2>&1";
 
       exec( $cmd, $output, $stat );
Index: /trunk/cluster_status.php
===================================================================
--- /trunk/cluster_status.php	(revision 30)
+++ /trunk/cluster_status.php	(revision 31)
@@ -183,5 +183,5 @@
    {
       $clusters = array( "alamo", "lonestar5", "stampede", "comet",
-                         "gordon", "jureca", "jacinto" );
+                         "stampede2-b", "jetstream", "jureca", "jacinto-b" );
    }
 
@@ -251,4 +251,17 @@
             break;
          }
+         case 'stampede2':
+         {
+            $host   = "us3@stampede2.tacc.utexas.edu";
+            $qstat  = `ssh $host '/usr/local/bin/showq 2>&1|grep "Total Jobs"'`;
+            $sparts = preg_split( '/\s+/', $qstat );
+            $tot    = $sparts[ 2 ];
+            $run    = $sparts[ 5 ];
+            $que    = $sparts[ 8 ];
+            $sta    = "up";
+            if ( $tot == ''  ||  $tot == '0' )
+               $sta    = "down";
+            break;
+         }
          case 'lonestar5':
          {
@@ -267,5 +280,6 @@
             {
                $run    = $sparts[ 5 ];
-               $que    = $sparts[ 8 ];
+//               $que    = $sparts[ 8 ];
+               $que    = $sparts[ 11 ];
             }
             break;
@@ -309,4 +323,19 @@
             break;
          }
+         case 'jetstream-local':
+         case 'jetstream':
+         {
+            $host   = "us3@js-157-184.jetstream-cloud.org";
+            $qstat  = `ssh $host '/usr/bin/sinfo -s -p batch -o "%a %F" |tail -1'`;
+            $sparts = preg_split( '/\s+/', $qstat );
+            $sta    = $sparts[ 0 ];
+            $knts   = $sparts[ 1 ];
+            $sparts = preg_split( '/\//', $knts );
+            $run    = $sparts[ 0 ];
+            $que    = $sparts[ 1 ];
+            if ( $sta == "" )
+               $sta    = "down";
+            break;
+         }
       }
 
@@ -328,5 +357,7 @@
       $data[] = $a;
 
-      if ( $clname == 'alamo'  ||  $clname == 'jacinto' )
+      if ( $clname == 'alamo'  ||
+           $clname == 'jacinto'  ||
+           $clname == 'jetstream' )
       {
          $a[ 'cluster' ] = $clname . "-local";
Index: /trunk/gridctl.php
===================================================================
--- /trunk/gridctl.php	(revision 30)
+++ /trunk/gridctl.php	(revision 31)
@@ -70,4 +70,5 @@
    // Checking we need to do for each entry
 echo "us3db=$us3_db  gfid=$gfacID\n";
+//write_log( " us3db=$us3_db  gfid=$gfacID" );
    switch ( $us3_db )
    {
@@ -99,5 +100,6 @@
       $status     = aira_status( $gfacID, $status_in );
 if($status != $status_in )
-write_log( "$loghdr Set to $status from $status_in" );
+ write_log( "$loghdr Set to $status from $status_in" );
+//write_log( "$loghdr    aira status=$status" );
    }
    else if ( is_gfac_job( $gfacID ) )
@@ -112,4 +114,5 @@
    else
    {
+//write_log( "$loghdr Local gfacID=$gfacID" );
       $status_gw  = $status;
       $status     = get_local_status( $gfacID );
@@ -136,4 +139,5 @@
 
 //echo "  st=$status\n";
+//write_log( "$loghdr switch status=$status" );
    switch ( $status )
    {
@@ -174,5 +178,5 @@
       case "COMPLETED":
       case "COMPLETE":
-write_log( "$loghdr   COMPLETE gfacID=$gfacID" );
+//write_log( "$loghdr   COMPLETE gfacID=$gfacID" );
          complete();
          break;
@@ -833,13 +837,19 @@
    global $self;
 
-   $cmd    = "/usr/bin/qstat -a $gfacID 2>&1|tail -n 1";
+   $is_jetstr = preg_match( "/jetstream/", $cluster );
+   if ( $is_jetstr )
+      $cmd    = "squeue -a $gfacID 2>&1|tail -n 1";
+   else
+      $cmd    = "/usr/bin/qstat -a $gfacID 2>&1|tail -n 1";
 //write_log( "$self cmd: $cmd" );
 //write_log( "$self cluster: $cluster" );
 //write_log( "$self gfacID: $gfacID" );
+
    if ( ! preg_match( "/us3iab/", $cluster ) )
    {
       $system = "$cluster.uthscsa.edu";
+      if ( $is_jetstr )
+         $system = "$cluster";
       $system = preg_replace( "/\-local/", "", $system );
-//write_log( "$self system: $system" );
       $cmd    = "/usr/bin/ssh -x us3@$system " . $cmd;
 //write_log( "$self  cmd: $cmd" );
@@ -849,5 +859,19 @@
 //write_log( "$self  result: $result" );
 
-   if ( $result == ""  ||  preg_match( "/^qstat: Unknown/", $result ) )
+///////////////////////////////////////////////////////////////////
+   $secwait    = 2;
+   $num_try    = 0;
+   // Sleep and retry up to 3 times if ssh has "ssh_exchange_identification" error
+   while ( preg_match( "/ssh_exchange_id/", $result )  &&  $num_try < 3 )
+   {
+      sleep( $secwait );
+      $num_try++;
+      $secwait   *= 2;
+write_log( "$me:   num_try=$num_try  secwait=$secwait" );
+   }
+///////////////////////////////////////////////////////////////////
+   if ( $result == ""  ||
+        preg_match( "/^qstat: Unknown/", $result )  ||
+        preg_match( "/ssh_exchange_id/", $result ) )
    {
       write_log( "$self get_local_status: Local job $gfacID unknown" );
@@ -857,6 +881,7 @@
 
    $values = preg_split( "/\s+/", $result );
-//write_log( "$self: get_local_status: job status = /{$values[9]}/");
-   switch ( $values[ 9 ] )
+   $jstat   = ( $is_jetstr == 0 ) ? $values[ 9 ] : $values[ 4 ];
+//write_log( "$self: get_local_status: job status = /$jstat/");
+   switch ( $jstat )
    {
       case "W" :                      // Waiting for execution time to be reached
@@ -873,4 +898,5 @@
       case "H" :                      // Held
       case "Q" :                      // Queued
+      case "PD" :                     // Queued
         $status = 'SUBMITTED';
         break;
@@ -1079,5 +1105,7 @@
    if ( preg_match( "/US3-A/i", $gfacID )  &&  $devmatch )
    {
+//write_log( "$loghdr status_in=$status_in status=$status gfacID=$gfacID" );
       $status_ex = getExperimentStatus( $gfacID );
+//write_log( "$loghdr   status_ex=$status_ex" );
 
       if ( $status_ex == 'COMPLETED' )