Index: trunk/gridctl.php
===================================================================
--- trunk/gridctl.php	(revision 3)
+++ trunk/gridctl.php	(revision 6)
@@ -1,6 +1,8 @@
 <?php
 
-include_once "/home/us3/bin/listen-config.php";
-include "/home/us3/bin/cleanup.php";
+$us3bin = exec( "ls -d ~us3/bin" );
+include_once "$us3bin/listen-config.php";
+include "$us3bin/cleanup_aira.php";
+include "$us3bin/cleanup_gfac.php";
 
 // Global variables
@@ -10,7 +12,9 @@
 $cluster    = '';
 
+//global $self;
+
 // Produce some output temporarily, so cron will send me message
 $now = time();
-//echo "Time started: " . date( 'Y-m-d H:i:s', $now ) . "\n";
+echo "Time started: " . date( 'Y-m-d H:i:s', $now ) . "\n";
 
 // Get data from global GFAC DB 
@@ -36,5 +40,8 @@
 
 if ( mysql_num_rows( $result ) == 0 )
+{
+//write_log( "$self: analysis read got numrows==0" );
    exit();  // Nothing to do
+}
 
 while ( list( $gfacID, $us3_db, $cluster, $status, $queue_msg, $time, $updateTime ) 
@@ -42,4 +49,43 @@
 {
    // Checking we need to do for each entry
+echo "us3db=$us3_db  gfid=$gfacID\n";
+   switch ( $us3_db )
+   {
+      case 'Xuslims3_cauma3' :
+      case 'Xuslims3_cauma3d' :
+      case 'Xuslims3_HHU' :
+      case 'Xuslims3_Uni_KN' :
+         $serviceURL  = "http://gridfarm005.ucs.indiana.edu:9090/ogce-rest/job";
+         break;
+
+      default :
+//         $serviceURL  = "http://gridfarm005.ucs.indiana.edu:8080/ogce-rest/job";
+         break;
+   }
+
+   $awork = array();
+   $awork = explode( "-", $gfacID );
+   $gfacLabl = $awork[0] . "-" . $awork[1] . "-" . $awork[2];
+   $loghdr   = $self . ":" . $gfacLabl . "...:";
+
+   // If entry is for Airvata/Thrift, get the true current status
+
+   if ( is_aira_job( $gfacID ) )
+   {
+      $status_in  = $status;
+//write_log( "$loghdr status_in=$status_in" );
+      $status     = aira_status( $gfacID, $status_in );
+if($status != $status_in )
+write_log( "$loghdr Set to $status from $status_in" );
+   }
+   else
+   {
+      $status_gw  = $status;
+      $status     = get_gfac_status( $gfacID );
+      //if ( $status == 'FINISHED' )
+      if ( $status_gw == 'COMPLETE' )
+         $status     = $status_gw;
+write_log( "$loghdr non-AThrift status=$status status_gw=$status_gw" );
+   }
 
    // Sometimes during testing, the us3_db entry is not set
@@ -47,5 +93,5 @@
    if ( strlen( $us3_db ) == 0 && $status != 'ERROR' ) 
    {
-      write_log( "$self: GFAC DB is NULL - $gfacID" );
+      write_log( "$loghdr GFAC DB is NULL - $gfacID" );
       mail_to_admin( "fail", "GFAC DB is NULL\n$gfacID" );
 
@@ -55,8 +101,9 @@
 
       if ( ! $result2 )
-         write_log( "$self: Query failed $query2 - " .  mysql_error( $gLink ) );
-
-   }
-
+         write_log( "$loghdr Query failed $query2 - " .  mysql_error( $gLink ) );
+
+   }
+
+//echo "  st=$status\n";
    switch ( $status )
    {
@@ -76,4 +123,7 @@
 
       case "RUNNING":
+      case "STARTED":
+      case "STAGING":
+      case "ACTIVE":
          running( $time );
          break;
@@ -84,4 +134,5 @@
 
       case "DATA":
+      case "RESULTS_GEN":
          wait_data( $time );
          break;
@@ -91,4 +142,5 @@
          break;
 
+      case "COMPLETED":
       case "COMPLETE":
          complete();
@@ -101,4 +153,11 @@
          break;
 
+      case "FINISHED":
+      case "DONE":
+         if ( is_aira_job( $gfacID ) )
+         {
+            complete();
+         }
+      case "PROCESSING":
       default:
          break;
@@ -113,4 +172,5 @@
    global $gLink;
    global $gfacID;
+   global $loghdr;
 
    $now = time();
@@ -127,5 +187,8 @@
 
       if ( ! in_array( $job_status, array( 'SUBMITTED', 'INITIALIZED', 'PENDING' ) ) )
+      {
+write_log( "$loghdr submitted:job_status=$job_status" );
          update_job_status( $job_status, $gfacID );
+      }
 
       return;
@@ -150,4 +213,5 @@
    global $gLink;
    global $gfacID;
+   global $loghdr;
 
    if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
@@ -185,4 +249,5 @@
    global $gLink;
    global $gfacID;
+   global $loghdr;
 
    $now = time();
@@ -200,5 +265,5 @@
          return;
 
-      if ( $job_status != 'ACTIVE' )
+      if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
          update_job_status( $job_status, $gfacID );
 
@@ -224,4 +289,5 @@
    global $gLink;
    global $gfacID;
+   global $loghdr;
 
    if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
@@ -231,5 +297,5 @@
       return;
 
-   if ( $job_status != 'ACTIVE' )
+   if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
    {
       update_job_status( $job_status, $gfacID );
@@ -261,4 +327,5 @@
    global $gLink;
    global $gfacID;
+   global $loghdr;
 
    $now = time();
@@ -308,4 +375,5 @@
    global $gLink;
    global $gfacID;
+   global $loghdr;
 
    if ( ( $job_status = get_gfac_status( $gfacID ) ) === false )
@@ -368,4 +436,5 @@
    global $gfacID;
    global $us3_db;
+   global $loghdr;
 
    // Double check that the gfacID exists
@@ -382,11 +451,24 @@
    list( $count ) = mysql_fetch_array( $result );
 
+if ($count==0)
+write_log( "$loghdr count = $count  gfacID = $gfacID" );
    if ( $count == 0 ) return;
 
    // Now check the us3 instance
    $requestID = get_us3_data();
+//write_log( "$loghdr requestID = $requestID  gfacID = $gfacID" );
    if ( $requestID == 0 ) return;
 
-   gfac_cleanup( $us3_db, $requestID, $gLink );
+   if ( preg_match( "/US3-AIRA/i", $gfacID ) )
+   {
+//write_log( "$loghdr CALLING aira_cleanup()" );
+      aira_cleanup( $us3_db, $requestID, $gLink );
+//write_log( "$loghdr RTN FR aira_cleanup()" );
+   }
+   else
+   {
+//write_log( "$loghdr CALLING gfac_cleanup()" );
+      gfac_cleanup( $us3_db, $requestID, $gLink );
+   }
 }
 
@@ -395,4 +477,7 @@
 {
   global $gLink;
+  global $query;
+  global $self;
+  global $loghdr;
   
   switch ( $job_status )
@@ -401,4 +486,5 @@
     case 'SUBMITED'    :
     case 'INITIALIZED' :
+    case 'UPDATING'    :
     case 'PENDING'     :
       $query   = "UPDATE analysis SET status='SUBMITTED' WHERE gfacID='$gfacID'";
@@ -406,4 +492,6 @@
       break;
 
+    case 'STARTED'     :
+    case 'RUNNING'     :
     case 'ACTIVE'      :
       $query   = "UPDATE analysis SET status='RUNNING' WHERE gfacID='$gfacID'";
@@ -411,11 +499,21 @@
       break;
 
+    case 'FINISHED'    :
+      $query   = "UPDATE analysis SET status='FINISHED' WHERE gfacID='$gfacID'";
+      $message = "NONE";
+      break;
+
+    case 'DONE'        :
+      $query   = "UPDATE analysis SET status='DONE' WHERE gfacID='$gfacID'";
+      $message = "NONE";
+      break;
+
     case 'COMPLETED'   :
-    case 'DONE'   :
+    case 'COMPLETE'   :
       $query   = "UPDATE analysis SET status='COMPLETE' WHERE gfacID='$gfacID'";
-      $message = "Job status request reports job is COMPLETE";
+      $message = "Job status request reports job is COMPLETED";
       break;
 
-    case 'DATA'   :
+    case 'DATA'        :
       $query   = "UPDATE analysis SET status='DATA' WHERE gfacID='$gfacID'";
       $message = "Job status request reports job is COMPLETE, waiting for data";
@@ -423,5 +521,5 @@
 
     case 'CANCELED'    :
-    case 'CANCELLED'    :
+    case 'CANCELLED'   :
       $query   = "UPDATE analysis SET status='CANCELED' WHERE gfacID='$gfacID'";
       $message = "Job status request reports job is CANCELED";
@@ -434,5 +532,6 @@
 
     case 'UNKNOWN'     :
-      // $query   = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
+write_log( "$loghdr job_status='UNKNOWN', reset to 'ERROR' " );
+      $query   = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
       $message = "Job status request reports job is not in the queue";
       break;
@@ -440,7 +539,7 @@
     default            :
       // We shouldn't ever get here
-      $query   = "";
+      $query   = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
       $message = "Job status was not recognized - $job_status";
-      write_log( "$self - update_job_status: " .
+      write_log( "$loghdr update_job_status: " .
                  "Job status was not recognized - $job_status\n" .
                  "gfacID = $gfacID\n" );
@@ -451,8 +550,11 @@
    $result =  mysql_query( $query, $gLink );
    if ( ! $result )
-      write_log( "$self: Query failed $query - " .  mysql_error( $gLink ) );
-
-   update_queue_messages( $message );
-   update_db( $message );
+      write_log( "$loghdr Query failed $query - " .  mysql_error( $gLink ) );
+
+   if ( $message != 'NONE' )
+   {
+      update_queue_messages( $message );
+      update_db( $message );
+   }
 }
 
@@ -466,4 +568,5 @@
    global $us3_db;
    global $updateTime;
+   global $loghdr;
 
    $us3_link = mysql_connect( $dbhost, $user, $passwd );
@@ -471,5 +574,5 @@
    if ( ! $us3_link )
    {
-      write_log( "$self: could not connect: $dbhost, $user, $passwd" );
+      write_log( "$loghdr could not connect: $dbhost, $user, $passwd" );
       mail_to_admin( "fail", "Could not connect to $dbhost" );
       return 0;
@@ -481,5 +584,5 @@
    if ( ! $result )
    {
-      write_log( "$self: could not select DB $us3_db" );
+      write_log( "$loghdr could not select DB $us3_db" );
       mail_to_admin( "fail", "Could not select DB $us3_db, $dbhost, $user, $passwd" );
       return 0;
@@ -503,5 +606,5 @@
 }
 
-// Function to determine if this is a gfac job or a local job
+// Function to determine if this is a gfac job or not
 function is_gfac_job( $gfacID )
 {
@@ -517,10 +620,35 @@
 }
 
+// Function to determine if this is an airavata/thrift job or not
+function is_aira_job( $gfacID )
+{
+   global $cluster;
+
+   if ( preg_match( "/US3-AIRA/i", $gfacID )  &&
+        ! preg_match( "/juropa/i", $cluster ) )
+   {
+      // Then it's an Airavata/Thrift job
+      return true;
+   }
+
+   return false;
+}
+
 // Function to get the current job status from GFAC
 function get_gfac_status( $gfacID )
 {
    global $serviceURL;
-
-   if ( ! is_gfac_job( $gfacID ) )
+   global $self;
+   global $loghdr;
+   global $cluster;
+
+   if ( is_aira_job( $gfacID ) )
+   {
+      $status_ex    = getExperimentStatus( $gfacID );
+      $gfac_status  = standard_status( $status_ex );
+      return $gfac_status;
+   }
+
+   else if ( ! is_gfac_job( $gfacID ) )
       return false;
 
@@ -534,5 +662,5 @@
    catch ( HttpException $e )
    {
-      write_log( "$self: Status not available - marking failed -  $gfacID" );
+      write_log( "$loghdr Status not available - marking failed -  $gfacID" );
       return 'GFAC_STATUS_UNAVAILABLE';
    }
@@ -547,11 +675,16 @@
                          'INITIALIZED',
                          'PENDING',
+                         'RUNNING',
                          'ACTIVE',
+                         'STARTED',
                          'COMPLETED',
+                         'FINISHED',
                          'DONE',
                          'DATA',
+                         'RESULTS_GEN',
                          'CANCELED',
                          'CANCELLED',
                          'FAILED',
+                         'STAGING',
                          'UNKNOWN');
    if ( ! in_array( $gfac_status, $status_types ) )
@@ -560,4 +693,7 @@
                               "Status: /$gfac_status/\n" );
 
+   if ( in_array( $gfac_status, array( 'DONE', 'DATA', 'RESULTS_GEN' ) ) )
+      $gfac_status = 'DATA';
+
    return $gfac_status;
 }
@@ -567,4 +703,5 @@
 {
    global $serviceURL;
+   global $self;
 
    // Make sure it's a GFAC job and status is appropriate for this call
@@ -575,5 +712,5 @@
    }
 
-   if ( ! in_array( $job_status, array( 'DONE', 'FAILED', 'COMPLETE' ) ) )
+   if ( ! in_array( $job_status, array( 'DONE', 'FAILED', 'COMPLETE', 'FINISHED' ) ) )
    {
       // Then it's not appropriate to request data
@@ -637,4 +774,5 @@
 {
    global $cluster;
+   global $self;
 
    $system = "$cluster.uthscsa.edu";
@@ -697,6 +835,6 @@
    // Insert message into queue_message table
    $query  = "INSERT INTO queue_messages SET " .
-             "message = '" . mysql_real_escape_string( $message, $gLink ) . "'," .
-             "analysisID = $analysisID ";
+             "message = '" . mysql_real_escape_string( $message, $gLink ) . "', " .
+             "analysisID = '$analysisID' ";
    $result = mysql_query( $query, $gLink );
    if ( ! $result )
@@ -755,5 +893,5 @@
    $headers  = "From: $org_name Admin<$admin_email>"     . "\n";
    $headers .= "Cc: $org_name Admin<$admin_email>"       . "\n";
-   $headers .= "Bcc: Dan Zollars<dzollars@gmail.com>"    . "\n";     // make sure
+   $headers .= "Bcc: Gary Gorbet<gegorbet@gmail.com>"    . "\n";     // make sure
 
    // Set the reply address
@@ -781,3 +919,141 @@
    mail( $admin_email, $subject, $message, $headers );
 }
+
+// Convert a status string to one of the standard DB status strings
+function standard_status( $status_in )
+{
+   switch ( $status_in )
+   {  // Map variations to standard gateway status values
+      case 'QUEUED' :
+      case 'LAUNCHED' :
+      case 'CREATED' :
+      case 'VALIDATED' :
+      case 'SCHEDULED' :
+      case 'submitted' :
+      case '' :
+         $status      = 'SUBMITTED';
+         break;
+
+      case 'EXECUTING' :
+      case 'ACTIVE' :
+      case 'running' :
+      case 'executing' :
+         $status      = 'RUNNING';
+         break;
+
+      case 'PENDING' :
+      case 'CANCELING' :
+         $status      = 'UPDATING';
+         break;
+
+      case 'CANCELLED' :
+      case 'canceled' :
+         $status      = 'CANCELED';
+         break;
+
+         $status      = 'DATA';
+         break;
+
+      case 'COMPLETED' :
+      case 'completed' :
+         $status      = 'COMPLETE';
+         break;
+
+      case 'FAILED_DATA' :
+      case 'SUBMIT_TIMEOUT' :
+      case 'RUN_TIMEOUT' :
+      case 'DATA_TIMEOUT' :
+         $status      = 'FAILED';
+         break;
+
+      case 'COMPLETE' :
+         $status      = 'DONE';
+         break;
+
+      case 'UNKNOWN' :
+         $status      = 'ERROR';
+         break;
+
+      // Where already standard value, retain value
+      case 'ERROR' :
+      case 'RUNNING' :
+      case 'SUBMITTED' :
+      case 'UPDATING' :
+      case 'CANCELED' :
+      case 'DATA' :
+      case 'FAILED' :
+      case 'DONE' :
+      case 'FINISHED' :
+      default :
+         $status   = $status_in;
+         break;
+   }
+
+   return $status;
+}
+
+function aira_status( $gfacID, $status_in )
+{
+   global $self;
+   global $loghdr;
+//echo "a_st: st_in$status_in : $gfacID\n";
+   //$status_gw = standard_status( $status_in );
+   $status_gw = $status_in;
+//echo "a_st:  st_db=$status_gw\n";
+   $status    = $status_gw;
+
+   if ( preg_match( "/US3-AIRA/i", $gfacID ) )
+   {
+      $status_ex = getExperimentStatus( $gfacID );
+
+      if ( $status_ex == 'COMPLETED' )
+      {  // Experiment is COMPLETED: check for 'FINISHED' or 'DONE'
+         if ( $status_gw == 'FINISHED'  ||  $status_gw == 'DONE' )
+         {  // COMPLETED + FINISHED/DONE : gateway status is now COMPLETE
+            $status    = 'COMPLETE';
+         }
+
+         else
+         {  // COMPLETED + NOT-FINISHED/DONE:  gw status now DONE
+            $status    = 'DONE';
+         }
+      }
+
+      else if ( $status_gw == 'FINISHED'  ||  $status_gw == 'DONE' )
+      {  // Gfac status == FINISHED/DONE:  leave as is (unless FAILED)
+         $status    = $status_gw;
+         if ( $status_ex == 'FAILED' )
+         {
+            sleep( 10 );
+            $status_ex = getExperimentStatus( $gfacID );
+            if ( $status_ex == 'FAILED' )
+            {
+               write_log( "$loghdr status still 'FAILED' after 10-second delay" );
+               sleep( 10 );
+               $status_ex = getExperimentStatus( $gfacID );
+               if ( $status_ex == 'FAILED' )
+                  write_log( "$loghdr status still 'FAILED' after 20-second delay" );
+               else
+                  write_log( "$loghdr status is $status_ex after 20-second delayed retry" );
+            }
+            write_log( "$loghdr status reset to 'COMPLETE'" );
+            $status    = 'COMPLETE';
+         }
+      }
+
+      else
+      {  // Experiment not COMPLETED/FINISHED/DONE: use experiment status
+         $status    = standard_status( $status_ex );
+      }
+
+write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
+      if ( $status != $status_gw )
+      {
+         update_job_status( $status, $gfacID );
+      }
+   }
+
+   return $status;
+}
+
 ?>