Changeset 25 for trunk

Timestamp:

Jan 13, 2017, 1:30:29 PM (8 years ago)

Author:

gegorbet

Message:

merge us3iab/airavata gridctl code

Location:

trunk

Files:

• cleanup.php (modified) (7 diffs)
• cleanup_aira.php (modified) (1 diff)
• cleanup_gfac.php (modified) (6 diffs)
• cluster_status.php (modified) (4 diffs)
• gridctl.php (modified) (27 diffs)
• manage-us3-pipe.php (modified) (1 diff)
• update_notice.php (modified) (1 diff)

trunk/cleanup.php

@@ -17 +17 @@
 function aira_cleanup( $us3_db, $reqID, $gfac_link )
 {
+   global $org_domain;
    global $dbhost;
    global $user;
@@ -141 +142 @@
    list( $status, $cluster, $id ) = mysql_fetch_array( $result );
 
-   if ( $cluster == 'bcf-local'  || $cluster == 'alamo-local' )
+   $is_us3iab  = preg_match( "/us3iab/", $cluster );
+   $is_local   = preg_match( "/-local/", $cluster );
+
+   if ( $is_us3iab  ||  $is_local )
    {
          $clushost = $cluster;
@@ -191 +195 @@
    $fn_tarfile = "analysis-results.tar";
    $num_try    = 0;
+$len1 = strlen( $fn_tarfile );
+write_log( "$me: fn_tarfile=$fn_tarfile  len(tar)=$len1" );
    while ( ! file_exists( $fn_tarfile ) && $num_try < 3 )
    {
       sleep( 10 );
       $num_try++;
+write_log( "$me:  tar-exists: num_try=$num_try" );
    }
 
@@ -200 +207 @@
    foreach ( $ofiles as $ofile )
    {
-      if ( preg_match( "/.*stderr$/", $ofile ) )
+      if ( preg_match( "/^" . $gfacID . ".*stderr$/", $ofile ) )
          $fn_stderr  = $ofile;
-      if ( preg_match( "/.*stdout$/", $ofile ) )
+      if ( preg_match( "/^" . $gfacID . ".*stdout$/", $ofile ) )
          $fn_stdout  = $ofile;
 //write_log( "$me:    ofile=$ofile" );
@@ -654 +661 @@
    global $editXMLFilename;
    global $stdout;
+   global $org_domain;
 
 global $me;
@@ -697 +705 @@
    $limshost    = $dbhost;
    if ( $limshost == 'localhost' )
+   {
       $limshost    = gethostname();
+      if ( ! preg_match( "/\./", $limshost ) )
+         $limshost    = $limshost . $org_domain;
+   }
 
    // Parse the editXMLFilename
@@ -841 +853 @@
    global $db;
    global $status;
-
-   // Figure out local working directory
-   if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
-   $pwd = chdir( "$work/$gfacID" );
+   $is_us3iab  = preg_match( "/us3iab/", $cluster );
 
    // Figure out remote directory
    $remoteDir = sprintf( "$work_remote/$db-%06d", $requestID );
+write_log( "$me: is_us3iab=$is_us3iab  remoteDir=$remoteDir" );
 
    // Get stdout, stderr, output/analysis-results.tar
    $output = array();
-//   $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
-//
-//   exec( $cmd, $output, $stat );
-//   if ( $stat != 0 ) 
-//      write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
-//     
-//   $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
-//   exec( $cmd, $output, $stat );
-//   if ( $stat != 0 ) 
-//      write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
-
-   $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
-   exec( $cmd, $output, $stat );
-   if ( $stat != 0 ) 
-      write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
-
-   $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
-   exec( $cmd, $output, $stat );
-   if ( $stat != 0 ) 
-   {
-      write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
-      sleep( 10 );
-      write_log( "$me: RETRY" );
+
+   if ( $is_us3iab == 0 )
+   {
+      // Figure out local working directory
+      if ( ! is_dir( "$work/$gfacID" ) ) mkdir( "$work/$gfacID", 0770 );
+      $pwd = chdir( "$work/$gfacID" );
+
+      $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/output/analysis-results.tar . 2>&1";
+
       exec( $cmd, $output, $stat );
-      if ( $stat != 0 ) 
+      if ( $stat != 0 )
          write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
-   }
-     
-   $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
-   exec( $cmd, $output, $stat );
-   if ( $stat != 0 ) 
-   {
-      write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
-      sleep( 10 );
-      write_log( "$me: RETRY" );
+
+      $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stdout . 2>&1";
+
       exec( $cmd, $output, $stat );
-      if ( $stat != 0 ) 
+      if ( $stat != 0 )
+      {
          write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
-   }
+         sleep( 10 );
+         write_log( "$me: RETRY" );
+         exec( $cmd, $output, $stat );
+         if ( $stat != 0 )
+            write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
+      }
+
+      $cmd = "scp us3@$cluster.uthscsa.edu:$remoteDir/stderr . 2>&1";
+
+      exec( $cmd, $output, $stat );
+      if ( $stat != 0 )
+      {
+         write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
+         sleep( 10 );
+         write_log( "$me: RETRY" );
+         exec( $cmd, $output, $stat );
+         if ( $stat != 0 )
+            write_log( "$me: Bad exec:\n$cmd\n" . implode( "\n", $output ) );
+      }
+   }
+   else
+   {
+      $pwd = chdir( "$remoteDir" );
+write_log( "$me: IS US3IAB: pwd=$pwd $remoteDir");
+   }
+
 
    // Write the files to gfacDB
 
-   if ( file_exists( "stderr" ) ) $stderr  = file_get_contents( "stderr" );
+   if ( file_exists( "stderr"  ) )
+   {  // Filter stderr to not have libnnls debug lines
+      exec( "mv stderr stderr+nnls", $output, $stat );
+      exec( "grep -vi nnls stderr+nnls >stderr", $output, $stat );
+      $stderr  = file_get_contents( "stderr" );
+   }
+   else
+      $stderr  = "";
    if ( file_exists( "stdout" ) ) $stdout  = file_get_contents( "stdout" );
-   if ( file_exists( "analysis-results.tar" ) ) 
-      $tarfile = file_get_contents( "analysis-results.tar" );
-
+   $fn1_tarfile = "analysis-results.tar";
+   $fn2_tarfile = "output/" . $fn1_tarfile;
+   if ( file_exists( $fn1_tarfile ) )
+      $tarfile = file_get_contents( $fn1_tarfile );
+   else if ( file_exists( $fn2_tarfile ) )
+      $tarfile = file_get_contents( $fn2_tarfile );
+
+   $lense = strlen( $stderr );
+   if ( $lense > 1000000 )
+   { // Replace exceptionally large stderr with smaller version
+      exec( "mv stderr stderr-orig", $output, $stat );
+      exec( "head -n 5000 stderr-orig >stderr-h", $output, $stat );
+      exec( "tail -n 5000 stderr-orig >stderr-t", $output, $stat );
+      exec( "cat stderr-h stderr-t >stderr", $output, $stat );
+      $stderr  = file_get_contents( "stderr" );
+   }
+$lent = strlen( $tarfile );
+write_log( "$me: tarfile size: $lent");
+$lene = strlen( $stderr );
+write_log( "$me: stderr size: $lene  (was $lense)");
+$leno = strlen( $stdout );
+write_log( "$me: stdout size: $leno");
+$estarf=mysql_real_escape_string($tarfile,$gfac_link);
+$lenf = strlen($estarf);
+write_log( "$me: es-tarfile size: $lenf");
+$esstdo=mysql_real_escape_string($stdout,$gfac_link);
+$leno = strlen($esstdo);
+write_log( "$me: es-stdout size: $leno");
+$esstde=mysql_real_escape_string($stderr,$gfac_link);
+$lene = strlen($esstde);
+write_log( "$me: es-stderr size: $lene");
    $query = "UPDATE analysis SET " .
             "stderr='"  . mysql_real_escape_string( $stderr,  $gfac_link ) . "'," .

trunk/cleanup_aira.php

@@ -14 +14 @@
 $me              = 'cleanup_aira.php';
 $class_dir       = $class_dir_p;
-include_once "cleanup.php";
+include_once "$us3bin/cleanup.php";
 
 ?>

trunk/cleanup_gfac.php

@@ -159 +159 @@
 //write_log( "$me:     db=$db; requestID=$requestID; status=$status; cluster=$cluster" );
 
-   if ( $cluster == 'bcf-local'  ||  $cluster == 'alamo-local' )
-   {
-      $clushost = $cluster;
-      $clushost = preg_replace( "/\-local/", "", $clushost );
+//   if ( $cluster == 'bcf-local'  ||  $cluster == 'alamo-local' )
+   if ( preg_match( "/\-local/", $cluster )  ||
+        preg_match( "/us3iab/",  $cluster ) )
+   {
+//      $clushost = $cluster;
+//      $clushost = preg_replace( "/\-local/", "", $clushost );
+      $parts    = explode( "-", $cluster );
+      $clushost = $parts[ 0 ];
       get_local_files( $gfac_link, $clushost, $requestID, $id, $gfacID );
-   }
+write_log( "$me:     clushost=$clushost  reqID=$requestID get_local_files() gfacID=$gfacID" );
+   }
+else
+write_log( "$me:     NO get_local_files()" );
 
    $query = "SELECT id, stderr, stdout, tarfile FROM analysis " .
@@ -328 +335 @@
    file_put_contents( $message_filename, $message_log, FILE_APPEND );
   // mysql_close( $gfac_link );
-//write_log( "$me: *messages.txt written" );
+write_log( "$me: *messages.txt written" );
 
    /////////
@@ -346 +353 @@
       return( -1 );
    }
-//write_log( "$me: HPCAnalysisResult stderr,stdout updated" );
 
    // Save the tarfile and expand it
@@ -368 +374 @@
    chdir( "$work/$gfacID" );
 
-   $f = fopen( "analysis.tar", "w" );
+   $f = fopen( "analysis-results.tar", "w" );
    fwrite( $f, $tarfile );
    fclose( $f );
-write_log( "$me: analysis.tar file written to work dir" );
+//write_log( "$me: analysis-results.tar file written to work dir" );
 
    $tar_out = array();
-   exec( "tar -xf analysis.tar 2>&1", $tar_out, $err );
+   exec( "tar -xf analysis-results.tar 2>&1", $tar_out, $err );
 
    if ( $err != 0 )
@@ -461 +467 @@
                   "noiseGUID='$noiseGUID'," .
                   "modelGUID='$modelGUID'," .
-                  "editedDataID=1, "        .
+                  "editedDataID=2, "        .
                   "modelID=1, "             .
                   "noiseType='$type',"      .
@@ -670 +676 @@
    mail_to_user( "success", "" );
 }
-
 ?>

trunk/cluster_status.php

@@ -4 +4 @@
 include "$us3bin/listen-config.php";
 
-$xml  = get_data();
-
-if ( $xml != "" )
-   parse( $xml );
+if ( ! preg_match( "/localhost/", $dbhost ) )
+{
+   $xml  = get_data();
+
+   if ( $xml != "" )
+      parse( $xml );
+}
 
 $data = array();
@@ -165 +168 @@
    global $self;
    global $data;
-
-   $clusters = array( "alamo", "lonestar5", "stampede",
-                      "comet", "gordon", "jureca", "jacinto" );
-//   $clusters = array( "alamo", "lonestar5", "stampede",
-//                      "comet", "gordon", "jureca"            );
+   global $dbhost;
+   global $org_domain;
+
+   if ( preg_match( "/localhost/", $dbhost ) )
+   {
+      if ( preg_match( "/attlocal/", $org_domain ) )
+         $clusters = array( "us3iab-devel" );
+      else
+         $clusters = array( "us3iab-node0" );
+   }
+   else
+   {
+      $clusters = array( "alamo", "lonestar5", "stampede", "comet",
+                         "gordon", "jureca", "jacinto" );
+   }
+
    foreach ( $clusters as $clname )
    {
       $a      = Array();
+//echo "$self:   clname=$clname\n";
+
       switch( $clname )
       {
-         case 'alamo':
-         {
-            $host   = "us3@alamo.uthscsa.edu";
-            $qstat  = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
+         case 'us3iab-node0':
+         case 'us3iab-node1':
+         case 'us3iab-devel':
+         {
+            $qstat  = `/usr/bin/qstat -B 2>&1|tail -1`;
+
             $sparts = preg_split( '/\s+/', $qstat );
             $que    = $sparts[ 3 ];
@@ -189 +207 @@
             break;
          }
+         case 'alamo':
+         {
+            $host   = "us3@alamo.uthscsa.edu";
+            $qstat  = `ssh $host '/usr/bin/qstat -B 2>&1|tail -1'`;
+            $sparts = preg_split( '/\s+/', $qstat );
+            $que    = $sparts[ 3 ];
+            $run    = $sparts[ 4 ];
+            $sta    = $sparts[ 10 ];
+            if ( $sta == "Active" )
+               $sta    = "up";
+            else
+               $sta    = "down";
+            break;
+         }
          case 'jacinto':
          {
@@ -289 +321 @@
       $a[ 'running' ] = $run;
       $a[ 'status'  ] = $sta;
+//echo "$self:  $clname  $que $run $sta\n";
 
       $data[] = $a;

trunk/gridctl.php

@@ -1 +1 @@
 <?php
+
+$us3bin = exec( "ls -d ~us3/lims/bin" );
+include_once "$us3bin/listen-config.php";
+//include "$us3bin/cleanup_aira.php";
+//include "$us3bin/cleanup_gfac.php";
 
 // Global variables
@@ -53 +58 @@
       if ( (  $me_devel  &&  !$job_devel )  ||
            ( !$me_devel  &&   $job_devel ) )
-      {  // If job not from appropriate Airavata server, skip processing
+      {  // Job type and Airavata server mismatch:  skip processing
          continue;
       }
-   }
-   else if ( $me_devel )
-   {  // For non-thrift and development, skip
-      continue;
    }
 
@@ -78 +79 @@
    }
 
-   $awork = array();
-   $awork = explode( "-", $gfacID );
-   $gfacLabl = $awork[0] . "-" . $awork[1] . "-" . $awork[2];
+//   $awork = array();
+//   $awork = explode( "-", $gfacID );
+//   $gfacLabl = $awork[0] . "-" . $awork[1] . "-" . $awork[2];
+   $gfacLabl = $gfacID;
    $loghdr   = $self . ":" . $gfacLabl . "...:";
-   $status_in = $status;
-   $status_gw = $status;
    $status_ex = $status;
 
@@ -91 +91 @@
    {
       $status_in  = $status;
+//write_log( "$loghdr status_in=$status_in" );
       $status     = aira_status( $gfacID, $status_in );
 if($status != $status_in )
 write_log( "$loghdr Set to $status from $status_in" );
    }
-   else
+   else if ( is_gfac_job( $gfacID ) )
    {
       $status_gw  = $status;
@@ -102 +103 @@
       if ( $status_gw == 'COMPLETE' )
          $status     = $status_gw;
-write_log( "$loghdr non-AThrift status=$status status_gw=$status_gw" );
+//write_log( "$loghdr non-AThrift status=$status status_gw=$status_gw" );
+   }
+   else
+   {
+      $status_gw  = $status;
+      $status     = get_local_status( $gfacID );
+      if ( $status_gw == 'COMPLETE'  ||  $status == 'UNKNOWN' )
+         $status     = $status_gw;
+//write_log( "$loghdr Local status=$status status_gw=$status_gw" );
    }
 
@@ -160 +169 @@
       case "COMPLETED":
       case "COMPLETE":
+write_log( "$loghdr   COMPLETE gfacID=$gfacID" );
          complete();
          break;
@@ -173 +183 @@
          if ( is_aira_job( $gfacID ) )
          {
-            $status_ex    = getExperimentStatus( $gfacID );
-write_log( "$loghdr     status=$status status_ex=$status_ex" );
-            if ( $status_ex === 'COMPLETED' )
-               complete();
+            complete();
          }
+write_log( "$loghdr   FINISHED gfacID=$gfacID" );
       case "PROCESSING":
       default:
@@ -207 +215 @@
       if ( ! in_array( $job_status, array( 'SUBMITTED', 'INITIALIZED', 'PENDING' ) ) )
       {
-//write_log( "$loghdr submitted:job_status=$job_status" );
+write_log( "$loghdr submitted:job_status=$job_status" );
          update_job_status( $job_status, $gfacID );
       }
@@ -242 +250 @@
    if ( ! in_array( $job_status, array( 'SUBMITTED', 'INITIALIZED', 'PENDING' ) ) )
    {
-//write_log( "$loghdr submit_timeout:job_status=$job_status" );
       update_job_status( $job_status, $gfacID );
       return;
@@ -286 +293 @@
 
       if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
-      {
-//write_log( "$loghdr running:job_status=$job_status" );
          update_job_status( $job_status, $gfacID );
-      }
 
       return;
@@ -322 +326 @@
    if ( ! in_array( $job_status, array( 'ACTIVE', 'RUNNING', 'STARTED' ) ) )
    {
-//write_log( "$loghdr run_timeout:job_status=$job_status" );
       update_job_status( $job_status, $gfacID );
       return;
@@ -365 +368 @@
       if ( $job_status != 'DATA' )
       {
-//write_log( "$loghdr wait_data:job_status=$job_status" );
          update_job_status( $job_status, $gfacID );
          return;
@@ -410 +412 @@
    if ( $job_status != 'DATA' )
    {
-//write_log( "$loghdr data_timeout:job_status=$job_status" );
       update_job_status( $job_status, $gfacID );
       return;
@@ -463 +464 @@
    global $us3_db;
    global $loghdr;
-   global $class_dir;
 
    // Double check that the gfacID exists
@@ -488 +488 @@
 
    $me_devel  = preg_match( "/class_devel/", $class_dir );
+   $me_local  = preg_match( "/class_local/", $class_dir );
 
    if ( preg_match( "/US3-A/i", $gfacID ) )
@@ -494 +495 @@
       if ( ( !$me_devel  &&  !$job_devel )  ||
            (  $me_devel  &&   $job_devel ) )
-      {  // If job from appropriate Airavata server, process it
+      {  // Job is of same type (prod/devel) as Server:  process it
 //write_log( "$loghdr CALLING aira_cleanup()" );
          aira_cleanup( $us3_db, $requestID, $gLink );
@@ -500 +501 @@
 //write_log( "$loghdr RTN FR aira_cleanup()" );
    }
-
-   else if ( ! $me_devel )
-   {  // If this is gridctl_pro and gfac (jureca), do GFAC cleanup
-//write_log( "$loghdr CALLING gfac_cleanup()" );
+   else if ( ! $me_local )
+   {
+write_log( "$loghdr CALLING gfac_cleanup() reqID=$requestID" );
       gfac_cleanup( $us3_db, $requestID, $gLink );
    }
@@ -521 +521 @@
     case 'SUBMITED'    :
     case 'INITIALIZED' :
+    case 'UPDATING'    :
+    case 'PENDING'     :
       $query   = "UPDATE analysis SET status='SUBMITTED' WHERE gfacID='$gfacID'";
-      $message = "Job status request reports job is " . $job_status;
-//write_log( "$loghdr update_job_status(SUBM) job_status=$job_status" );
+      $message = "Job status request reports job is SUBMITTED";
       break;
 
@@ -569 +570 @@
       $query   = "UPDATE analysis SET status='ERROR' WHERE gfacID='$gfacID'";
       $message = "Job status request reports job is not in the queue";
-      break;
-
-    case 'UPDATING'    :
-    case 'PENDING'     :
-      $message = "Job status request reports job is " . $job_status;
       break;
 
@@ -695 +691 @@
 
       $gfac_status  = standard_status( $status_ex );
-write_log( "$loghdr get_gfac_status: status_ex=$status_ex gfac_status=$gfac_status" );
       return $gfac_status;
    }
 
    else if ( ! is_gfac_job( $gfacID ) )
+   {
       return false;
+   }
 
    $url = "$serviceURL/jobstatus/$gfacID";
@@ -758 +755 @@
    {
       // Then it's not a GFAC job
-      return false;
+      $job_status = get_local_status( $gfacID );
+      return $job_status;
    }
 
@@ -825 +823 @@
    global $self;
 
-   $system = "$cluster.uthscsa.edu";
-   $system = preg_replace( "/\-local/", "", $system );
-   $cmd    = "/usr/bin/ssh -x us3@$system qstat -a $gfacID 2>&1";
+   $cmd    = "/usr/bin/qstat -a $gfacID 2>&1|tail -n 1";
+   if ( ! preg_match( "/us3iab/", $cluster ) )
+   {
+      $system = "$cluster.uthscsa.edu";
+      $system = preg_replace( "/\-local/", "", $system );
+      $cmd    = "/usr/bin/ssh -x us3@$system " . $cmd;
+   }
 
    $result = exec( $cmd );
@@ -834 +836 @@
    {
       write_log( "$self get_local_status: Local job $gfacID unknown" );
+write_log( "$self get_local_status: result=$result" );
       return 'UNKNOWN';
    }
 
    $values = preg_split( "/\s+/", $result );
-//   write_log( "$self: get_local_status: job status = /{$values[9]}/");
+//write_log( "$self: get_local_status: job status = /{$values[9]}/");
    switch ( $values[ 9 ] )
    {
@@ -999 +1002 @@
       case 'canceled' :
          $status      = 'CANCELED';
+         break;
+
+         $status      = 'DATA';
          break;
 
@@ -1043 +1049 @@
    global $self;
    global $loghdr;
-   global $class_dir;
 //echo "a_st: st_in$status_in : $gfacID\n";
    //$status_gw = standard_status( $status_in );
@@ -1054 +1059 @@
                   (  $me_devel  &&   $job_devel ) );
 
-//write_log( "$loghdr  gfacID=$gfacID  devmatch=$devmatch" );
-//write_log( "$loghdr   me_d=$me_devel  jo_d=$job_devel  cd=$class_dir" );
    if ( preg_match( "/US3-A/i", $gfacID )  &&  $devmatch )
-   {  // Airavata job and development/production type is right
+   {
       $status_ex = getExperimentStatus( $gfacID );
-//write_log( "$loghdr status_ex $status_ex" );
 
       if ( $status_ex == 'COMPLETED' )
@@ -1101 +1103 @@
       }
 
-if(preg_match("/US3-A/i",$gfacID))
-//if(preg_match("/US3-ADEV/i",$gfacID))
-write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
 //write_log( "$loghdr status/_in/_gw/_ex=$status/$status_in/$status_gw/$status_ex" );
-//write_log( "  me_d=$me_devel jo_d=$job_devel dm=$devmatch cd=$class_dir" );
       if ( $status != $status_gw )
       {

trunk/manage-us3-pipe.php

@@ -3 +3 @@
 $us3bin = exec( "ls -d ~us3/lims/bin" );
 include "$us3bin/listen-config.php";
-include "/srv/www/htdocs/common/class/experiment_status.php";
+include "$class_dir/experiment_status.php";
 
 write_log( "$self: Starting" );

trunk/update_notice.php

@@ -24 +24 @@
 $dbhost     = "localhost";
 $dbuser     = "root";
-//$dbpassw    = exec( "cat ~us3/.sec/.pwsq" );
 $dbpassw    = exec( "cat ~/.sec/.pwsq" );