airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ramin...@apache.org
Subject git commit: stated using Jcraft for SSHprovider communication. AIRAVATA-1082
Date Tue, 18 Mar 2014 20:17:47 GMT
Repository: airavata
Updated Branches:
  refs/heads/master e58a4427a -> 31f3883c2


stated using Jcraft for SSHprovider communication. AIRAVATA-1082


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/31f3883c
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/31f3883c
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/31f3883c

Branch: refs/heads/master
Commit: 31f3883c23f0ae6eb92e306eddfbfa214bfb41c4
Parents: e58a442
Author: raminder <raminder@apache.org>
Authored: Tue Mar 18 16:17:36 2014 -0400
Committer: raminder <raminder@apache.org>
Committed: Tue Mar 18 16:17:36 2014 -0400

----------------------------------------------------------------------
 .../client/samples/CreateLaunchExperiment.java  |  2 +-
 .../samples/CreateLaunchExperimentUS3.java      |  4 +-
 .../main/resources/airavata-client.properties   | 10 +--
 modules/gfac/gfac-core/pom.xml                  |  8 +-
 .../gfac/provider/impl/SSHProvider.java         | 82 +++++++++++---------
 .../apache/airavata/gsi/ssh/api/Cluster.java    | 10 ++-
 6 files changed, 62 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/31f3883c/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 7d4b129..efcf4a0 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -49,7 +49,7 @@ import java.util.Set;
 public class CreateLaunchExperiment {
 
     //FIXME: Read from a config file
-    public static final String THRIFT_SERVER_HOST = "localhost";
+    public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org";
     public static final int THRIFT_SERVER_PORT = 8930;
     private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
     private static final String DEFAULT_USER = "defauly.registry.user";

http://git-wip-us.apache.org/repos/asf/airavata/blob/31f3883c/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
index eb76412..24fd58d 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
@@ -37,7 +37,7 @@ public class CreateLaunchExperimentUS3 {
     public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org";
     public static final int THRIFT_SERVER_PORT = 8930;
     private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
-    private static final String DEFAULT_USER = "defauly.registry.user";
+    private static final String DEFAULT_USER = "default.registry.user";
     private static final String DEFAULT_GATEWAY = "default.registry.gateway";
     public static void main(String[] args) {
         try {
@@ -70,7 +70,7 @@ public class CreateLaunchExperimentUS3 {
                                      return;
                                  }else {
                                      if (JobState.COMPLETE.equals(jobStatus.getJobState()))
{
-                                         System.out.println("Job completed Job ID: " + key);
+                                         System.out.println("Job completed Job ID: " + jobStatus.getJobState().toString());
                                          return;
                                      }else{
                                         System.out.println("Job ID:" + key + jobStatuses.get(key).getJobState().toString());

http://git-wip-us.apache.org/repos/asf/airavata/blob/31f3883c/airavata-api/airavata-client-sdks/java-client-samples/src/main/resources/airavata-client.properties
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/resources/airavata-client.properties
b/airavata-api/airavata-client-sdks/java-client-samples/src/main/resources/airavata-client.properties
index 2a463b1..83be989 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/resources/airavata-client.properties
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/resources/airavata-client.properties
@@ -34,14 +34,14 @@ class.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.Airava
 
 ########################Registry JPA Implementation Settings########################
 #for mysql [AiravataJPARegistry]
-#registry.jdbc.driver=com.mysql.jdbc.Driver
-#registry.jdbc.url=jdbc:mysql://localhost:3306/persistent_data
+registry.jdbc.driver=com.mysql.jdbc.Driver
+registry.jdbc.url=jdbc:mysql://gw111.iu.xsede.org:3306/airavata
 
 #for derby [AiravataJPARegistry]
-registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver
-registry.jdbc.url=jdbc:derby://localhost:1527/persistent_data;create=true;user=airavata;password=airavata
+#registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#registry.jdbc.url=jdbc:derby://localhost:1527/persistent_data;create=true;user=airavata;password=airavata
 registry.jdbc.user=airavata
-registry.jdbc.password=airavata
+registry.jdbc.password=airavata12
 start.derby.server.mode=true
 
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/31f3883c/modules/gfac/gfac-core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/pom.xml b/modules/gfac/gfac-core/pom.xml
index 5e0965d..070f16c 100644
--- a/modules/gfac/gfac-core/pom.xml
+++ b/modules/gfac/gfac-core/pom.xml
@@ -95,13 +95,7 @@
             <version>${project.version}</version>
         </dependency>
 
-        <!-- SSH -->
-        <dependency>
-            <groupId>net.schmizz</groupId>
-            <artifactId>sshj</artifactId>
-            <version>0.8.0</version>
-        </dependency>
-
+   
         <!-- Credential Store -->
         <dependency>
             <groupId>org.apache.airavata</groupId>

http://git-wip-us.apache.org/repos/asf/airavata/blob/31f3883c/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
index 1b44c1b..f857d79 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
@@ -31,13 +31,6 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Random;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import net.schmizz.sshj.connection.ConnectionException;
-import net.schmizz.sshj.connection.channel.direct.Session;
-import net.schmizz.sshj.connection.channel.direct.Session.Command;
-import net.schmizz.sshj.transport.TransportException;
-import net.schmizz.sshj.xfer.scp.SCPFileTransfer;
 
 import org.apache.airavata.commons.gfac.type.ActualParameter;
 import org.apache.airavata.commons.gfac.type.MappingFactory;
@@ -49,6 +42,11 @@ import org.apache.airavata.gfac.context.security.SSHSecurityContext;
 import org.apache.airavata.gfac.provider.GFacProvider;
 import org.apache.airavata.gfac.provider.GFacProviderException;
 import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.CommandExecutor;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
+import org.apache.airavata.gsi.ssh.impl.StandardOutReader;
 import org.apache.airavata.model.workspace.experiment.JobState;
 import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
 import org.apache.airavata.schemas.gfac.NameValuePairType;
@@ -64,7 +62,7 @@ import sun.reflect.generics.reflectiveObjects.NotImplementedException;
  */
 public class SSHProvider extends AbstractProvider implements GFacProvider{
     private static final Logger log = LoggerFactory.getLogger(SSHProvider.class);
-    private SSHSecurityContext securityContext;
+    private Cluster cluster;
     private String jobID = null;
     private String taskID = null;
     // we keep gsisshprovider to support qsub submission incase of hpc scenario with ssh
@@ -75,18 +73,17 @@ public class SSHProvider extends AbstractProvider implements GFacProvider{
     	taskID = jobExecutionContext.getTaskData().getTaskID();
 		if (!((SSHHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getHpcResource())
{
             jobID = "SSH_" + jobExecutionContext.getApplicationContext().getHostDescription().getType().getHostAddress()
+ "_" + Calendar.getInstance().getTimeInMillis();
-
-            securityContext = (SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT);
+            cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
+            
             ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
             String remoteFile = app.getStaticWorkingDirectory() + File.separatorChar + Constants.EXECUTABLE_NAME;
             details.setJobDescription(remoteFile);
             GFacUtils.saveJobStatus(details, JobState.SETUP, taskID);
             log.info(remoteFile);
             try {
-                File runscript = createShellScript(jobExecutionContext);
-                SCPFileTransfer fileTransfer = securityContext.getSSHClient().newSCPFileTransfer();
-                fileTransfer.upload(runscript.getAbsolutePath(), remoteFile);
-            } catch (IOException e) {
+            	File runscript = createShellScript(jobExecutionContext);
+                cluster.scpTo(remoteFile, runscript.getAbsolutePath());
+            } catch (Exception e) {
                 throw new GFacProviderException(e.getLocalizedMessage(), e);
             }
         }else{
@@ -98,41 +95,32 @@ public class SSHProvider extends AbstractProvider implements GFacProvider{
     public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException
{
         if (gsiSshProvider == null) {
             ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
-            Session session = null;
             try {
-                session = securityContext.getSession(jobExecutionContext.getApplicationContext().getHostDescription().getType().getHostAddress());
                 /*
                  * Execute
                  */
                 String execuable = app.getStaticWorkingDirectory() + File.separatorChar +
Constants.EXECUTABLE_NAME;
                 details.setJobDescription(execuable);
                 GFacUtils.updateJobStatus(details, JobState.SUBMITTED);
-                Command cmd = session.exec("/bin/chmod 755 " + execuable + "; " + execuable);
-                log.info("stdout=" + GFacUtils.readFromStream(session.getInputStream()));
-                cmd.join(Constants.COMMAND_EXECUTION_TIMEOUT, TimeUnit.SECONDS);
+                RawCommandInfo rawCommandInfo = new RawCommandInfo("/bin/chmod 755 " + execuable
+ "; " + execuable);
 
-                /*
-                 * check return value. usually not very helpful to draw conclusions
-                 * based on return values so don't bother. just provide warning in
-                 * the log messages
-                 */
-                if (cmd.getExitStatus() != 0) {
-                    log.error("Process finished with non zero return value. Process may have
failed");
-                } else {
-                    log.info("Process finished with return value of zero.");
-                }
+                StandardOutReader jobIDReaderCommandOutput = new StandardOutReader();
+                
+                CommandExecutor.executeCommand(rawCommandInfo, cluster.getSession(), jobIDReaderCommandOutput);
+                String stdOutputString = getOutputifAvailable(jobIDReaderCommandOutput, "Error
submitting job to resource");
+
+                log.info("stdout=" + stdOutputString);
+             
                 GFacUtils.updateJobStatus(details, JobState.COMPLETE);
-            } catch (ConnectionException e) {
-                throw new GFacProviderException(e.getMessage(), e);
-            } catch (TransportException e) {
-                throw new GFacProviderException(e.getMessage(), e);
-            } catch (IOException e) {
-                throw new GFacProviderException(e.getMessage(), e);
-            }catch (Exception e) {
+            } catch (Exception e) {
                 throw new GFacProviderException(e.getMessage(), e);
             } finally {
-                if (securityContext != null) {
-                    securityContext.closeSession(session);
+                if (cluster != null) {
+                	try {
+						cluster.disconnect();
+					} catch (SSHApiException e) {
+						throw new GFacProviderException(e.getMessage(), e);
+					}
                 }
             }
         } else {
@@ -245,5 +233,23 @@ public class SSHProvider extends AbstractProvider implements GFacProvider{
             }
         }
     }
+    /**
+     * This method will read standard output and if there's any it will be parsed
+     * @param jobIDReaderCommandOutput
+     * @param errorMsg
+     * @return
+     * @throws SSHApiException
+     */
+    private String getOutputifAvailable(StandardOutReader jobIDReaderCommandOutput, String
errorMsg) throws SSHApiException {
+        String stdOutputString = jobIDReaderCommandOutput.getStdOutputString();
+        String stdErrorString = jobIDReaderCommandOutput.getStdErrorString();
+
+        if(stdOutputString == null || stdOutputString.isEmpty() || (stdErrorString != null
&& !stdErrorString.isEmpty())){
+            log.error("Standard Error output : " + stdErrorString);
+            throw new SSHApiException(errorMsg + stdErrorString);
+        }
+        return stdOutputString;
+    }
+
 
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/31f3883c/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
index f8cd543..1f49bc7 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
@@ -25,6 +25,8 @@ import java.util.List;
 import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
 import org.apache.airavata.gsi.ssh.impl.JobStatus;
 
+import com.jcraft.jsch.Session;
+
 /**
  * This interface represents a Cluster machine
  * End users of the API can implement this and come up with their own
@@ -110,7 +112,13 @@ public interface Cluster {
      */
     public List<String> listDirectory(String directoryPath) throws SSHApiException;
 
-
+    /**
+     * This method can be used to get created ssh session
+     * to reuse the created session.
+     * @throws SSHApiException
+     */
+    public Session getSession() throws SSHApiException;
+    
     /**
      * This method can be used to close the connections initialized
      * to handle graceful shutdown of the system


Mime
View raw message