airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lah...@apache.org
Subject svn commit: r1530072 - in /airavata/sandbox/gsissh/src: main/java/org/apache/airavata/gsi/ssh/api/ main/java/org/apache/airavata/gsi/ssh/api/job/ main/java/org/apache/airavata/gsi/ssh/impl/ main/java/org/apache/airavata/gsi/ssh/util/ main/resources/ te...
Date Mon, 07 Oct 2013 21:22:26 GMT
Author: lahiru
Date: Mon Oct  7 21:22:26 2013
New Revision: 1530072

URL: http://svn.apache.org/r1530072
Log:
fixing issues in ssh authentication.

Modified:
    airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
    airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
    airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
    airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
    airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt
    airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
    airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java

Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java?rev=1530072&r1=1530071&r2=1530072&view=diff
==============================================================================
--- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java (original)
+++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java Mon
Oct  7 21:22:26 2013
@@ -63,9 +63,17 @@ public interface Cluster {
      *
      * @param rFile remote file location
      * @param lFile local file path of the file which needs to copy to remote location
+     * @return the successful remote file path of the transfer
      * @throws SSHApiException throws exception during error
      */
-    public void scpTo(String rFile, String lFile) throws SSHApiException;
+    public String scpTo(String rFile, String lFile) throws SSHApiException;
+
+    /**
+     * This will create directories in computing resources
+     * @param directoryPath the full qualified path for the directory user wants to create
+     * @throws SSHApiException throws during error
+     */
+    public void makeDirectory(String directoryPath) throws SSHApiException;
 
     /**
      * This will get the job description of a job which is there in the cluster

Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java?rev=1530072&r1=1530071&r2=1530072&view=diff
==============================================================================
--- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
(original)
+++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
Mon Oct  7 21:22:26 2013
@@ -119,7 +119,8 @@ public class JobDescriptor {
      * @param minutes
      */
     public void setMaxWallTime(String minutes) {
-        this.getJobDescriptorDocument().getJobDescriptor().setMaxWallTime(minutes);
+        this.getJobDescriptorDocument().getJobDescriptor().setMaxWallTime(
+                CommonUtils.maxWallTimeCalculator(Integer.parseInt(minutes)));
 
     }
 

Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java?rev=1530072&r1=1530071&r2=1530072&view=diff
==============================================================================
--- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
(original)
+++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java
Mon Oct  7 21:22:26 2013
@@ -260,7 +260,8 @@ public class PBSCluster implements Clust
             Source text = new StreamSource(new ByteArrayInputStream(jobDescriptor.toXML().getBytes()));
             transformer.transform(text, new StreamResult(results));
 
-            log.debug("generated PBS:" + results.toString());
+            System.out.println(results.toString());
+//            log.debug("generated PBS:" + results.toString());
 
             // creating a temporary file using pbs script generated above
             int number = new SecureRandom().nextInt();
@@ -455,9 +456,9 @@ public class PBSCluster implements Clust
         return jobDescriptor;
     }
 
-    public void scpTo(String rFile, String lFile) throws SSHApiException {
+    public String scpTo(String rFile, String lFile) throws SSHApiException {
         try {
-            SSHUtils.scpTo(rFile, lFile, session);
+            return SSHUtils.scpTo(rFile, lFile, session);
         } catch (IOException e) {
             throw new SSHApiException("Failed during scping local file:" + lFile + " to remote
file "
                     + serverInfo.getHost() + ":rFile", e);
@@ -467,6 +468,18 @@ public class PBSCluster implements Clust
         }
     }
 
+    public void makeDirectory(String directoryPath) throws SSHApiException {
+        try {
+            SSHUtils.makeDirectory(directoryPath, session);
+        } catch (IOException e) {
+            throw new SSHApiException("Failed during creating directory:" + directoryPath
+ " to remote file "
+                    + serverInfo.getHost() + ":rFile", e);
+        } catch (JSchException e) {
+            throw new SSHApiException("Failed during creating directory :" + directoryPath
+ " to remote file "
+                    + serverInfo.getHost() + ":rFile", e);
+        }
+    }
+
 //
 //    public String submitAsyncJob(Job jobDescriptor, JobSubmissionListener listener) throws
SSHApiException {
 ////        final Cluster cluster = this;

Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java?rev=1530072&r1=1530071&r2=1530072&view=diff
==============================================================================
--- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java (original)
+++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java Mon
Oct  7 21:22:26 2013
@@ -196,7 +196,7 @@ public class SSHUtils {
 
         channel.disconnect();
         session.disconnect();
-        int i = lFile.lastIndexOf("\\");
+        int i = lFile.lastIndexOf(File.separator);
         String substring = lFile.substring(i + 1);
         return rFile + File.separator + substring;
     }
@@ -273,7 +273,7 @@ public class SSHUtils {
         out.close();
 
         channel.disconnect();
-        int i = lFile.lastIndexOf("\\");
+        int i = lFile.lastIndexOf(File.separator);
         String substring = lFile.substring(i + 1);
         return rFile + File.separator + substring;
     }
@@ -281,7 +281,7 @@ public class SSHUtils {
     public static Session makeDirectory(String path, Session session) throws IOException,
JSchException, SSHApiException {
 
         // exec 'scp -t rfile' remotely
-        String command = "mkdir " + path;
+        String command = "mkdir -p " + path;
         Channel channel = session.openChannel("exec");
         CommandOutput commandOutput = new SystemCommandOutput();
         ((ChannelExec) channel).setCommand(command);
@@ -306,9 +306,6 @@ public class SSHUtils {
         commandOutput.onOutput(channel);
 
         channel.disconnect();
-        session.disconnect();
-
-        channel.disconnect();
         return session;
     }
 

Modified: airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt
URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt?rev=1530072&r1=1530071&r2=1530072&view=diff
==============================================================================
--- airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt (original)
+++ airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt Mon Oct  7 21:22:26 2013
@@ -10,32 +10,46 @@
 <xsl:template match="/ns:JobDescriptor">
 #! /bin/sh
 # PBS batch job script built by Globus job manager
-#
-    <xsl:if test="shellName != @xsl:nil">
+#   <xsl:choose>
+    <xsl:when test="ns:shellName">
 ##PBS -S <xsl:value-of select="ns:shellName"/>
-    </xsl:if>
-    <xsl:if test="queueName != @xsl:nil">
+    </xsl:when></xsl:choose>
+    <xsl:choose>
+    <xsl:when test="ns:queueName">
 #PBS -q <xsl:value-of select="ns:queueName"/>
-    </xsl:if>
-    <xsl:if test="mailOptions != @xsl:nil">
+    </xsl:when>
+    </xsl:choose>
+    <xsl:choose>
+    <xsl:when test="ns:mailOptions">
 #PBS -m <xsl:value-of select="ns:mailOptions"/>
-    </xsl:if>
-<xsl:if test="acountString != @xsl:nil">
+    </xsl:when>
+    </xsl:choose>
+    <xsl:choose>
+<xsl:when test="ns:acountString">
 #PBS -A <xsl:value-of select="ns:acountString"/>
-    </xsl:if>
-    <xsl:if test="maxWallTime != @xsl:nil">
+    </xsl:when>
+    </xsl:choose>
+    <xsl:choose>
+    <xsl:when test="ns:maxWallTime">
 #PBS -l walltime=<xsl:value-of select="ns:maxWallTime"/>
-    </xsl:if>
-    <xsl:if test="standardOutFile != @xsl:nil">
+    </xsl:when>
+    </xsl:choose>
+    <xsl:choose>
+    <xsl:when test="ns:standardOutFile">
 #PBS -o <xsl:value-of select="ns:standardOutFile"/>
-    </xsl:if>
-    <xsl:if test="standardOutFile != @xsl:nil">
+    </xsl:when>
+    </xsl:choose>
+    <xsl:choose>
+    <xsl:when test="ns:standardOutFile">
 #PBS -e <xsl:value-of select="ns:standardErrorFile"/>
-    </xsl:if>
-    <xsl:if test="(nodes != @xsl:nil) and (processesPerNode != @xsl:nil)">
+    </xsl:when>
+    </xsl:choose>
+    <xsl:choose>
+    <xsl:when test="(ns:nodes) and (ns:processesPerNode)">
 #PBS -l nodes=<xsl:value-of select="ns:nodes"/>:ppn=<xsl:value-of select="ns:processesPerNode"/>
 <xsl:text>&#xa;</xsl:text>
-    </xsl:if>
+    </xsl:when>
+    </xsl:choose>
 <xsl:for-each select="ns:exports/ns:name">
 <xsl:value-of select="."/>=<xsl:value-of select="./@value"/><xsl:text>&#xa;</xsl:text>
 export<xsl:text>   </xsl:text><xsl:value-of select="."/>
@@ -45,7 +59,8 @@ export<xsl:text>   </xsl:text><xsl:value
       <xsl:value-of select="."/><xsl:text>   </xsl:text>
     </xsl:for-each>
 cd <xsl:text>   </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text>&#xa;</xsl:text>
-<xsl:value-of select="ns:jobSubmitterCommand"/><xsl:text>   </xsl:text><xsl:value-of
select="ns:executablePath"/><xsl:text>   </xsl:text>
+    <xsl:choose><xsl:when test="ns:jobSubmitterCommand">
+<xsl:value-of select="ns:jobSubmitterCommand"/><xsl:text>   </xsl:text></xsl:when></xsl:choose><xsl:value-of
select="ns:executablePath"/><xsl:text>   </xsl:text>
 <xsl:for-each select="ns:inputs/ns:input">
       <xsl:value-of select="."/><xsl:text>   </xsl:text>
     </xsl:for-each>

Modified: airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java?rev=1530072&r1=1530071&r2=1530072&view=diff
==============================================================================
--- airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
(original)
+++ airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
Mon Oct  7 21:22:26 2013
@@ -162,7 +162,7 @@ public class DefaultSSHApiTest {
         jobDescriptor.setNodes(1);
         jobDescriptor.setProcessesPerNode(1);
         jobDescriptor.setQueueName("normal");
-        jobDescriptor.setMaxWallTime("1:00:00");
+        jobDescriptor.setMaxWallTime("60");
         jobDescriptor.setAcountString("sds128");
         List<String> inputs = new ArrayList<String>();
         inputs.add("Hello World");
@@ -245,7 +245,7 @@ public class DefaultSSHApiTest {
         jobDescriptor.setNodes(1);
         jobDescriptor.setProcessesPerNode(100);
         jobDescriptor.setQueueName("normal");
-        jobDescriptor.setMaxWallTime("1:00:00");
+        jobDescriptor.setMaxWallTime("60");
         jobDescriptor.setAcountString("sds128");
         List<String> inputs = new ArrayList<String>();
         inputs.add("Hello World");
@@ -270,7 +270,7 @@ public class DefaultSSHApiTest {
         ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
 
 
-        Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin1/");
+        Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/");
 
 
         // Execute command
@@ -378,7 +378,7 @@ public class DefaultSSHApiTest {
         jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err");
         jobDescriptor.setNodes(1);
         jobDescriptor.setProcessesPerNode(1);
-        jobDescriptor.setMaxWallTime("1:00:00");
+        jobDescriptor.setMaxWallTime("60");
         jobDescriptor.setQueueName("normal");
         jobDescriptor.setAcountString("sds128");
         List<String> inputs = new ArrayList<String>();

Modified: airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java
URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java?rev=1530072&r1=1530071&r2=1530072&view=diff
==============================================================================
--- airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java
(original)
+++ airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java
Mon Oct  7 21:22:26 2013
@@ -21,7 +21,6 @@
 
 package org.apache.airavata.gsi.ssh.impl;
 
-import com.jcraft.jsch.UserInfo;
 import org.apache.airavata.gsi.ssh.api.*;
 import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
 import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
@@ -32,10 +31,11 @@ import org.testng.AssertJUnit;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import javax.swing.*;
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
+import java.util.UUID;
 
 /**
  * User: AmilaJ (amilaj@apache.org)
@@ -56,10 +56,10 @@ public class VanilaSSHTest {
 
         this.hostName = "bigred2.uits.iu.edu";
         System.setProperty("my.ssh.user", "lginnali");
-        System.setProperty("my.ssh.user.password","");
-//        System.setProperty("my.ssh.user","i want to be free");
+        System.setProperty("my.ssh.user.password", "");
         this.userName = System.getProperty("my.ssh.user");
         this.password = System.getProperty("my.ssh.user.password");
+        System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh");
 //        this.passPhrase = System.getProperty("my.ssh.user.pass.phrase");
 
         if (this.userName == null || this.password == null) {
@@ -72,7 +72,7 @@ public class VanilaSSHTest {
 
 
     @Test
-    public void testSimpleCommand1() throws Exception{
+    public void testSimpleCommand1() throws Exception {
 
         System.out.println("Starting vanila SSH test ....");
 
@@ -94,7 +94,7 @@ public class VanilaSSHTest {
     }
 
     @Test
-    public void testSimpleCommand2() throws Exception{
+    public void testSimpleCommand2() throws Exception {
 
         System.out.println("Starting vanila SSH test ....");
 
@@ -120,17 +120,34 @@ public class VanilaSSHTest {
     }
 
     @Test
-    public void testSimplePBSJob() throws Exception{
+    public void testSimplePBSJob() throws Exception {
 
         AuthenticationInfo authenticationInfo = new DefaultPasswordAuthenticationInfo(this.password);
-         // Server info
+        // Server info
         ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
         Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/torque-4.2.3.1/bin/");
 
+        String date = new Date().toString();
+        date = date.replaceAll(" ", "_");
+        date = date.replaceAll(":", "_");
 
-        // Execute command
+        String pomFile = System.getProperty("basedir") + File.separator + "pom.xml";
+
+        // Constructing theworking directory for demonstration and creating directories in
the remote
+        // resource
         String workingDirectory = File.separator + "N" + File.separator + "u" + File.separator
+
-                "lginnali" + File.separator + "BigRed2" + File.separator + "myjob";
+                "lginnali" + File.separator + "BigRed2";
+        workingDirectory = workingDirectory + File.separator
+                + date + "_" + UUID.randomUUID();
+        pbsCluster.makeDirectory(workingDirectory);
+        Thread.sleep(1000);
+        pbsCluster.makeDirectory(workingDirectory + File.separator + "inputs");
+        Thread.sleep(1000);
+        pbsCluster.makeDirectory(workingDirectory + File.separator +  "outputs");
+
+
+        // doing file transfer to the remote resource
+        String s = pbsCluster.scpTo(workingDirectory + File.separator + "inputs", pomFile);
         // constructing the job object
         JobDescriptor jobDescriptor = new JobDescriptor();
         jobDescriptor.setWorkingDirectory(workingDirectory);
@@ -147,7 +164,7 @@ public class VanilaSSHTest {
         jobDescriptor.setMaxWallTime("5");
         jobDescriptor.setJobSubmitter("aprun -n 1");
         List<String> inputs = new ArrayList<String>();
-        inputs.add("Hello World");
+        inputs.add(s);
         jobDescriptor.setInputValues(inputs);
         //finished construction of job object
         System.out.println(jobDescriptor.toXML());



Mime
View raw message