Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AD09110180 for ; Mon, 7 Oct 2013 21:22:51 +0000 (UTC) Received: (qmail 88326 invoked by uid 500); 7 Oct 2013 21:22:50 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 88269 invoked by uid 500); 7 Oct 2013 21:22:50 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 88080 invoked by uid 99); 7 Oct 2013 21:22:49 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Oct 2013 21:22:49 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Oct 2013 21:22:47 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 134B42388A5E; Mon, 7 Oct 2013 21:22:27 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1530072 - in /airavata/sandbox/gsissh/src: main/java/org/apache/airavata/gsi/ssh/api/ main/java/org/apache/airavata/gsi/ssh/api/job/ main/java/org/apache/airavata/gsi/ssh/impl/ main/java/org/apache/airavata/gsi/ssh/util/ main/resources/ te... Date: Mon, 07 Oct 2013 21:22:26 -0000 To: commits@airavata.apache.org From: lahiru@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131007212227.134B42388A5E@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: lahiru Date: Mon Oct 7 21:22:26 2013 New Revision: 1530072 URL: http://svn.apache.org/r1530072 Log: fixing issues in ssh authentication. Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java?rev=1530072&r1=1530071&r2=1530072&view=diff ============================================================================== --- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java (original) +++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java Mon Oct 7 21:22:26 2013 @@ -63,9 +63,17 @@ public interface Cluster { * * @param rFile remote file location * @param lFile local file path of the file which needs to copy to remote location + * @return the successful remote file path of the transfer * @throws SSHApiException throws exception during error */ - public void scpTo(String rFile, String lFile) throws SSHApiException; + public String scpTo(String rFile, String lFile) throws SSHApiException; + + /** + * This will create directories in computing resources + * @param directoryPath the full qualified path for the directory user wants to create + * @throws SSHApiException throws during error + */ + public void makeDirectory(String directoryPath) throws SSHApiException; /** * This will get the job description of a job which is there in the cluster Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java?rev=1530072&r1=1530071&r2=1530072&view=diff ============================================================================== --- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java (original) +++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java Mon Oct 7 21:22:26 2013 @@ -119,7 +119,8 @@ public class JobDescriptor { * @param minutes */ public void setMaxWallTime(String minutes) { - this.getJobDescriptorDocument().getJobDescriptor().setMaxWallTime(minutes); + this.getJobDescriptorDocument().getJobDescriptor().setMaxWallTime( + CommonUtils.maxWallTimeCalculator(Integer.parseInt(minutes))); } Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java?rev=1530072&r1=1530071&r2=1530072&view=diff ============================================================================== --- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java (original) +++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/PBSCluster.java Mon Oct 7 21:22:26 2013 @@ -260,7 +260,8 @@ public class PBSCluster implements Clust Source text = new StreamSource(new ByteArrayInputStream(jobDescriptor.toXML().getBytes())); transformer.transform(text, new StreamResult(results)); - log.debug("generated PBS:" + results.toString()); + System.out.println(results.toString()); +// log.debug("generated PBS:" + results.toString()); // creating a temporary file using pbs script generated above int number = new SecureRandom().nextInt(); @@ -455,9 +456,9 @@ public class PBSCluster implements Clust return jobDescriptor; } - public void scpTo(String rFile, String lFile) throws SSHApiException { + public String scpTo(String rFile, String lFile) throws SSHApiException { try { - SSHUtils.scpTo(rFile, lFile, session); + return SSHUtils.scpTo(rFile, lFile, session); } catch (IOException e) { throw new SSHApiException("Failed during scping local file:" + lFile + " to remote file " + serverInfo.getHost() + ":rFile", e); @@ -467,6 +468,18 @@ public class PBSCluster implements Clust } } + public void makeDirectory(String directoryPath) throws SSHApiException { + try { + SSHUtils.makeDirectory(directoryPath, session); + } catch (IOException e) { + throw new SSHApiException("Failed during creating directory:" + directoryPath + " to remote file " + + serverInfo.getHost() + ":rFile", e); + } catch (JSchException e) { + throw new SSHApiException("Failed during creating directory :" + directoryPath + " to remote file " + + serverInfo.getHost() + ":rFile", e); + } + } + // // public String submitAsyncJob(Job jobDescriptor, JobSubmissionListener listener) throws SSHApiException { //// final Cluster cluster = this; Modified: airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java?rev=1530072&r1=1530071&r2=1530072&view=diff ============================================================================== --- airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java (original) +++ airavata/sandbox/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java Mon Oct 7 21:22:26 2013 @@ -196,7 +196,7 @@ public class SSHUtils { channel.disconnect(); session.disconnect(); - int i = lFile.lastIndexOf("\\"); + int i = lFile.lastIndexOf(File.separator); String substring = lFile.substring(i + 1); return rFile + File.separator + substring; } @@ -273,7 +273,7 @@ public class SSHUtils { out.close(); channel.disconnect(); - int i = lFile.lastIndexOf("\\"); + int i = lFile.lastIndexOf(File.separator); String substring = lFile.substring(i + 1); return rFile + File.separator + substring; } @@ -281,7 +281,7 @@ public class SSHUtils { public static Session makeDirectory(String path, Session session) throws IOException, JSchException, SSHApiException { // exec 'scp -t rfile' remotely - String command = "mkdir " + path; + String command = "mkdir -p " + path; Channel channel = session.openChannel("exec"); CommandOutput commandOutput = new SystemCommandOutput(); ((ChannelExec) channel).setCommand(command); @@ -306,9 +306,6 @@ public class SSHUtils { commandOutput.onOutput(channel); channel.disconnect(); - session.disconnect(); - - channel.disconnect(); return session; } Modified: airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt?rev=1530072&r1=1530071&r2=1530072&view=diff ============================================================================== --- airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt (original) +++ airavata/sandbox/gsissh/src/main/resources/PBSTemplate.xslt Mon Oct 7 21:22:26 2013 @@ -10,32 +10,46 @@ #! /bin/sh # PBS batch job script built by Globus job manager -# - +# + ##PBS -S - - + + + #PBS -q - - + + + + #PBS -m - - + + + + #PBS -A - - + + + + #PBS -l walltime= - - + + + + #PBS -o - - + + + + #PBS -e - - + + + + #PBS -l nodes=:ppn= - + + = export @@ -45,7 +59,8 @@ export cd - + + Modified: airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java?rev=1530072&r1=1530071&r2=1530072&view=diff ============================================================================== --- airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java (original) +++ airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java Mon Oct 7 21:22:26 2013 @@ -162,7 +162,7 @@ public class DefaultSSHApiTest { jobDescriptor.setNodes(1); jobDescriptor.setProcessesPerNode(1); jobDescriptor.setQueueName("normal"); - jobDescriptor.setMaxWallTime("1:00:00"); + jobDescriptor.setMaxWallTime("60"); jobDescriptor.setAcountString("sds128"); List inputs = new ArrayList(); inputs.add("Hello World"); @@ -245,7 +245,7 @@ public class DefaultSSHApiTest { jobDescriptor.setNodes(1); jobDescriptor.setProcessesPerNode(100); jobDescriptor.setQueueName("normal"); - jobDescriptor.setMaxWallTime("1:00:00"); + jobDescriptor.setMaxWallTime("60"); jobDescriptor.setAcountString("sds128"); List inputs = new ArrayList(); inputs.add("Hello World"); @@ -270,7 +270,7 @@ public class DefaultSSHApiTest { ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu"); - Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin1/"); + Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/"); // Execute command @@ -378,7 +378,7 @@ public class DefaultSSHApiTest { jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err"); jobDescriptor.setNodes(1); jobDescriptor.setProcessesPerNode(1); - jobDescriptor.setMaxWallTime("1:00:00"); + jobDescriptor.setMaxWallTime("60"); jobDescriptor.setQueueName("normal"); jobDescriptor.setAcountString("sds128"); List inputs = new ArrayList(); Modified: airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java URL: http://svn.apache.org/viewvc/airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java?rev=1530072&r1=1530071&r2=1530072&view=diff ============================================================================== --- airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java (original) +++ airavata/sandbox/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/VanilaSSHTest.java Mon Oct 7 21:22:26 2013 @@ -21,7 +21,6 @@ package org.apache.airavata.gsi.ssh.impl; -import com.jcraft.jsch.UserInfo; import org.apache.airavata.gsi.ssh.api.*; import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo; import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; @@ -32,10 +31,11 @@ import org.testng.AssertJUnit; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import javax.swing.*; import java.io.File; import java.util.ArrayList; +import java.util.Date; import java.util.List; +import java.util.UUID; /** * User: AmilaJ (amilaj@apache.org) @@ -56,10 +56,10 @@ public class VanilaSSHTest { this.hostName = "bigred2.uits.iu.edu"; System.setProperty("my.ssh.user", "lginnali"); - System.setProperty("my.ssh.user.password",""); -// System.setProperty("my.ssh.user","i want to be free"); + System.setProperty("my.ssh.user.password", ""); this.userName = System.getProperty("my.ssh.user"); this.password = System.getProperty("my.ssh.user.password"); + System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh"); // this.passPhrase = System.getProperty("my.ssh.user.pass.phrase"); if (this.userName == null || this.password == null) { @@ -72,7 +72,7 @@ public class VanilaSSHTest { @Test - public void testSimpleCommand1() throws Exception{ + public void testSimpleCommand1() throws Exception { System.out.println("Starting vanila SSH test ...."); @@ -94,7 +94,7 @@ public class VanilaSSHTest { } @Test - public void testSimpleCommand2() throws Exception{ + public void testSimpleCommand2() throws Exception { System.out.println("Starting vanila SSH test ...."); @@ -120,17 +120,34 @@ public class VanilaSSHTest { } @Test - public void testSimplePBSJob() throws Exception{ + public void testSimplePBSJob() throws Exception { AuthenticationInfo authenticationInfo = new DefaultPasswordAuthenticationInfo(this.password); - // Server info + // Server info ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName); Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/torque-4.2.3.1/bin/"); + String date = new Date().toString(); + date = date.replaceAll(" ", "_"); + date = date.replaceAll(":", "_"); - // Execute command + String pomFile = System.getProperty("basedir") + File.separator + "pom.xml"; + + // Constructing theworking directory for demonstration and creating directories in the remote + // resource String workingDirectory = File.separator + "N" + File.separator + "u" + File.separator + - "lginnali" + File.separator + "BigRed2" + File.separator + "myjob"; + "lginnali" + File.separator + "BigRed2"; + workingDirectory = workingDirectory + File.separator + + date + "_" + UUID.randomUUID(); + pbsCluster.makeDirectory(workingDirectory); + Thread.sleep(1000); + pbsCluster.makeDirectory(workingDirectory + File.separator + "inputs"); + Thread.sleep(1000); + pbsCluster.makeDirectory(workingDirectory + File.separator + "outputs"); + + + // doing file transfer to the remote resource + String s = pbsCluster.scpTo(workingDirectory + File.separator + "inputs", pomFile); // constructing the job object JobDescriptor jobDescriptor = new JobDescriptor(); jobDescriptor.setWorkingDirectory(workingDirectory); @@ -147,7 +164,7 @@ public class VanilaSSHTest { jobDescriptor.setMaxWallTime("5"); jobDescriptor.setJobSubmitter("aprun -n 1"); List inputs = new ArrayList(); - inputs.add("Hello World"); + inputs.add(s); jobDescriptor.setInputValues(inputs); //finished construction of job object System.out.println(jobDescriptor.toXML());