airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From machris...@apache.org
Subject [04/13] airavata git commit: fixing bug in moving checkpoint files
Date Tue, 23 May 2017 14:21:05 GMT
fixing bug in moving checkpoint files


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/1108b57c
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/1108b57c
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/1108b57c

Branch: refs/heads/develop
Commit: 1108b57c961ad2ffd684ed4bef6a8e7f84bf7088
Parents: 868fc55
Author: scnakandala <supun.nakandala@gmail.com>
Authored: Wed May 3 16:04:16 2017 -0400
Committer: scnakandala <supun.nakandala@gmail.com>
Committed: Wed May 3 17:19:16 2017 -0400

----------------------------------------------------------------------
 .../gfac/core/cluster/RemoteCluster.java        | 12 ++++++++-
 .../airavata/gfac/impl/BESRemoteCluster.java    | 22 +++++++++++++---
 .../airavata/gfac/impl/HPCRemoteCluster.java    | 27 ++++++++++++++++++++
 .../airavata/gfac/impl/LocalRemoteCluster.java  | 18 ++++++++++---
 .../gfac/impl/task/SCPDataStageTask.java        | 18 ++++++++++---
 5 files changed, 87 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
index 9a5c757..ebf5f6b 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
@@ -23,7 +23,6 @@ import com.jcraft.jsch.Session;
 import org.apache.airavata.gfac.core.GFacException;
 import org.apache.airavata.gfac.core.SSHApiException;
 import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
 import org.apache.airavata.model.status.JobStatus;
 
 import java.util.List;
@@ -79,6 +78,17 @@ public interface RemoteCluster { // FIXME: replace SSHApiException with
suitable
 							  boolean ignoreEmptyFile) throws GFacException;
 
 	/**
+	 * This method can be used to get the file name of a file giving the extension. It assumes
that there will be only
+	 * one file with that extension. In case if there are more than one file one random file
name from the matching ones
+	 * will be returned.
+	 * @param fileExtension
+	 * @param parentPath
+	 * @param session
+	 * @return
+	 */
+	public String getFileNameFromExtension(String fileExtension, String parentPath, Session
session) throws GFacException;
+
+	/**
 	 * This will create directories in computing resources
 	 *
 	 * @param directoryPath the full qualified path for the directory user wants to create

http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
index de0c70f..4f22c6a 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
@@ -22,10 +22,11 @@ package org.apache.airavata.gfac.impl;
 import com.jcraft.jsch.Session;
 import org.apache.airavata.gfac.core.GFacException;
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
-import org.apache.airavata.gfac.core.SSHApiException;
 import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
-import org.apache.airavata.gfac.core.cluster.*;
+import org.apache.airavata.gfac.core.cluster.AbstractRemoteCluster;
+import org.apache.airavata.gfac.core.cluster.CommandInfo;
+import org.apache.airavata.gfac.core.cluster.JobSubmissionOutput;
+import org.apache.airavata.gfac.core.cluster.ServerInfo;
 import org.apache.airavata.model.status.JobStatus;
 
 import java.util.List;
@@ -56,6 +57,21 @@ public class BESRemoteCluster extends AbstractRemoteCluster{
 
     }
 
+    /**
+     * This method can be used to get the file name of a file giving the extension. It assumes
that there will be only
+     * one file with that extension. In case if there are more than one file one random file
name from the matching ones
+     * will be returned.
+     *
+     * @param fileExtension
+     * @param parentPath
+     * @param session
+     * @return
+     */
+    @Override
+    public String getFileNameFromExtension(String fileExtension, String parentPath, Session
session) {
+        return null;
+    }
+
     @Override
     public void makeDirectory(String directoryPath) throws GFacException {
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
index c1bca29..3d972b0 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
@@ -190,6 +190,33 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
 		}
 	}
 
+	/**
+	 * This method can be used to get the file name of a file giving the extension. It assumes
that there will be only
+	 * one file with that extension. In case if there are more than one file one random file
name from the matching ones
+	 * will be returned.
+	 *
+	 * @param fileExtension
+	 * @param parentPath
+	 * @param session
+	 * @return
+	 */
+	@Override
+	public String getFileNameFromExtension(String fileExtension, String parentPath, Session
session) throws GFacException {
+		try {
+			List<String> fileNames = SSHUtils.listDirectory(parentPath, session);
+			for(String fileName : fileNames){
+				if(fileName.endsWith(fileExtension)){
+					return fileName;
+				}
+			}
+			log.warn("No matching file found for extension: " + fileExtension + " in the " + parentPath
+ " directory");
+			return null;
+		} catch (Exception e) {
+			e.printStackTrace();
+			throw new GFacException("Failed to list directory " + parentPath);
+		}
+	}
+
 	@Override
 	public void makeDirectory(String directoryPath) throws GFacException {
 		int retryCount = 0;

http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
index 878e3b6..64a96db 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
@@ -21,12 +21,9 @@ package org.apache.airavata.gfac.impl;
 
 import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.Session;
-import org.apache.airavata.common.exception.AiravataException;
 import org.apache.airavata.gfac.core.GFacException;
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
-import org.apache.airavata.gfac.core.SSHApiException;
 import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
 import org.apache.airavata.gfac.core.cluster.*;
 import org.apache.airavata.model.status.JobStatus;
 import org.slf4j.Logger;
@@ -131,6 +128,21 @@ public class LocalRemoteCluster extends AbstractRemoteCluster {
         }
     }
 
+    /**
+     * This method can be used to get the file name of a file giving the extension. It assumes
that there will be only
+     * one file with that extension. In case if there are more than one file one random file
name from the matching ones
+     * will be returned.
+     *
+     * @param fileExtension
+     * @param parentPath
+     * @param session
+     * @return
+     */
+    @Override
+    public String getFileNameFromExtension(String fileExtension, String parentPath, Session
session) {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public void makeDirectory(String directoryPath) throws GFacException {
         Path dirPath = Paths.get(directoryPath);

http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
index 4e4aa86..9a42124 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
@@ -133,6 +133,20 @@ public class SCPDataStageTask implements Task {
             URI sourceURI = new URI(subTaskModel.getSource());
             String fileName = sourceURI.getPath().substring(sourceURI.getPath().lastIndexOf(File.separator)
+ 1,
                     sourceURI.getPath().length());
+
+            authenticationInfo = Factory.getComputerResourceSSHKeyAuthentication(processContext);
+            ServerInfo serverInfo = processContext.getComputeResourceServerInfo();
+            Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo);
+
+            //Wildcard for file name. Has to find the correct name.
+            if(fileName.startsWith("*.")){
+                String temp = taskContext.getParentProcessContext().getDataMovementRemoteCluster()
+                        .getFileNameFromExtension(fileName.substring(2), inputPath, sshSession);
+                if(temp != null && temp != ""){
+                    fileName = temp;
+                }
+            }
+
             URI destinationURI = null;
             if (subTaskModel.getDestination().startsWith("dummy")) {
                 destinationURI = TaskUtils.getDestinationURI(taskContext, hostName, inputPath,
fileName);
@@ -149,11 +163,9 @@ public class SCPDataStageTask implements Task {
                 return status;
             }
 
-            authenticationInfo = Factory.getComputerResourceSSHKeyAuthentication(processContext);
+
             status = new TaskStatus(TaskState.COMPLETED);
 
-            ServerInfo serverInfo = processContext.getComputeResourceServerInfo();
-            Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo);
             if (processState == ProcessState.INPUT_DATA_STAGING) {
                 inputDataStaging(taskContext, sshSession, sourceURI, destinationURI);
                 status.setReason("Successfully staged input data");


Mime
View raw message