airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chath...@apache.org
Subject airavata git commit: ignoring empty output files when output staging
Date Wed, 20 Jan 2016 19:43:20 GMT
Repository: airavata
Updated Branches:
  refs/heads/develop d79055a69 -> 462843a47


ignoring empty output files when output staging


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/462843a4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/462843a4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/462843a4

Branch: refs/heads/develop
Commit: 462843a47c276806a57bd197400f0664eef13dcd
Parents: d79055a
Author: Chathuri Wimalasena <chathuri@apache.org>
Authored: Wed Jan 20 14:44:27 2016 -0500
Committer: Chathuri Wimalasena <chathuri@apache.org>
Committed: Wed Jan 20 14:44:27 2016 -0500

----------------------------------------------------------------------
 .../apache/airavata/gfac/core/cluster/RemoteCluster.java |  2 +-
 .../org/apache/airavata/gfac/impl/BESRemoteCluster.java  |  2 +-
 .../org/apache/airavata/gfac/impl/HPCRemoteCluster.java  |  6 +++---
 .../apache/airavata/gfac/impl/LocalRemoteCluster.java    |  2 +-
 .../java/org/apache/airavata/gfac/impl/SSHUtils.java     |  4 ++--
 .../apache/airavata/gfac/impl/task/SCPDataStageTask.java | 11 ++---------
 .../apache/airavata/gfac/impl/task/utils/StreamData.java |  2 +-
 7 files changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
index 9e4544c..f50d660 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
@@ -70,7 +70,7 @@ public interface RemoteCluster { // FIXME: replace SSHApiException with
suitable
      * @param inOrOut direction to file transfer , to the remote cluster(DIRECTION.IN) or
from the remote cluster(DIRECTION.OUT)
 	 *
 	 */
-	public void scpThirdParty(String sourceFile, String destinationFile ,Session session , DIRECTION
inOrOut) throws SSHApiException;
+	public void scpThirdParty(String sourceFile, String destinationFile ,Session session , DIRECTION
inOrOut, boolean ignoreEmptyFile) throws SSHApiException;
 
 	/**
 	 * This will create directories in computing resources

http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
index 1e41eda..0dd69fe 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
@@ -52,7 +52,7 @@ public class BESRemoteCluster extends AbstractRemoteCluster{
     }
 
     @Override
-    public void scpThirdParty(String sourceFile, String destinationFile, Session session,
DIRECTION inOrOut) throws SSHApiException {
+    public void scpThirdParty(String sourceFile, String destinationFile, Session session,
DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException {
 
     }
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
index 022c8bc..d4988b5 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
@@ -165,16 +165,16 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
 	}
 
 	@Override
-	public void scpThirdParty(String sourceFile, String destinationFile, Session clientSession,
DIRECTION direction) throws SSHApiException {
+	public void scpThirdParty(String sourceFile, String destinationFile, Session clientSession,
DIRECTION direction, boolean ignoreEmptyFile) throws SSHApiException {
 		try {
 			if(!session.isConnected()){
 				session = getOpenSession();
 			}
 			log.info("Transferring from:" + sourceFile + " To: " + destinationFile);
             if (direction == DIRECTION.TO) {
-                SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session);
+                SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session,
ignoreEmptyFile);
             } else {
-                SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession);
+                SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession,
ignoreEmptyFile);
             }
         } catch (IOException | JSchException e) {
 			throw new SSHApiException("Failed scp file:" + sourceFile + " to remote file "

http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
index d1aa6e0..1bab250 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
@@ -71,7 +71,7 @@ public class LocalRemoteCluster extends AbstractRemoteCluster {
     }
 
     @Override
-    public void scpThirdParty(String sourceFile, String destinationFile, Session session,
DIRECTION inOrOut) throws SSHApiException {
+    public void scpThirdParty(String sourceFile, String destinationFile, Session session,
DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException {
         throw new UnsupportedOperationException("Scp third party is not support with LocalRemoteCluster");
     }
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
index 2281ebc..a8e1017 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
@@ -272,7 +272,7 @@ public class SSHUtils {
      * @param destinationSession JSch Session for target
      * @return returns the final local file path of the new file came from the remote resource
      */
-    public static void scpThirdParty(String sourceFile, Session sourceSession, String destinationFile,
Session destinationSession) throws
+    public static void scpThirdParty(String sourceFile, Session sourceSession, String destinationFile,
Session destinationSession, boolean ignoreEmptyFile) throws
             IOException, JSchException {
         OutputStream sout = null;
         InputStream sin = null;
@@ -346,7 +346,7 @@ public class SSHUtils {
                         break;
                     }
                 }
-                if (fileSize == 0L){
+                if (fileSize == 0L && !ignoreEmptyFile){
                     String error = "Input file is empty...";
                     log.error(error);
                     throw new JSchException(error);

http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
index 605b972..71f689a 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
@@ -25,16 +25,10 @@ import com.jcraft.jsch.Session;
 import org.apache.airavata.common.exception.AiravataException;
 import org.apache.airavata.common.exception.ApplicationSettingsException;
 import org.apache.airavata.common.utils.ThriftUtils;
-import org.apache.airavata.credential.store.credential.Credential;
-import org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.credential.store.store.CredentialStoreException;
 import org.apache.airavata.gfac.core.GFacException;
 import org.apache.airavata.gfac.core.GFacUtils;
 import org.apache.airavata.gfac.core.SSHApiException;
 import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
-import org.apache.airavata.gfac.core.authentication.SSHPasswordAuthentication;
 import org.apache.airavata.gfac.core.cluster.CommandInfo;
 import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
 import org.apache.airavata.gfac.core.cluster.RemoteCluster;
@@ -60,7 +54,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -258,7 +251,7 @@ public class SCPDataStageTask implements Task {
          * scp third party file transfer 'to' compute resource.
          */
         taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(),
-                destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.TO);
+                destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.TO, false);
     }
 
     private void outputDataStaging(TaskContext taskContext, Session sshSession, URI sourceURI,
URI destinationURI)
@@ -268,7 +261,7 @@ public class SCPDataStageTask implements Task {
          * scp third party file transfer 'from' comute resource.
          */
         taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(),
-                destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM);
+                destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM, true);
         // update output locations
         GFacUtils.saveExperimentOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(),
destinationURI.getPath());
         GFacUtils.saveProcessOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(),
destinationURI.getPath());

http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
index d89c7d3..76b678a 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
@@ -158,7 +158,7 @@ public class StreamData extends TimerTask  {
          * scp third party file transfer 'from' comute resource.
          */
         taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(),
-                destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM);
+                destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM, true);
         // update output locations
         GFacUtils.saveExperimentOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(),
destinationURI.getPath());
         GFacUtils.saveProcessOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(),
destinationURI.getPath());


Mime
View raw message