airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dimuthu...@apache.org
Subject [airavata] branch develop updated: Detecting zero byte files and ignoring data staging
Date Thu, 05 Apr 2018 18:28:12 GMT
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git


The following commit(s) were added to refs/heads/develop by this push:
     new 721a55a  Detecting zero byte files and ignoring data staging
721a55a is described below

commit 721a55abf5b7dff1260f4fd23395003b5460f5e0
Author: dimuthu <dimuthu.upeksha2@gmail.com>
AuthorDate: Thu Apr 5 14:28:04 2018 -0400

    Detecting zero byte files and ignoring data staging
---
 .../helix/impl/task/staging/DataStagingTask.java         | 12 +++++++++++-
 .../helix/impl/task/staging/InputDataStagingTask.java    | 10 ++++++++++
 .../helix/impl/task/staging/OutputDataStagingTask.java   | 16 ++++++++++++----
 3 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
index 99e1a41..e8c70e5 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
@@ -134,7 +134,7 @@ public abstract class DataStagingTask extends AiravataTask {
         return filePath;
     }
 
-    protected void transferFileToStorage(String sourcePath, String destPath, String fileName,
AgentAdaptor adaptor,
+    protected boolean transferFileToStorage(String sourcePath, String destPath, String fileName,
AgentAdaptor adaptor,
                               StorageResourceAdaptor storageResourceAdaptor) throws TaskOnFailException
{
         String localSourceFilePath = getLocalDataPath(fileName);
 
@@ -148,6 +148,15 @@ public abstract class DataStagingTask extends AiravataTask {
                         localSourceFilePath, true, e);
             }
 
+            File localFile = new File(localSourceFilePath);
+            if (localFile.exists()) {
+                if (localFile.length() == 0) {
+                    logger.warn("Local file " + localSourceFilePath +" size is 0 so ignoring
the upload");
+                    return false;
+                }
+            } else {
+                throw new TaskOnFailException("Local file does not exist at " + localSourceFilePath,
true, null);
+            }
             // Uploading output file to the storage resource
             try {
                 logger.info("Uploading the output file to " + destPath + " from local path
" + localSourceFilePath);
@@ -158,6 +167,7 @@ public abstract class DataStagingTask extends AiravataTask {
                         localSourceFilePath, true, e);
             }
 
+            return true;
         } finally {
             logger.info("Deleting temporary file " + localSourceFilePath);
             deleteTempFile(localSourceFilePath);
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
index b1c8915..4a8a307 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
@@ -103,6 +103,16 @@ public class InputDataStagingTask extends DataStagingTask {
                     throw new TaskOnFailException("Failed downloading input file " + sourceFileName
+ " to the local path " + localSourceFilePath, true, e);
                 }
 
+                File localFile = new File(localSourceFilePath);
+                if (localFile.exists()) {
+                    if (localFile.length() == 0) {
+                        logger.warn("Local file " + localSourceFilePath +" size is 0 so ignoring
the upload");
+                        return onSuccess("Input staging has skipped as file size is 0");
+                    }
+                } else {
+                    throw new TaskOnFailException("Local file does not exist at " + localSourceFilePath,
true, null);
+                }
+
                 // Uploading input file to the compute resource
                 try {
                     logger.info("Uploading the input file to " + destinationURI.getPath()
+ " from local path " + localSourceFilePath);
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
index 39894f2..2c2d201 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
@@ -151,16 +151,24 @@ public class OutputDataStagingTask extends DataStagingTask {
                     }
 
                     logger.info("Transferring file " + sourceFileName);
-                    transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(),
sourceFileName, adaptor, storageResourceAdaptor);
-                    saveExperimentOutput(processOutput.getName(), destinationURI.toString());
+                    boolean transferred = transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(),
sourceFileName, adaptor, storageResourceAdaptor);
+                    if (transferred) {
+                        saveExperimentOutput(processOutput.getName(), destinationURI.toString());
+                    } else {
+                        logger.warn("File " + sourceFileName + " did not transfer");
+                    }
                 }
                 return onSuccess("Output data staging task " + getTaskId() + " successfully
completed");
 
             } else {
                 // Downloading input file from the storage resource
                 assert processOutput != null;
-                transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(), sourceFileName,
adaptor, storageResourceAdaptor);
-                saveExperimentOutput(processOutput.getName(), destinationURI.toString());
+                boolean transferred = transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(),
sourceFileName, adaptor, storageResourceAdaptor);
+                if (transferred) {
+                    saveExperimentOutput(processOutput.getName(), destinationURI.toString());
+                } else {
+                    logger.warn("File " + sourceFileName + " did not transfer");
+                }
                 return onSuccess("Output data staging task " + getTaskId() + " successfully
completed");
             }
 

-- 
To stop receiving notification emails like this one, please contact
dimuthuupe@apache.org.

Mime
View raw message