Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CEE3111268 for ; Tue, 6 May 2014 14:57:11 +0000 (UTC) Received: (qmail 79494 invoked by uid 500); 6 May 2014 14:38:04 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 79404 invoked by uid 500); 6 May 2014 14:38:03 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 79322 invoked by uid 99); 6 May 2014 14:38:00 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 May 2014 14:38:00 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id AE7DB935588; Tue, 6 May 2014 14:38:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: raminder@apache.org To: commits@airavata.apache.org Date: Tue, 06 May 2014 14:38:00 -0000 Message-Id: <6e358c15412e4bb9912254c74beedddc@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] git commit: Fixed output population. AIRAVATA-1185 Repository: airavata Updated Branches: refs/heads/master 14c1841ca -> ee79b70b6 Fixed output population. AIRAVATA-1185 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/9be225ac Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/9be225ac Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/9be225ac Branch: refs/heads/master Commit: 9be225ac4013ffea801b7d1b516fc09f830740f4 Parents: 0eabb93 Author: raminder Authored: Tue May 6 10:27:16 2014 -0400 Committer: raminder Committed: Tue May 6 10:27:16 2014 -0400 ---------------------------------------------------------------------- .../apache/airavata/gfac/utils/OutputUtils.java | 45 ++++++++++++++++ .../gfac/handler/GSISSHOutputHandler.java | 54 +++++++++++++------- .../airavata/gfac/handler/SSHOutputHandler.java | 28 ++++++---- 3 files changed, 98 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/9be225ac/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java index efcbcd5..fa47745 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java @@ -21,6 +21,7 @@ package org.apache.airavata.gfac.utils; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; @@ -30,6 +31,8 @@ import org.apache.airavata.common.utils.StringUtil; import org.apache.airavata.commons.gfac.type.ActualParameter; import org.apache.airavata.commons.gfac.type.MappingFactory; import org.apache.airavata.gfac.handler.GFacHandlerException; +import org.apache.airavata.model.workspace.experiment.DataObjectType; +import org.apache.airavata.schemas.gfac.DataType; import org.apache.airavata.schemas.gfac.StdErrParameterType; import org.apache.airavata.schemas.gfac.StdOutParameterType; @@ -71,6 +74,48 @@ public class OutputUtils { return result; } + + public static void fillOutputFromStdout1(Map output, String stdout, String stderr,List outputArray) throws Exception { + + if (stdout == null || stdout.equals("")){ + throw new GFacHandlerException("Standard output is empty."); + } + + Set keys = output.keySet(); + for (String paramName : keys) { + ActualParameter actual = (ActualParameter) output.get(paramName); + // if parameter value is not already set, we let it go + + if (actual == null) { + continue; + } + if ("StdOut".equals(actual.getType().getType().toString())) { + DataObjectType out = new DataObjectType(); + out.setKey(paramName); + out.setType(DataType.STD_OUT.toString()); + out.setValue(stdout); + outputArray.add(out); + } else if ("StdErr".equals(actual.getType().getType().toString())) { + DataObjectType out = new DataObjectType(); + out.setKey(paramName); + out.setType(DataType.STD_ERR.toString()); + out.setValue(stderr); + outputArray.add(out); + }else if("URI".equals(actual.getType().getType().toString())){ + continue; + } + else { + String parseStdout = parseStdout(stdout, paramName); + if (parseStdout != null) { + DataObjectType out = new DataObjectType(); + out.setKey(paramName); + out.setType(DataType.STRING.toString()); + out.setValue(parseStdout); + outputArray.add(out); + } + } + } + } private static String parseStdout(String stdout, String outParam) throws Exception { String regex = Pattern.quote(outParam) + regexPattern; http://git-wip-us.apache.org/repos/asf/airavata/blob/9be225ac/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java index fd80d56..6eff137 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java @@ -20,8 +20,18 @@ */ package org.apache.airavata.gfac.handler; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + import net.schmizz.sshj.connection.ConnectionException; import net.schmizz.sshj.transport.TransportException; + import org.apache.airavata.common.exception.ApplicationSettingsException; import org.apache.airavata.common.utils.Constants; import org.apache.airavata.commons.gfac.type.ActualParameter; @@ -35,7 +45,14 @@ import org.apache.airavata.gfac.utils.GFacUtils; import org.apache.airavata.gfac.utils.OutputUtils; import org.apache.airavata.gsi.ssh.api.Cluster; import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; -import org.apache.airavata.model.workspace.experiment.*; +import org.apache.airavata.model.workspace.experiment.CorrectiveAction; +import org.apache.airavata.model.workspace.experiment.DataObjectType; +import org.apache.airavata.model.workspace.experiment.DataTransferDetails; +import org.apache.airavata.model.workspace.experiment.ErrorCategory; +import org.apache.airavata.model.workspace.experiment.JobDetails; +import org.apache.airavata.model.workspace.experiment.TaskDetails; +import org.apache.airavata.model.workspace.experiment.TransferState; +import org.apache.airavata.model.workspace.experiment.TransferStatus; import org.apache.airavata.registry.cpi.ChildDataType; import org.apache.airavata.registry.cpi.DataType; import org.apache.airavata.registry.cpi.RegistryException; @@ -46,12 +63,8 @@ import org.apache.xmlbeans.XmlException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; public class GSISSHOutputHandler extends AbstractHandler{ private static final Logger log = LoggerFactory.getLogger(GSISSHOutputHandler.class); @@ -154,7 +167,7 @@ public class GSISSHOutputHandler extends AbstractHandler{ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); - Map stringMap = new HashMap(); + List outputArray = new ArrayList(); Map output = jobExecutionContext.getOutMessageContext().getParameters(); Set keys = output.keySet(); for (String paramName : keys) { @@ -163,31 +176,36 @@ public class GSISSHOutputHandler extends AbstractHandler{ List outputList = cluster.listDirectory(app.getOutputDataDirectory()); if (outputList.size() == 0 || outputList.get(0).isEmpty()) { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr, outputArray); + break; } else { String valueList = outputList.get(0); cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir); jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList); - ((URIParameterType) actualParameter.getType()).setValue(valueList); - stringMap = new HashMap(); - stringMap.put(paramName, actualParameter); + DataObjectType dataObjectType = new DataObjectType(); + dataObjectType.setValue(valueList); + dataObjectType.setKey(paramName); + dataObjectType.setType(org.apache.airavata.schemas.gfac.DataType.URI.toString()); + outputArray.add(dataObjectType); } } else { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + OutputUtils.fillOutputFromStdout1(output, stdOutStr,stdErrStr, outputArray); + break; } } - if (stringMap == null || stringMap.isEmpty()) { + if (outputArray == null || outputArray.isEmpty()) { throw new GFacHandlerException( "Empty Output returned from the Application, Double check the application" + "and ApplicationDescriptor output Parameter Names"); } - status.setTransferState(TransferState.DOWNLOAD); - detail.setTransferStatus(status); - registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); - app.setStandardError(localStdErrFile.getAbsolutePath()); app.setStandardOutput(localStdOutFile.getAbsolutePath()); app.setOutputDataDirectory(outputDataDir); + status.setTransferState(TransferState.DOWNLOAD); + detail.setTransferStatus(status); + detail.setTransferDescription(outputDataDir); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID()); } catch (XmlException e) { throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e); } catch (ConnectionException e) { http://git-wip-us.apache.org/repos/asf/airavata/blob/9be225ac/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java index 42cebba..c70469f 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java @@ -22,6 +22,7 @@ package org.apache.airavata.gfac.handler; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -150,7 +151,7 @@ public class SSHOutputHandler extends AbstractHandler{ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); - Map stringMap = new HashMap(); + List outputArray = new ArrayList(); Map output = jobExecutionContext.getOutMessageContext().getParameters(); Set keys = output.keySet(); for (String paramName : keys) { @@ -159,31 +160,36 @@ public class SSHOutputHandler extends AbstractHandler{ List outputList = cluster.listDirectory(app.getOutputDataDirectory()); if (outputList.size() == 0 || outputList.get(0).isEmpty()) { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray); + break; } else { String valueList = outputList.get(0); cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir); jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList); - ((URIParameterType) actualParameter.getType()).setValue(valueList); - stringMap = new HashMap(); - stringMap.put(paramName, actualParameter); + DataObjectType dataObjectType = new DataObjectType(); + dataObjectType.setValue(valueList); + dataObjectType.setKey(paramName); + dataObjectType.setType(org.apache.airavata.schemas.gfac.DataType.URI.toString()); + outputArray.add(dataObjectType); } } else { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray); } } - if (stringMap == null || stringMap.isEmpty()) { + if (outputArray == null || outputArray.isEmpty()) { throw new GFacHandlerException( "Empty Output returned from the Application, Double check the application" + "and ApplicationDescriptor output Parameter Names"); } - status.setTransferState(TransferState.DOWNLOAD); - detail.setTransferStatus(status); - registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); - app.setStandardError(localStdErrFile.getAbsolutePath()); app.setStandardOutput(localStdOutFile.getAbsolutePath()); app.setOutputDataDirectory(outputDataDir); + status.setTransferState(TransferState.DOWNLOAD); + detail.setTransferStatus(status); + detail.setTransferDescription(outputDataDir); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID()); + } catch (XmlException e) { throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e); } catch (ConnectionException e) {