airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shame...@apache.org
Subject airavata git commit: Added template language to pre and post commands
Date Mon, 17 Oct 2016 21:15:51 GMT
Repository: airavata
Updated Branches:
  refs/heads/develop e04785a4c -> 8bc2dcc4e


Added template language to pre and post commands


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/8bc2dcc4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/8bc2dcc4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/8bc2dcc4

Branch: refs/heads/develop
Commit: 8bc2dcc4eb3a45c09fa32903c38bd97e349b4599
Parents: e04785a
Author: Shameera Rathnayaka <shameerainfo@gmail.com>
Authored: Mon Oct 17 17:15:06 2016 -0400
Committer: Shameera Rathnayaka <shameerainfo@gmail.com>
Committed: Mon Oct 17 17:15:41 2016 -0400

----------------------------------------------------------------------
 .../apache/airavata/gfac/core/GFacUtils.java    | 48 ++++++++++++--------
 .../apache/airavata/gfac/core/GroovyMap.java    |  9 +++-
 .../org/apache/airavata/gfac/core/Script.java   |  4 ++
 .../gfac/core/context/ProcessContext.java       | 20 +++++---
 .../airavata/gfac/core/GFacUtilsTest.java       | 13 ++++++
 .../impl/task/DefaultJobSubmissionTask.java     |  2 +-
 .../gfac/impl/task/ForkJobSubmissionTask.java   |  2 +-
 .../gfac/impl/task/LocalJobSubmissionTask.java  |  2 +-
 8 files changed, 69 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index fe9fd01..b69764e 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -21,6 +21,7 @@ package org.apache.airavata.gfac.core;
 
 import groovy.lang.Writable;
 import groovy.text.GStringTemplateEngine;
+import groovy.text.SimpleTemplateEngine;
 import groovy.text.TemplateEngine;
 import org.apache.airavata.common.exception.ApplicationSettingsException;
 import org.apache.airavata.common.utils.*;
@@ -64,9 +65,6 @@ import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
-import javax.xml.transform.*;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
 import javax.xml.xpath.*;
 import java.io.*;
 import java.net.InetAddress;
@@ -416,8 +414,11 @@ public class GFacUtils {
     public static String getZKGfacServersParentPath() {
         return ZKPaths.makePath(ZkConstants.ZOOKEEPER_SERVERS_NODE, ZkConstants.ZOOKEEPER_GFAC_SERVER_NODE);
     }
-
-    public static GroovyMap creatGroovyMap(ProcessContext processContext, TaskContext taskContext)
+    public static GroovyMap crateGroovyMap(ProcessContext processContext)
+            throws ApplicationSettingsException, AppCatalogException, GFacException {
+        return createGroovyMap(processContext, null);
+    }
+    public static GroovyMap createGroovyMap(ProcessContext processContext, TaskContext taskContext)
             throws GFacException, AppCatalogException, ApplicationSettingsException {
 
         GroovyMap groovyMap = new GroovyMap();
@@ -430,6 +431,10 @@ public class GFacUtils {
         groovyMap.add(Script.EXECUTABLE_PATH, processContext.getApplicationDeploymentDescription().getExecutablePath());
         groovyMap.add(Script.STANDARD_OUT_FILE, processContext.getStdoutLocation());
         groovyMap.add(Script.STANDARD_ERROR_FILE, processContext.getStderrLocation());
+        groovyMap.add(Script.SCRATCH_LOCATION, processContext.getScratchLocation());
+        groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
+        groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName());
+        groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName());
 
         ComputeResourcePreference crp = getComputeResourcePreference(processContext);
         if (isValid(crp.getAllocationProjectNumber())) {
@@ -448,14 +453,16 @@ public class GFacUtils {
         groovyMap.add(Script.USER_NAME, processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName());
         groovyMap.add(Script.SHELL_NAME, "/bin/bash");
         // get walltime
-        try {
-            JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel());
-            if (jobSubmissionTaskModel.getWallTime() > 0) {
-                groovyMap.add(Script.MAX_WALL_TIME,
-                        GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
+        if (taskContext != null) {
+            try {
+                JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel)
taskContext.getSubTaskModel());
+                if (jobSubmissionTaskModel.getWallTime() > 0) {
+                    groovyMap.add(Script.MAX_WALL_TIME,
+                            GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
+                }
+            } catch (TException e) {
+                log.error("Error while getting job submission sub task model", e);
             }
-        } catch (TException e) {
-            log.error("Error while getting job submissiont sub task model", e);
         }
 
         // NOTE: Give precedence to data comes with experiment
@@ -522,7 +529,7 @@ public class GFacUtils {
         if (preJobCommands != null) {
             List<String> preJobCmdCollect = preJobCommands.stream()
                     .sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder())
-                    .map(map -> map.getCommand())
+                    .map(map -> parseCommands(map.getCommand(), groovyMap))
                     .collect(Collectors.toList());
             groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect);
         }
@@ -531,7 +538,7 @@ public class GFacUtils {
         if (postJobCommands != null) {
             List<String> postJobCmdCollect = postJobCommands.stream()
                     .sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder())
-                    .map(map -> map.getCommand())
+                    .map(map -> parseCommands(map.getCommand(), groovyMap))
                     .collect(Collectors.toList());
             groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect);
         }
@@ -703,11 +710,14 @@ public class GFacUtils {
         return i;
     }
 
-    private static String parseCommand(String value, ProcessContext context) {
-        String parsedValue = value.replaceAll("\\$workingDir", context.getWorkingDir());
-        parsedValue = parsedValue.replaceAll("\\$inputDir", context.getInputDir());
-        parsedValue = parsedValue.replaceAll("\\$outputDir", context.getOutputDir());
-        return parsedValue;
+    static String parseCommands(String value, GroovyMap bindMap) {
+        TemplateEngine templateEngine = new GStringTemplateEngine();
+        try {
+            return templateEngine.createTemplate(value).make(bindMap).toString();
+        } catch (ClassNotFoundException | IOException e) {
+            throw new IllegalArgumentException("Error while parsing command " + value
+                    + " , Invalid command or incomplete bind map");
+        }
     }
 
     public static ResourceJobManager getResourceJobManager(ProcessContext processContext)
{

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
index d618d40..1abc878 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
@@ -45,7 +45,8 @@ public class GroovyMap extends HashMap<String, Object> {
 
     public GroovyMap() {
         super();
-        addDefaultValues(); // to mitigate groovy exception groovy.lang.MissingPropertyException:
No such property: <name> for class: groovy.lang.Binding
+        // to mitigate groovy exception groovy.lang.MissingPropertyException: No such property:
<name> for class: groovy.lang.Binding
+        addDefaultValues();
     }
 
     public GroovyMap add(Script name, Object value){
@@ -77,6 +78,7 @@ public class GroovyMap extends HashMap<String, Object> {
                 .add(Script.RESERVATION, null)
                 .add(Script.EXPORTS, null)
                 .add(Script.MODULE_COMMANDS, null)
+                .add(Script.SCRATCH_LOCATION, null)
                 .add(Script.WORKING_DIR, null)
                 .add(Script.PRE_JOB_COMMANDS, null)
                 .add(Script.JOB_SUBMITTER_COMMAND, null)
@@ -88,7 +90,10 @@ public class GroovyMap extends HashMap<String, Object> {
                 .add(Script.CHASSIS_NAME, null)
                 .add(Script.INPUT_DIR, null)
                 .add(Script.OUTPUT_DIR, null)
-                .add(Script.USER_NAME, null);
+                .add(Script.USER_NAME, null)
+                .add(Script.GATEWAY_ID, null)
+                .add(Script.GATEWAY_USER_NAME, null)
+                .add(Script.APPLICATION_NAME, null);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
index 1e3ecd9..3ac5797 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
@@ -54,6 +54,7 @@ public enum Script {
     RESERVATION("reservation"),
     EXPORTS("exports"),
     MODULE_COMMANDS("moduleCommands"),
+    SCRATCH_LOCATION("scratchLocation"),
     WORKING_DIR("workingDirectory"),
     PRE_JOB_COMMANDS("preJobCommands"),
     JOB_SUBMITTER_COMMAND("jobSubmitterCommand"),
@@ -66,6 +67,9 @@ public enum Script {
     INPUT_DIR("inputDir"),
     OUTPUT_DIR("outputDir"),
     USER_NAME("userName"),
+    GATEWAY_ID("gatewayId"),
+    GATEWAY_USER_NAME("gatewayUserName"),
+    APPLICATION_NAME("applicationName"),
     ;
 
     String name;

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
index 8a2cc4e..8079e2a 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
@@ -64,6 +64,7 @@ public class ProcessContext {
 	private final String tokenId;
 	private ProcessModel processModel;
 	private String workingDir;
+	private String scratchLocation;
 	private String inputDir;
 	private String outputDir;
 	private String localWorkingDir;
@@ -166,13 +167,7 @@ public class ProcessContext {
             if (processModel.getProcessResourceSchedule().getStaticWorkingDir() != null){
                 workingDir = processModel.getProcessResourceSchedule().getStaticWorkingDir();
             }else {
-                String scratchLocation = null;
-				String overrideScratchLocation = processModel.getProcessResourceSchedule().getOverrideScratchLocation();
-                if (overrideScratchLocation != null && !overrideScratchLocation.equals(""))
{
-					scratchLocation = overrideScratchLocation;
-				} else {
-					scratchLocation = computeResourcePreference.getScratchLocation();
-				}
+                String scratchLocation = getScratchLocation();
                 workingDir = (scratchLocation.endsWith("/") ? scratchLocation + processId
: scratchLocation + "/" +
                         processId);
             }
@@ -180,6 +175,16 @@ public class ProcessContext {
 		return workingDir;
 	}
 
+	public String getScratchLocation() {
+		if (scratchLocation == null) {
+			scratchLocation = processModel.getProcessResourceSchedule().getOverrideScratchLocation();
+			if(scratchLocation == null || scratchLocation.isEmpty()){
+				scratchLocation = computeResourcePreference.getScratchLocation();
+			}
+		}
+		return scratchLocation;
+	}
+
 	public void setWorkingDir(String workingDir) {
 		this.workingDir = workingDir;
 	}
@@ -524,5 +529,6 @@ public class ProcessContext {
 		this.recoveryWithCancel = recoveryWithCancel;
 	}
 
+
 }
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
index 3262f86..cf56780 100644
--- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
@@ -46,4 +46,17 @@ public class GFacUtilsTest {
         String shared = GFacUtils.getQoS(qos, "compute");
         Assert.assertNull(shared);
     }
+
+    @Test
+    public void parserCommandTest() throws Exception {
+        String command = "mkdir -p $scratchLocation/$gatewayId/$gatewayUserName/$applicationName";
+        GroovyMap groovyMap = new GroovyMap();
+        groovyMap.add(Script.SCRATCH_LOCATION, "/my/scratch");
+        groovyMap.add(Script.GATEWAY_ID, "seagrid");
+        groovyMap.add(Script.GATEWAY_USER_NAME, "John");
+        groovyMap.add(Script.APPLICATION_NAME, "gaussian");
+        String value = GFacUtils.parseCommands(command, groovyMap);
+        Assert.assertNotNull(value);
+        Assert.assertEquals("mkdir -p /my/scratch/seagrid/John/gaussian", value);
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
index cabb77f..657de00 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
@@ -72,7 +72,7 @@ public class DefaultJobSubmissionTask implements JobSubmissionTask {
 		    JobModel jobModel = processContext.getJobModel();
 		    jobModel.setTaskId(taskContext.getTaskId());
 		    RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
-		    GroovyMap groovyMap = GFacUtils.creatGroovyMap(processContext,taskContext);
+			GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
 			jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
 			ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
 		    JobManagerConfiguration jConfig = null;

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
index e231cb9..1b43116 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
@@ -64,7 +64,7 @@ public class ForkJobSubmissionTask implements JobSubmissionTask {
             JobModel jobModel = processContext.getJobModel();
             jobModel.setTaskId(taskContext.getTaskId());
             RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
-            GroovyMap groovyMap = GFacUtils.creatGroovyMap(processContext, taskContext);
+            GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
             jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
             ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
             JobManagerConfiguration jConfig = null;

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
index a8b8e39..7e989e4 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
@@ -68,7 +68,7 @@ public class LocalJobSubmissionTask implements JobSubmissionTask{
             jobModel.setTaskId(taskContext.getTaskId());
 
             RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
-            GroovyMap groovyMap = GFacUtils.creatGroovyMap(processContext,taskContext);
+            GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext,taskContext);
 
             String jobId = AiravataUtils.getId("JOB_ID_");
             jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());


Mime
View raw message