airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shame...@apache.org
Subject airavata git commit: Fixed yaml configuration parsing issues
Date Wed, 08 Jul 2015 20:29:52 GMT
Repository: airavata
Updated Branches:
  refs/heads/master c4c03f3fb -> eaf941ae4


Fixed yaml configuration parsing issues


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/eaf941ae
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/eaf941ae
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/eaf941ae

Branch: refs/heads/master
Commit: eaf941ae4112da96a5a15ed2dfa076de4002ea2d
Parents: c4c03f3
Author: Shameera Rathanyaka <shameerainfo@gmail.com>
Authored: Wed Jul 8 16:29:44 2015 -0400
Committer: Shameera Rathanyaka <shameerainfo@gmail.com>
Committed: Wed Jul 8 16:29:44 2015 -0400

----------------------------------------------------------------------
 .../server/src/main/resources/gfac-config.yaml  | 106 ++++++++++---------
 .../core/config/DataTransferTaskConfig.java     |  17 +++
 .../gfac/core/config/GFacYamlConfigruation.java |  35 ++++--
 .../core/config/JobSubmitterTaskConfig.java     |  16 +++
 .../org/apache/airavata/gfac/impl/Factory.java  |  35 ++++--
 5 files changed, 144 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/configuration/server/src/main/resources/gfac-config.yaml
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/gfac-config.yaml b/modules/configuration/server/src/main/resources/gfac-config.yaml
index 5530547..46ece9c 100644
--- a/modules/configuration/server/src/main/resources/gfac-config.yaml
+++ b/modules/configuration/server/src/main/resources/gfac-config.yaml
@@ -18,62 +18,72 @@
 #
 ################################################################
 
-config:
- jobSubmitters:
-   - submissionProtocol: SSH
-     taskClass: org.apache.airavata.task.adapters.SSHJobSubmissionTask
-     properties:
-      - password: pwd123
-      - passPhrase: test
-      - privateKey: key
-      - publicKey: pubkey
+jobSubmitters:
+  - submissionProtocol: SSH
+    taskClass: org.apache.airavata.gfac.impl.task.SSHJobSubmissionTask
+    properties:
+     - password: pwd123
+       passPhrase: test
+       privateKey: key
+       publicKey: pubk
+ # - submissionProtocol: GSISSH
+ #   taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask
 
-   - submissionProtocol: GSISSH
-     taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask
+  - submissionProtocol: LOCAL
+    taskClass: org.apache.airavata.gfac.impl.task.LocalJobSubmissionTask
 
- commonTasks:
-    - type: SETUP
-      taskClass: org.apache.airavata.task.common.SetupTask
+commonTasks:
+   - type: SETUP
+     taskClass: org.apache.airavata.task.common.SetupTask
 
-    - type: CLEANUP
-      taskClass: org.apache.airavata.task.common.CleanupTask
+   - type: CLEANUP
+     taskClass: org.apache.airavata.task.common.CleanupTask
 
- fileTransferTasks:
-   - transferProtocol: SCP
-     taskClass: org.apache.airavata.task.adapters.SCPFileTransferTask
-     properties:
-      - password: pwd123
-      - passPhrase: test
-      - privateKey: key
-      - publicKey: pubkey
+fileTransferTasks:
+  - transferProtocol: SCP
+    taskClass: org.apache.airavata.gfac.impl.task.SCPInputDataStageTask
+    properties:
+     - password: pwd123
+       passPhrase: test
+       privateKey: key
+       publicKey: pubkey
 
-   - transferProtocol: HTTP
-     taskClass: org.apache.airavata.task.adapters.HTTPFileTransferTask
+  #- transferProtocol: SFTP
+  #  taskClass: org.apache.airavata.task.adapters.SFTPFileTransferTask
 
-   - transferProtocol: GRIDFTP
-     taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask
+  #- transferProtocol: GRIDFTP
+  #  taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask
 
- resources:
-   - jobManagerType: PBS
-     commandOutputParser: org.apache.airavata.commmand.PBSOutputParser
-     emailParser: org.apache.airavata.email.parser.PBSEmailParser
-     resourceEmailAddresses:
-       - pbsconsult@sdsc.edu  # gordon
-       - adm@trident.bigred2.uits.iu.edu # Bigred2
-       - root <adm@trident.bigred2.uits.iu.edu> # Bigred2
-       - root <adm@scyld.localdomain> # alamo
+  #- transferProtocol: LOCAL
+  #  taskClass: org.apache.airavata.task.adapters.LocalFileTransferTask
 
-   - jobManagerType: SLURM
-     commandOutputParser: org.apache.airavata.commmand.SLURMOutputParser
-     emailParser: org.apache.airavata.email.parser.SLURMEmailParser
-     resourceEmailAddresses:
-       - SDSC Admin <slurm@comet-fe3.sdsc.edu> # comet
-       - slurm@batch1.stampede.tacc.utexas.edu # stampede
+resources:
+  - jobManagerType: PBS
+    commandOutputParser: org.apache.airavata.gfac.impl.job.PBSOutputParser
+    emailParser: org.apache.airavata.gfac.monitor.email.parser.PBSEmailParser
+    resourceEmailAddresses:
+      - pbsconsult@sdsc.edu  # gordon
+      - adm@trident.bigred2.uits.iu.edu # Bigred2
+      - root <adm@trident.bigred2.uits.iu.edu> # Bigred2
+      - root <adm@scyld.localdomain> # alamo
 
-   - jobManagerType: UGE
-     commandOutputParser: org.apache.airavata.commmand.UGEOutputParser
-     emailParser: org.apache.airavata.email.parser.UGEEmailParser
-     resourceEmailAddresses:
-       - ls4.tacc.utexas.edu # contain Lonestar
+  - jobManagerType: SLURM
+    commandOutputParser: org.apache.airavata.gfac.impl.job.SlurmOutputParser
+    emailParser: org.apache.airavata.gfac.monitor.email.parser.SLURMEmailParser
+    resourceEmailAddresses:
+      - SDSC Admin <slurm@comet-fe3.sdsc.edu> # comet
+      - slurm@batch1.stampede.tacc.utexas.edu # stampede
+
+  - jobManagerType: UGE
+    commandOutputParser: org.apache.airavata.gfac.impl.job.UGEOutputParser
+    emailParser: org.apache.airavata.gfac.monitor.email.parser.UGEEmailParser
+    resourceEmailAddresses:
+      - ls4.tacc.utexas.edu # contain Lonestar
+
+  - jobManagerType: LSF
+    commandOutputParser: org.apache.airavata.gfac.impl.job.LSFOutputParser
+    emailParser: org.apache.airavata.gfac.monitor.email.parser.LSFEmailParser
+    resourceEmailAddresses:
+      - iu.xsede.edu # test resource mail address
 
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
index a4817fb..76728f0 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
@@ -22,9 +22,14 @@ package org.apache.airavata.gfac.core.config;
 
 import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
 
+import java.util.HashMap;
+import java.util.Map;
+
 public class DataTransferTaskConfig {
 	private DataMovementProtocol transferProtocol;
 	private String taskClass;
+	private Map<String,String> properties = new HashMap<>();
+
 
 	public DataMovementProtocol getTransferProtocol() {
 		return transferProtocol;
@@ -41,4 +46,16 @@ public class DataTransferTaskConfig {
 	public void setTaskClass(String taskClass) {
 		this.taskClass = taskClass;
 	}
+
+	public void addProperty(String key, String value) {
+		properties.put(key, value);
+	}
+
+	public void addProperties(Map<String, String> propMap) {
+		propMap.forEach(properties::put);
+	}
+
+	public Map<String,String> getProperties(){
+		return properties;
+	}
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
index 5dbad87..fa8ce32 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
@@ -46,6 +46,7 @@ public class GFacYamlConfigruation {
 	private static final String COMMAND_OUTPUT_PARSER = "commandOutputParser";
 	private static final String EMAIL_PARSER = "emailParser";
 	private static final String RESOURCE_EMAIL_ADDRESS = "resourceEmailAddress";
+	private static final String PROPERTIES = "properties";
 
 	private List<JobSubmitterTaskConfig> jobSubmitters = new ArrayList<>();
 	private List<DataTransferTaskConfig> fileTransferTasks = new ArrayList<>();
@@ -71,26 +72,44 @@ public class GFacYamlConfigruation {
 		if (load instanceof Map) {
 			Map<String, Object> loadMap = (Map<String, Object>) load;
 			String identifier;
-			List<Map<String, String>> jobSubYamls = (List<Map<String, String>>)
loadMap.get(JOB_SUBMITTERS);
+			List<Map<String,Object >> jobSubYamls = (List<Map<String, Object>>)
loadMap.get(JOB_SUBMITTERS);
 			JobSubmitterTaskConfig jobSubmitterTaskConfig;
 			if (jobSubYamls != null) {
-				for (Map<String, String> jobSub : jobSubYamls) {
+				for (Map<String, Object> jobSub : jobSubYamls) {
 					jobSubmitterTaskConfig = new JobSubmitterTaskConfig();
-					identifier = jobSub.get (SUBMISSIO_PROTOCOL);
+					identifier = ((String) jobSub.get(SUBMISSIO_PROTOCOL));
 					jobSubmitterTaskConfig.setSubmissionProtocol(JobSubmissionProtocol.valueOf(identifier));
-					jobSubmitterTaskConfig.setTaskClass(jobSub.get(TASK_CLASS));
+					jobSubmitterTaskConfig.setTaskClass(((String) jobSub.get(TASK_CLASS)));
+					Object propertiesObj = jobSub.get(PROPERTIES);
+					List propertiesList;
+					if (propertiesObj instanceof List) {
+						propertiesList = ((List) propertiesObj);
+						if (propertiesList.size() > 0) {
+							Map<String, String> props = (Map<String, String>) propertiesList.get(0);
+							jobSubmitterTaskConfig.addProperties(props);
+						}
+					}
 					jobSubmitters.add(jobSubmitterTaskConfig);
 				}
 			}
 
-			List<Map<String, String>> fileTransYamls = (List<Map<String, String>>)
loadMap.get(FILE_TRANSFER_TASKS);
+			List<Map<String, Object>> fileTransYamls = (List<Map<String, Object>>)
loadMap.get(FILE_TRANSFER_TASKS);
 			DataTransferTaskConfig dataTransferTaskConfig;
 			if (fileTransYamls != null) {
-				for (Map<String, String> fileTransConfig : fileTransYamls) {
+				for (Map<String, Object> fileTransConfig : fileTransYamls) {
 					dataTransferTaskConfig = new DataTransferTaskConfig();
-					identifier = fileTransConfig.get (TRANSFER_PROTOCOL);
+					identifier = ((String) fileTransConfig.get(TRANSFER_PROTOCOL));
 					dataTransferTaskConfig.setTransferProtocol(DataMovementProtocol.valueOf(identifier));
-					dataTransferTaskConfig.setTaskClass(fileTransConfig.get(TASK_CLASS));
+					dataTransferTaskConfig.setTaskClass(((String) fileTransConfig.get(TASK_CLASS)));
+					Object propertiesObj = fileTransConfig.get(PROPERTIES);
+					List propertiesList;
+					if (propertiesObj instanceof List) {
+						propertiesList = (List) propertiesObj;
+						if (propertiesList.size() > 0) {
+							Map<String, String> props = (Map<String, String>) propertiesList.get(0);
+							dataTransferTaskConfig.addProperties(props);
+						}
+					}
 					fileTransferTasks.add(dataTransferTaskConfig);
 				}
 			}

http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
index daea282..6377f88 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
@@ -22,9 +22,13 @@ package org.apache.airavata.gfac.core.config;
 
 import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
 
+import java.util.HashMap;
+import java.util.Map;
+
 public class JobSubmitterTaskConfig {
 	private JobSubmissionProtocol submissionProtocol;
 	private String taskClass;
+	private Map<String,String> properties = new HashMap<>();
 
 	public JobSubmissionProtocol getSubmissionProtocol() {
 		return submissionProtocol;
@@ -41,4 +45,16 @@ public class JobSubmitterTaskConfig {
 	public void setTaskClass(String taskClass) {
 		this.taskClass = taskClass;
 	}
+
+	public void addProperty(String key, String value) {
+		properties.put(key, value);
+	}
+
+	public void addProperties(Map<String, String> propMap) {
+		propMap.forEach(properties::put);
+	}
+
+	public Map<String,String> getProperties(){
+		return properties;
+	}
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
index 2b629bb..fbb8f2f 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
@@ -40,6 +40,7 @@ import org.apache.airavata.gfac.core.monitor.JobMonitor;
 import org.apache.airavata.gfac.core.scheduler.HostScheduler;
 import org.apache.airavata.gfac.core.task.JobSubmissionTask;
 import org.apache.airavata.gfac.core.task.Task;
+import org.apache.airavata.gfac.core.task.TaskException;
 import org.apache.airavata.gfac.impl.job.LSFJobConfiguration;
 import org.apache.airavata.gfac.impl.job.LSFOutputParser;
 import org.apache.airavata.gfac.impl.job.PBSJobConfiguration;
@@ -67,6 +68,8 @@ import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
@@ -259,18 +262,32 @@ public abstract class Factory {
 
 	private static void loadConfiguration() throws GFacException {
 		GFacYamlConfigruation config = new GFacYamlConfigruation();
-		for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) {
-			jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), null);
-		}
+		try {
+			for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) {
+				String taskClass = jobSubmitterTaskConfig.getTaskClass();
+				Class<?> aClass = Class.forName(taskClass);
+				Constructor<?> constructor = aClass.getConstructor();
+				JobSubmissionTask task = (JobSubmissionTask) constructor.newInstance();
+				task.init(jobSubmitterTaskConfig.getProperties());
+				jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), task);
+			}
 
-		for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) {
-			dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), null);
-		}
+			for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) {
+				String taskClass = dataTransferTaskConfig.getTaskClass();
+				Class<?> aClass = Class.forName(taskClass);
+				Constructor<?> constructor = aClass.getConstructor();
+				Task task = (Task) constructor.newInstance();
+				task.init(dataTransferTaskConfig.getProperties());
+				dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), task);
+			}
 
-		for (ResourceConfig resourceConfig : config.getResourceConfiguration()) {
-			resources.put(resourceConfig.getJobManagerType(), resourceConfig);
+			for (ResourceConfig resourceConfig : config.getResourceConfiguration()) {
+				resources.put(resourceConfig.getJobManagerType(), resourceConfig);
+			}
+			readConfig = true;
+		} catch (Exception e) {
+			throw new GFacException("Gfac config issue", e);
 		}
-		readConfig = true;
 	}
 
 


Mime
View raw message