airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lah...@apache.org
Subject [1/2] git commit: fixing https://issues.apache.org/jira/browse/AIRAVATA-1200
Date Tue, 06 May 2014 17:43:16 GMT
Repository: airavata
Updated Branches:
  refs/heads/master 79750b7b5 -> 86f46412e


fixing https://issues.apache.org/jira/browse/AIRAVATA-1200


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/7ec932b5
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/7ec932b5
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/7ec932b5

Branch: refs/heads/master
Commit: 7ec932b5e48b9b1c42355ffc42abada59bed1eb3
Parents: a9e022b
Author: lahiru <lahiru@apache.org>
Authored: Tue May 6 13:42:40 2014 -0400
Committer: lahiru <lahiru@apache.org>
Committed: Tue May 6 13:42:40 2014 -0400

----------------------------------------------------------------------
 .../server/OrchestratorServerHandler.java       | 37 +++++++++++++++
 .../core/OrchestratorConfiguration.java         | 11 +++--
 .../core/context/OrchestratorContext.java       |  2 +-
 .../core/utils/OrchestratorUtils.java           |  3 +-
 .../core/validator/JobMetadataValidator.java    |  9 ++--
 .../impl/AbstractJobMetadataValidator.java      | 30 ------------
 .../validator/impl/SimpleAppDataValidator.java  | 40 ++++------------
 .../airavata/orchestrator/cpi/Orchestrator.java | 11 +++++
 .../cpi/impl/SimpleOrchestratorImpl.java        | 50 ++++++++++++--------
 .../orchestrator.cpi.service.thrift             |  4 +-
 10 files changed, 102 insertions(+), 95 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
index 073f65d..f456c65 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
@@ -107,6 +107,43 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface
{
         }
         return true;
     }
+
+
+    /**
+     * This method will validate the experiment before launching, if is failed we do not
run the launch in airavata
+     * thrift service (only if validation is enabled
+     * @param experimentId
+     * @return
+     * @throws TException
+     */
+    public boolean validateExperiment(String experimentId) throws TException {
+        //TODO: Write the Orchestrator implementaion
+        try {
+            List<TaskDetails> tasks = orchestrator.createTasks(experimentId);
+            if (tasks.size() > 1) {
+                log.info("There are multiple tasks for this experiment, So Orchestrator will
launch multiple Jobs");
+            }
+            List<String> ids = registry.getIds(RegistryModelType.WORKFLOW_NODE_DETAIL,WorkflowNodeConstants.EXPERIMENT_ID,experimentId);
+            for (String workflowNodeId : ids) {
+                WorkflowNodeDetails workflowNodeDetail = (WorkflowNodeDetails)registry.get(RegistryModelType.WORKFLOW_NODE_DETAIL,
workflowNodeId);
+                List<Object> taskDetailList = registry.get(RegistryModelType.TASK_DETAIL,
TaskDetailConstants.NODE_ID, workflowNodeId);
+                for (Object o : taskDetailList) {
+                    TaskDetails taskID = (TaskDetails) o;
+                    //iterate through all the generated tasks and performs the job submisssion+monitoring
+                    Experiment experiment = (Experiment) registry.get(RegistryModelType.EXPERIMENT,
experimentId);
+                    if (experiment == null) {
+                        log.error("Error retrieving the Experiment by the given experimentID:
" + experimentId);
+                        return false;
+                    }
+                    return orchestrator.validateExperiment(experiment, workflowNodeDetail,
taskID);
+                }
+            }
+
+        } catch (Exception e) {
+            throw new TException(e);
+        }
+        return false;
+    }
     public boolean terminateExperiment(String experimentId) throws TException {
     	try {
 			orchestrator.cancelExperiment(experimentId);

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
index fcb3552..b0cbe07 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
@@ -23,6 +23,7 @@ package org.apache.airavata.orchestrator.core;
 import org.apache.airavata.client.api.AiravataAPI;
 
 import java.net.URL;
+import java.util.List;
 
 /**
  * This keeps configuration of orchestrator, mostly this keep static
@@ -46,17 +47,17 @@ public class OrchestratorConfiguration {
 
     private boolean embeddedMode;
 
-    private String validatorClass;
+    private List<String> validatorClasses;
 
     private boolean enableValidation;
 
 
-    public String getValidatorClass() {
-        return validatorClass;
+    public List<String> getValidatorClasses() {
+        return validatorClasses;
     }
 
-    public void setValidatorClass(String validatorClass) {
-        this.validatorClass = validatorClass;
+    public void setValidatorClasses(List<String> validatorClasses) {
+        this.validatorClasses = validatorClasses;
     }
 
     public boolean isEmbeddedMode() {

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
index eb3f463..8de0482 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
@@ -29,7 +29,7 @@ import org.apache.airavata.registry.api.AiravataRegistry2;
 import org.apache.airavata.registry.cpi.Registry;
 
 /**
- * This is the context object used in orchestrator whic h
+ * This is the context object used in orchestrator which
  */
 public class OrchestratorContext {
     private List<GFACInstance> gfacInstanceList;

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
index 5abbe03..0af1ed8 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
@@ -21,6 +21,7 @@
 package org.apache.airavata.orchestrator.core.utils;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.airavata.common.exception.ApplicationSettingsException;
 import org.apache.airavata.common.utils.ServerSettings;
@@ -53,7 +54,7 @@ public class OrchestratorUtils {
         orchestratorConfiguration.setEmbeddedMode(Boolean.valueOf(ServerSettings.getSetting(OrchestratorConstants.EMBEDDED_MODE)));
         orchestratorConfiguration.setEnableValidation(Boolean.valueOf(ServerSettings.getSetting(OrchestratorConstants.ENABLE_VALIDATION)));
         if (orchestratorConfiguration.isEnableValidation()) {
-            orchestratorConfiguration.setValidatorClass((String) ServerSettings.getSetting(OrchestratorConstants.JOB_VALIDATOR));
+            orchestratorConfiguration.setValidatorClasses(Arrays.asList(ServerSettings.getSetting(OrchestratorConstants.JOB_VALIDATOR).split(",")));
         }
         return orchestratorConfiguration;
     }

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
index d365c87..1c4eaba 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
@@ -20,13 +20,12 @@
 */
 package org.apache.airavata.orchestrator.core.validator;
 
+import org.apache.airavata.model.workspace.experiment.Experiment;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
 import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
 
 public interface JobMetadataValidator {
 
-    boolean runBasicValidation(String experimentID)throws OrchestratorException;
-
-    boolean runAppSpecificValidation(String experimentID)throws OrchestratorException;
-
-    boolean validate(String experimentID) throws OrchestratorException;
+    boolean validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails
taskID) throws OrchestratorException;
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java
deleted file mode 100644
index e83d5c9..0000000
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.orchestrator.core.validator.impl;
-
-import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
-
-public abstract class AbstractJobMetadataValidator implements JobMetadataValidator {
-    public boolean runBasicValidation(String experimentID) {
-        //todo implement basic validation which is not an application specific
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
index 53ecfe8..a756a0a 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
@@ -21,13 +21,16 @@
 package org.apache.airavata.orchestrator.core.validator.impl;
 
 import org.apache.airavata.model.workspace.experiment.Experiment;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
 import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
+import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
 import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
 import org.apache.airavata.registry.cpi.Registry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class SimpleAppDataValidator extends AbstractJobMetadataValidator {
+public class SimpleAppDataValidator implements JobMetadataValidator {
     private final static Logger logger = LoggerFactory.getLogger(SimpleAppDataValidator.class);
 
     private Registry registry;
@@ -36,38 +39,11 @@ public class SimpleAppDataValidator extends AbstractJobMetadataValidator
{
         this.registry = RegistryFactory.getDefaultRegistry();
     }
 
-    public boolean runAppSpecificValidation(String experimentID) throws OrchestratorException{
-        // implement simple application specific validator to be used for
-        // all the applications.
-        return true;  //To change body of implemented methods use File | Settings | File
Templates.
-    }
-
-    public boolean validate(String experimentID) throws OrchestratorException {
+    public boolean validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail,
TaskDetails taskID) throws OrchestratorException {
         boolean result = false;
-        if (super.runBasicValidation(experimentID)) {
-
-            Experiment experiment = null;
-            try {
-                experiment = (Experiment) registry.get(org.apache.airavata.registry.cpi.RegistryModelType.EXPERIMENT,
experimentID);
-            } catch (Exception e) {
-                throw new OrchestratorException(e);
-            }
-            if (experiment.getUserConfigurationData().isAiravataAutoSchedule()) {
-                logger.error("We dont' support auto scheduling at this point, We will simply
use user data as it is");
-            }
-
-            /* todo like this do more validation and if they are suppose to fail return false
otherwise give some
-               log messages in server side logs
-             */
-            if (runAppSpecificValidation(experimentID)) {
-                return true;
-            }
-            String error = "Application data validation steps failed";
-            logger.error(error);
-            return false;
+        if (experiment.getUserConfigurationData().isAiravataAutoSchedule()) {
+            logger.error("We dont' support auto scheduling at this point, We will simply
use user data as it is");
         }
-        String error = "Basic validation steps failed";
-        logger.error(error);
-        return false;
+        return true;
     }
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
index d02155e..3a47585 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
@@ -34,6 +34,17 @@ import java.util.List;
 public interface Orchestrator {
 
     /**
+     * This method can be used to run all custom validators plugged in to the orchestrator
and make
+     * sure the experiment is ready to launch and if its not this will return false
+     * @param experiment
+     * @param workflowNodeDetail
+     * @param taskID
+     * @return boolean if the experiments are valids after executing all the validators return
true otherwise it iwll return false
+     * @throws OrchestratorException
+     */
+     boolean validateExperiment(Experiment experiment, WorkflowNodeDetails workflowNodeDetail,
+                            TaskDetails taskID) throws OrchestratorException;
+    /**
      * After creating the experiment Data user have the
      * experimentID as the handler to the experiment, during the launchExperiment
      * We just have to give the experimentID

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
index afd7425..08df639 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
@@ -49,9 +49,6 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
     // this is going to be null unless the thread count is 0
     private JobSubmitter jobSubmitter = null;
 
-    private JobMetadataValidator jobMetadataValidator = null;
-
-
 
     public SimpleOrchestratorImpl() throws OrchestratorException {
         try {
@@ -60,14 +57,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
                 Class<? extends JobSubmitter> aClass = Class.forName(submitterClass.trim()).asSubclass(JobSubmitter.class);
                 jobSubmitter = aClass.newInstance();
                 jobSubmitter.initialize(this.orchestratorContext);
-                String validatorClzz = this.orchestratorContext.getOrchestratorConfiguration().getValidatorClass();
-                if (this.orchestratorConfiguration.isEnableValidation()) {
-                    if (validatorClzz == null) {
-                        logger.error("Job validation class is not properly set, so Validation
will be turned off !");
-                    }
-                    Class<? extends JobMetadataValidator> vClass = Class.forName(validatorClzz.trim()).asSubclass(JobMetadataValidator.class);
-                    jobMetadataValidator = vClass.newInstance();
-                }
+
             } catch (Exception e) {
                 String error = "Error creating JobSubmitter in non threaded mode ";
                 logger.error(error);
@@ -123,7 +113,35 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
         return tasks;
     }
 
-	public void cancelExperiment(String experimentID)
+    public boolean validateExperiment(Experiment experiment, WorkflowNodeDetails workflowNodeDetail,
TaskDetails taskID) throws OrchestratorException {
+        if (this.orchestratorConfiguration.isEnableValidation()) {
+            List<String> validatorClzzez = this.orchestratorContext.getOrchestratorConfiguration().getValidatorClasses();
+            for (String validator : validatorClzzez) {
+                try {
+                    Class<? extends JobMetadataValidator> vClass = Class.forName(validator.trim()).asSubclass(JobMetadataValidator.class);
+                    JobMetadataValidator jobMetadataValidator = vClass.newInstance();
+                    if(jobMetadataValidator.validate(experiment, workflowNodeDetail, taskID)){
+                        logger.info("Validation of " + validator + " is SUCCESSFUL");
+                    }else {
+                        logger.error("Validation of " + validator + " is FAILED");
+                        return false;
+                    }
+                } catch (ClassNotFoundException e) {
+                    logger.error("Error loading the validation class: ", validator, e);
+                    return false;
+                } catch (InstantiationException e) {
+                   logger.error("Error loading the validation class: ", validator, e);
+                    return false;
+                } catch (IllegalAccessException e) {
+                   logger.error("Error loading the validation class: ", validator, e);
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    public void cancelExperiment(String experimentID)
 			throws OrchestratorException {
         throw new OrchestratorException(new OperationNotSupportedException());
     }
@@ -137,14 +155,6 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
         this.executor = executor;
     }
 
-    public JobMetadataValidator getJobMetadataValidator() {
-        return jobMetadataValidator;
-    }
-
-    public void setJobMetadataValidator(JobMetadataValidator jobMetadataValidator) {
-        this.jobMetadataValidator = jobMetadataValidator;
-    }
-
     public JobSubmitter getJobSubmitter() {
         return jobSubmitter;
     }

http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
b/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
index c03e81f..6dc452e 100644
--- a/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
+++ b/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
@@ -43,5 +43,7 @@ service OrchestratorService {
     **/
   bool launchExperiment (1: required string experimentId),
 
-  bool terminateExperiment (1: required string experimentId)
+  bool terminateExperiment (1: required string experimentId),
+
+  bool validateExperiment(1: required string experimentId)
 }
\ No newline at end of file


Mime
View raw message