airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sma...@apache.org
Subject airavata git commit: Improving Batch Queue Validations
Date Sat, 02 May 2015 19:03:06 GMT
Repository: airavata
Updated Branches:
  refs/heads/master 5cc8b43d0 -> 1e2f784a7


Improving Batch Queue Validations


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/1e2f784a
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/1e2f784a
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/1e2f784a

Branch: refs/heads/master
Commit: 1e2f784a711d1e0af8d0d91c3586269ccc2dcf7b
Parents: 5cc8b43
Author: Suresh Marru <smarru@apache.org>
Authored: Sat May 2 15:02:02 2015 -0400
Committer: Suresh Marru <smarru@apache.org>
Committed: Sat May 2 15:02:02 2015 -0400

----------------------------------------------------------------------
 .../main/resources/airavata-server.properties   |   2 +-
 .../test/resources/airavata-server.properties   |   2 +-
 .../main/resources/airavata-server.properties   |   2 +-
 .../src/test/resources/orchestrator.properties  |   2 +-
 .../src/test/resources/orchestrator.properties  |   2 +-
 .../validator/impl/BatchQueueValidator.java     | 146 +++++++++++++++++++
 .../validator/impl/SimpleAppDataValidator.java  | 132 -----------------
 7 files changed, 151 insertions(+), 137 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/1e2f784a/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index e604bd3..2df1531 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -247,7 +247,7 @@ rabbitmq.exchange.name=airavata_rabbitmq_exchange
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
 job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
 submitter.interval=10000
 threadpool.size=10
 start.submitter=true

http://git-wip-us.apache.org/repos/asf/airavata/blob/1e2f784a/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
b/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
index 37209a8..197d4e4 100644
--- a/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
+++ b/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
@@ -235,7 +235,7 @@ rabbitmq.exchange.name=airavata_rabbitmq_exchange
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
 job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
 submitter.interval=10000
 threadpool.size=10
 start.submitter=true

http://git-wip-us.apache.org/repos/asf/airavata/blob/1e2f784a/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
b/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
index 39f827d..580f598 100644
--- a/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
+++ b/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
@@ -218,7 +218,7 @@ rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
 job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
 submitter.interval=10000
 threadpool.size=10
 start.submitter=true

http://git-wip-us.apache.org/repos/asf/airavata/blob/1e2f784a/modules/gfac/airavata-gfac-service/src/test/resources/orchestrator.properties
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/test/resources/orchestrator.properties
b/modules/gfac/airavata-gfac-service/src/test/resources/orchestrator.properties
index 5ebfe4f..35c0427 100644
--- a/modules/gfac/airavata-gfac-service/src/test/resources/orchestrator.properties
+++ b/modules/gfac/airavata-gfac-service/src/test/resources/orchestrator.properties
@@ -18,7 +18,7 @@
 # under the License.
 #
 job.submitter=org.apache.airavata.orchestrator.core.impl.EmbeddedGFACJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator
 submitter.interval=10000
 threadpool.size=0
 start.submitter=true

http://git-wip-us.apache.org/repos/asf/airavata/blob/1e2f784a/modules/orchestrator/airavata-orchestrator-service/src/test/resources/orchestrator.properties
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/test/resources/orchestrator.properties
b/modules/orchestrator/airavata-orchestrator-service/src/test/resources/orchestrator.properties
index a1f0169..e84c429 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/test/resources/orchestrator.properties
+++ b/modules/orchestrator/airavata-orchestrator-service/src/test/resources/orchestrator.properties
@@ -18,7 +18,7 @@
 # under the License.
 #
 job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator
 submitter.interval=10000
 threadpool.size=0
 start.submitter=true

http://git-wip-us.apache.org/repos/asf/airavata/blob/1e2f784a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/BatchQueueValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/BatchQueueValidator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/BatchQueueValidator.java
new file mode 100644
index 0000000..8cd88fa
--- /dev/null
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/BatchQueueValidator.java
@@ -0,0 +1,146 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.orchestrator.core.validator.impl;
+
+import org.airavata.appcatalog.cpi.AppCatalog;
+import org.airavata.appcatalog.cpi.AppCatalogException;
+import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
+import org.apache.airavata.model.appcatalog.computeresource.BatchQueue;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.error.ValidationResults;
+import org.apache.airavata.model.error.ValidatorResult;
+import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
+import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
+import org.apache.airavata.registry.cpi.Registry;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class BatchQueueValidator implements JobMetadataValidator {
+    private final static Logger logger = LoggerFactory.getLogger(BatchQueueValidator.class);
+
+    private Registry registry;
+    private AppCatalog appCatalog;
+
+    public BatchQueueValidator() {
+        try {
+            this.registry = RegistryFactory.getDefaultRegistry();
+            this.appCatalog = AppCatalogFactory.getAppCatalog();
+        } catch (RegistryException e) {
+            logger.error("Unable to initialize registry", e);
+        } catch (AppCatalogException e) {
+            logger.error("Unable to initialize Application Catalog", e);
+        }
+    }
+
+    public ValidationResults validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail,
TaskDetails taskID) {
+        ValidationResults validationResults = new ValidationResults();
+        validationResults.setValidationState(true);
+        try {
+            List<ValidatorResult> validatorResultList = validateUserConfiguration(experiment,
taskID);
+            for (ValidatorResult result : validatorResultList){
+                if (!result.isResult()){
+                    validationResults.setValidationState(false);
+                    break;
+                }
+            }
+            validationResults.setValidationResultList(validatorResultList);
+        } catch (AppCatalogException e) {
+            logger.error("Error while validating user configuration", e);
+        }
+        return validationResults;
+    }
+
+    private List<ValidatorResult> validateUserConfiguration (Experiment experiment,
TaskDetails taskDetail) throws AppCatalogException{
+        List<ValidatorResult> validatorResultList = new ArrayList<ValidatorResult>();
+        try {
+            UserConfigurationData userConfigurationData = experiment.getUserConfigurationData();
+            ComputationalResourceScheduling computationalResourceScheduling = userConfigurationData.getComputationalResourceScheduling();
+            if (userConfigurationData.isAiravataAutoSchedule()) {
+                logger.error("Auto-Schedule is not yet supported. Experiment will proceed
with provided scheduling information");
+                ValidatorResult validatorResult = new ValidatorResult();
+                validatorResult.setResult(false);
+                validatorResultList.add(validatorResult);
+            }
+            ComputeResourceDescription computeResource = appCatalog.getComputeResource().getComputeResource(taskDetail.getTaskScheduling().getResourceHostId());
+            List<BatchQueue> batchQueues = computeResource.getBatchQueues();
+
+            if (batchQueues != null && !batchQueues.isEmpty()){
+                if (computationalResourceScheduling != null){
+                    String experimentQueueName = computationalResourceScheduling.getQueueName().trim();
+                    int experimentWallTimeLimit = computationalResourceScheduling.getWallTimeLimit();
+                    ValidatorResult queueNameResult = new ValidatorResult();
+
+                    //Set the validation to false. Once all the queue's are looped, if nothing
matches, then this gets passed.
+                    queueNameResult.setResult(false);
+                    queueNameResult.setErrorDetails("The specified queue" + experimentQueueName
+
+                            "does not exist. If you believe this is an error, contact the
administrator to verify App-Catalog Configurations");
+                    for (BatchQueue queue : batchQueues){
+                        String resourceQueueName = queue.getQueueName();
+                        int maxQueueRunTime = queue.getMaxRunTime();
+                        if (resourceQueueName != null && resourceQueueName.equals(experimentQueueName)){
+                            queueNameResult.setResult(true);
+                            queueNameResult.setErrorDetails("");
+
+                            //Validate if the specified wall time is within allowable limit
+                            ValidatorResult wallTimeResult = new ValidatorResult();
+                            if (maxQueueRunTime == 0) {
+                                wallTimeResult.setResult(true);
+                                wallTimeResult.setErrorDetails("Maximum wall time is not
configured for the queue," +
+                                        "Validation is being skipped");
+                                logger.info("Maximum wall time is not configured for the
queue" +
+                                        "Validation is being skipped");
+                            } else {
+                                if (maxQueueRunTime < experimentWallTimeLimit){
+                                    wallTimeResult.setResult(false);
+                                    wallTimeResult.setErrorDetails("Job Execution walltime
" + experimentWallTimeLimit +
+                                            "exceeds the allowable walltime"  + maxQueueRunTime
+
+                                            "for queue " + resourceQueueName);
+                                } else {
+                                    wallTimeResult.setResult(true);
+                                    wallTimeResult.setErrorDetails("");
+                                }
+                            }
+                            validatorResultList.add(wallTimeResult);
+                        }
+                    }
+                    validatorResultList.add(queueNameResult);
+
+                }
+            } else {
+                // for some compute resources, you dnt need to specify queue names
+                ValidatorResult result = new ValidatorResult();
+                logger.info("There are not queues defined under the compute resource. Airavata
assumes this experiment " +
+                        "does not need a queue name...");
+                result.setResult(true);
+                validatorResultList.add(result);
+            }
+        } catch (AppCatalogException e) {
+            logger.error("Error while getting information from App catalog", e);
+            throw new AppCatalogException("Error while getting information from App catalog",
e);
+        }
+        return validatorResultList;
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/1e2f784a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
deleted file mode 100644
index 21e2ac7..0000000
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.orchestrator.core.validator.impl;
-
-import org.airavata.appcatalog.cpi.AppCatalog;
-import org.airavata.appcatalog.cpi.AppCatalogException;
-import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
-import org.apache.airavata.model.appcatalog.computeresource.BatchQueue;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.airavata.model.error.ValidationResults;
-import org.apache.airavata.model.error.ValidatorResult;
-import org.apache.airavata.model.workspace.experiment.*;
-import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
-import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
-import org.apache.airavata.registry.cpi.Registry;
-import org.apache.airavata.registry.cpi.RegistryException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class SimpleAppDataValidator implements JobMetadataValidator {
-    private final static Logger logger = LoggerFactory.getLogger(SimpleAppDataValidator.class);
-
-    private Registry registry;
-    private AppCatalog appCatalog;
-
-    public SimpleAppDataValidator() {
-        try {
-            this.registry = RegistryFactory.getDefaultRegistry();
-            this.appCatalog = AppCatalogFactory.getAppCatalog();
-        } catch (RegistryException e) {
-            logger.error("Unable to initialize registry", e);
-        } catch (AppCatalogException e) {
-            logger.error("Unable to initialize Application Catalog", e);
-        }
-    }
-
-    public ValidationResults validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail,
TaskDetails taskID) {
-        ValidationResults validationResults = new ValidationResults();
-        validationResults.setValidationState(true);
-        try {
-            List<ValidatorResult> validatorResultList = validateUserConfiguration(experiment,
taskID);
-            for (ValidatorResult result : validatorResultList){
-                if (!result.isResult()){
-                    validationResults.setValidationState(false);
-                    break;
-                }
-            }
-            validationResults.setValidationResultList(validatorResultList);
-        } catch (AppCatalogException e) {
-            logger.error("Error while validating user configuration", e);
-        }
-        return validationResults;
-    }
-
-    private List<ValidatorResult> validateUserConfiguration (Experiment experiment,
TaskDetails taskDetail) throws AppCatalogException{
-        List<ValidatorResult> validatorResultList = new ArrayList<ValidatorResult>();
-        try {
-            UserConfigurationData userConfigurationData = experiment.getUserConfigurationData();
-            ComputationalResourceScheduling computationalResourceScheduling = userConfigurationData.getComputationalResourceScheduling();
-            if (userConfigurationData.isAiravataAutoSchedule()) {
-                logger.error("We dont' support auto scheduling at this point, We will simply
use user data as it is");
-                ValidatorResult validatorResult = new ValidatorResult();
-                validatorResult.setResult(false);
-                validatorResultList.add(validatorResult);
-            }
-            ComputeResourceDescription computeResource = appCatalog.getComputeResource().getComputeResource(taskDetail.getTaskScheduling().getResourceHostId());
-            List<BatchQueue> batchQueues = computeResource.getBatchQueues();
-
-            if (batchQueues != null && !batchQueues.isEmpty()){
-                if (computationalResourceScheduling != null){
-                    String queueName = computationalResourceScheduling.getQueueName().trim();
-                    int wallTimeLimit = computationalResourceScheduling.getWallTimeLimit();
-                    ValidatorResult queueNameResult = new ValidatorResult();
-
-                    queueNameResult.setResult(false);
-                    queueNameResult.setErrorDetails("Unable to find queue name from appcatalog
configured queues.. Either you " +
-                            "specified a wrong queue name or you did not configure app catalog
correctly...");
-                    for (BatchQueue queue : batchQueues){
-                        String configuredQname = queue.getQueueName();
-                        int maxRunTime = queue.getMaxRunTime();
-                        if (configuredQname != null && configuredQname.equals(queueName)){
-                            ValidatorResult wallTimeResult = new ValidatorResult();
-                            wallTimeResult.setResult(false);
-                            wallTimeResult.setErrorDetails("Provided walltime period exceeds
max walltime configured..");
-                            queueNameResult.setResult(true);
-                            queueNameResult.setErrorDetails("");
-                            if (maxRunTime != 0 && maxRunTime > wallTimeLimit){
-                                wallTimeResult.setResult(true);
-                                wallTimeResult.setErrorDetails("");
-                            }
-                            validatorResultList.add(wallTimeResult);
-                        }
-                    }
-                    validatorResultList.add(queueNameResult);
-
-                }
-            } else {
-                // for some compute resources, you dnt need to specify queue names
-                ValidatorResult result = new ValidatorResult();
-                logger.info("There are not queues defined under the compute resource. Airavata
assumes this experiment " +
-                        "does not need a queue name...");
-                result.setResult(true);
-                validatorResultList.add(result);
-            }
-        } catch (AppCatalogException e) {
-            logger.error("Error while getting information from App catalog", e);
-            throw new AppCatalogException("Error while getting information from App catalog",
e);
-        }
-        return validatorResultList;
-    }
-}


Mime
View raw message