airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shame...@apache.org
Subject [13/15] git commit: Renamed enable.job.restriction.validation property name to enable.max.job.count removed jobcountvalidator class from validator list
Date Fri, 26 Sep 2014 15:19:59 GMT
Renamed enable.job.restriction.validation property name to enable.max.job.count removed jobcountvalidator
class from validator list


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/3cfc7114
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/3cfc7114
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/3cfc7114

Branch: refs/heads/orchestratorJobThrottleFeature
Commit: 3cfc7114bcd810c376496f150e32b7e2b51e4169
Parents: b8cacbc
Author: shamrath <shameerainfo@gmail.com>
Authored: Wed Sep 24 10:44:02 2014 -0400
Committer: shamrath <shameerainfo@gmail.com>
Committed: Wed Sep 24 10:44:02 2014 -0400

----------------------------------------------------------------------
 .../airavata/common/utils/ServerSettings.java   |  11 +-
 .../main/resources/airavata-server.properties   |   4 +-
 .../main/resources/airavata-server.properties   |   2 +-
 .../monitor/impl/pull/qstat/HPCPullMonitor.java |   4 +-
 .../core/validator/impl/JobCountValidator.java  | 154 -------------------
 .../cpi/impl/SimpleOrchestratorImpl.java        |   3 +-
 6 files changed, 9 insertions(+), 169 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/3cfc7114/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index 6c7bb3b..6be6ded 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -71,8 +71,7 @@ public class ServerSettings extends ApplicationSettings {
     private static final String MY_PROXY_LIFETIME = "myproxy.life";
     private static final String ACTIVITY_LISTENERS = "activity.listeners";
 
-    private static final String ENABLE_JOB_RESTRICTION_VALIDATION = "enable.job.restriction.validation";
-    private static final String GLOBAL_MAX_JOB_COUNT = "global.max.job.count";
+    private static final String ENABLE_MAX_JOB_COUNT_CHECK = "enable.max.job.count.check";
 
     private static boolean stopAllThreads = false;
 
@@ -104,12 +103,8 @@ public class ServerSettings extends ApplicationSettings {
         return getSetting(SERVER_CONTEXT_ROOT, "axis2");
     }
 
-    public static String getGlobalMaxJobCount() throws ApplicationSettingsException {
-        return getSetting(GLOBAL_MAX_JOB_COUNT);
-    }
-
-    public static String getEnableJobRestrictionValidation() {
-        return getSetting(ENABLE_JOB_RESTRICTION_VALIDATION, "false");
+    public static String getEnableMaxJobCountCheck() {
+        return getSetting(ENABLE_MAX_JOB_COUNT_CHECK, "false");
     }
 
     public static String getCredentialStoreDBUser() throws ApplicationSettingsException {

http://git-wip-us.apache.org/repos/asf/airavata/blob/3cfc7114/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index ba938f2..d1af845 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -193,13 +193,13 @@ activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdato
 ###---------------------------Orchestrator module Configurations---------------------------###
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
 job.submitter=org.apache.airavata.orchestrator.core.impl.GFACServiceJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator,org.apache.airavata.orchestrator.core.validator.impl.JobCountValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
 submitter.interval=10000
 threadpool.size=10
 start.submitter=true
 embedded.mode=true
 enable.validation=true
-enable.job.restriction.validation=true
+enable.max.job.count.check=true
 orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
 
 ###---------------------------API Server module Configurations---------------------------###

http://git-wip-us.apache.org/repos/asf/airavata/blob/3cfc7114/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
b/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
index bfc0504..40e8870 100644
--- a/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
+++ b/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
@@ -287,7 +287,7 @@ threadpool.size=10
 start.submitter=true
 embedded.mode=true
 enable.validation=true
-enable.job.restriction.validation=true
+enable.max.job.count.check=true
 orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
 
 ###---------------------------API Server module Configurations---------------------------###

http://git-wip-us.apache.org/repos/asf/airavata/blob/3cfc7114/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
index 022b0a6..5f83b43 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
@@ -266,7 +266,7 @@ public class HPCPullMonitor extends PullMonitor {
             ZooKeeper zk = null;
             for (MonitorID completedJob : completedJobs) {
                 CommonUtils.removeMonitorFromQueue(queue, completedJob);
-                if (ServerSettings.getEnableJobRestrictionValidation().equals("true")) {
// is job restriction available?
+                if (ServerSettings.getEnableMaxJobCountCheck().equals("true")) { // is job
restriction available?
                     TaskDetails taskDetails = completedJob.getJobExecutionContext().getTaskData();
                     ComputeResourceDescription computeResourceDesc = CommonUtils.getComputeResourceDescription(
                             taskDetails);
@@ -286,7 +286,7 @@ public class HPCPullMonitor extends PullMonitor {
                     }
                 }
             }
-            if (ServerSettings.getEnableJobRestrictionValidation().equals("true") &&
completedJobs.size() > 0) {
+            if (ServerSettings.getEnableMaxJobCountCheck().equals("true") && completedJobs.size()
> 0) {
                 // reduce completed job count from zookeeper
                 CommonUtils.updateZkWithJobCount(zk, jobRemoveCountMap, false);
             }

http://git-wip-us.apache.org/repos/asf/airavata/blob/3cfc7114/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
deleted file mode 100644
index f1cc5f9..0000000
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.orchestrator.core.validator.impl;
-
-import org.airavata.appcatalog.cpi.AppCatalog;
-import org.airavata.appcatalog.cpi.AppCatalogException;
-import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.AiravataUtils;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.common.utils.RequestData;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.credential.store.credential.AuditInfo;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.credential.store.store.CredentialReaderFactory;
-import org.apache.airavata.gfac.monitor.util.CommonUtils;
-import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
-import org.apache.airavata.model.appcatalog.computeresource.BatchQueue;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
-import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
-import org.apache.airavata.model.error.ValidatorResult;
-import org.apache.airavata.model.workspace.experiment.Experiment;
-import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
-import org.apache.airavata.orchestrator.core.context.OrchestratorContext;
-import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
-import org.apache.airavata.persistance.registry.jpa.model.TaskDetail;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Job count validator validate max job submission count for a given resource under given
community user name.
- */
-public class JobCountValidator implements JobMetadataValidator {
-    private static final Logger logger = LoggerFactory.getLogger(JobCountValidator.class);
-
-    @Override
-    public ValidatorResult validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail,
TaskDetails taskID,
-                                    String credStoreToken) {
-        ValidatorResult result;
-        try {
-            ComputeResourceDescription computeResourceDes = CommonUtils.getComputeResourceDescription(taskID);
-            if (computeResourceDes.getBatchQueuesSize() > 0) {
-                BatchQueue batchQueue = CommonUtils.getBatchQueueByName(computeResourceDes.getBatchQueues(),
-                        taskID.getTaskScheduling().getQueueName());
-                if (batchQueue == null) {
-                    throw new IllegalArgumentException("Invalid queue name, There is no queue
with name :" +
-                            taskID.getTaskScheduling().getQueueName());
-                }
-                int resourceMaxJobCount = batchQueue.getMaxJobsInQueue();
-                if (resourceMaxJobCount > 0) {
-                    for (JobSubmissionInterface jobSubmissionInterface : computeResourceDes.getJobSubmissionInterfaces())
{
-                        switch (jobSubmissionInterface.getJobSubmissionProtocol()) {
-                            case LOCAL:
-                                // nothing to do
-                                return new ValidatorResult(true);
-                            case SSH:
-                                SSHJobSubmission sshJobSubmission =
-                                        AppCatalogFactory.getAppCatalog().getComputeResource().getSSHJobSubmission(
-                                                jobSubmissionInterface.getJobSubmissionInterfaceId());
-                                switch (sshJobSubmission.getSecurityProtocol()) {
-                                    case GSI:
-                                        // gsi
-                                        RequestData requestData = new RequestData(ServerSettings.getDefaultUserGateway());
-                                        requestData.setTokenId(credStoreToken);
-                                        return isJobSpaceAvailable(requestData.getMyProxyUserName(),
-                                                computeResourceDes.getHostName(), batchQueue.getQueueName(),
resourceMaxJobCount);
-                                    case SSH_KEYS:
-                                        CredentialReader credentialReader = CredentialReaderFactory.createCredentialStoreReader();
-                                        AuditInfo auditInfo = credentialReader.getAuditInfo(experiment.getUserName(),
credStoreToken);
-                                        return isJobSpaceAvailable(auditInfo.getCommunityUser().getUserName(),
-                                                computeResourceDes.getHostName(), batchQueue.getQueueName(),
resourceMaxJobCount);
-                                    // ssh
-                                    default:
-                                        result = new ValidatorResult(false);
-                                        result.setErrorDetails("Doesn't support " + sshJobSubmission.getSecurityProtocol()
+
-                                                " protocol yet");
-                                        return result;
-                                }
-                            default:
-                                result = new ValidatorResult(false);
-                                result.setErrorDetails("Doesn't support " +
-                                        jobSubmissionInterface.getJobSubmissionProtocol()
+ " protocol yet");
-                                return result;
-                        }
-                    }
-                    result = new ValidatorResult(false);
-                    result.setErrorDetails("No JobSubmission interface found");
-                    return result;
-
-                }// end of inner if
-            }// end of outer if
-            return new ValidatorResult(true);
-        } catch (Exception e) {
-            logger.error("Exception occur while running job count validation process ", e);
-            result = new ValidatorResult(false);
-            result.setErrorDetails("Exception occur while running job count validation process
");
-            return result;
-        }
-
-    }
-
-    private ValidatorResult isJobSpaceAvailable(String communityUserName, String computeHostName,
String queueName, int resourceMaxJobCount)
-            throws ApplicationSettingsException {
-        if (communityUserName == null) {
-            throw new IllegalArgumentException("Community user name should not be null");
-        }
-        if (computeHostName == null) {
-            throw new IllegalArgumentException("Compute resource should not be null");
-        }
-        String keyPath = new StringBuilder("/" + Constants.STAT).append("/").append(communityUserName)
-                .append("/").toString();
-        String key = keyPath + computeHostName + "/" + Constants.JOB + "/" + queueName;
-        Map<String, Integer> jobCountMap = AiravataUtils.getJobCountMap(OrchestratorContext.getZk());
-        if (jobCountMap.containsKey(key)) {
-            int count = jobCountMap.get(key);
-            logger.info("Submitted job count = " + count + ", max job count = " + resourceMaxJobCount);
-            if (count < resourceMaxJobCount) {
-                return new ValidatorResult(true);
-            }
-        } else {
-            logger.info("Job count map doesn't has key : " + key);
-            return new ValidatorResult(true);
-        }
-        logger.info("Resource " + computeHostName + " doesn't has space to submit another
job, " +
-                "Configured resource max job count is " + resourceMaxJobCount + ".");
-        ValidatorResult result = new ValidatorResult(false);
-        result.setErrorDetails("Please honour to the gobal max job count " + resourceMaxJobCount);
-        return result;
-    }
-}

http://git-wip-us.apache.org/repos/asf/airavata/blob/3cfc7114/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
index c0760dd..53edc83 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
@@ -36,7 +36,6 @@ import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
 import org.apache.airavata.orchestrator.core.job.JobSubmitter;
 import org.apache.airavata.orchestrator.core.utils.OrchestratorUtils;
 import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
-import org.apache.airavata.orchestrator.core.validator.impl.JobCountValidator;
 import org.apache.airavata.registry.cpi.ChildDataType;
 import org.apache.airavata.registry.cpi.Registry;
 import org.apache.airavata.registry.cpi.RegistryModelType;
@@ -110,7 +109,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
         // creating monitorID to register with monitoring queue
         // this is a special case because amqp has to be in place before submitting the job
         try {
-            if (ServerSettings.getEnableJobRestrictionValidation().equals("true") &&
+            if (ServerSettings.getEnableMaxJobCountCheck().equals("true") &&
                     task.getTaskScheduling().getQueueName() != null) {
                 ComputeResourceDescription computeResourceDes = CommonUtils.getComputeResourceDescription(task);
                 String communityUserName = OrchestratorUtils.getCommunityUserName(experiment,
computeResourceDes, task,


Mime
View raw message