airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shame...@apache.org
Subject [08/15] git commit: Added job count validation enable/disable capability
Date Fri, 26 Sep 2014 15:19:54 GMT
Added job count validation enable/disable capability


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/a78137b7
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/a78137b7
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/a78137b7

Branch: refs/heads/orchestratorJobThrottleFeature
Commit: a78137b789e4573e57fd614ea015fd18ca43ecbb
Parents: d899c1c
Author: shamrath <shameerainfo@gmail.com>
Authored: Thu Sep 18 14:55:04 2014 -0400
Committer: shamrath <shameerainfo@gmail.com>
Committed: Thu Sep 18 14:55:04 2014 -0400

----------------------------------------------------------------------
 .../airavata/common/utils/ServerSettings.java   |  5 ++++
 .../main/resources/airavata-server.properties   |  3 ++-
 .../handlers/GridPullMonitorHandler.java        |  4 ++-
 .../monitor/impl/pull/qstat/HPCPullMonitor.java | 20 +++++++-------
 .../core/validator/impl/JobCountValidator.java  | 28 +++++++++++++++-----
 5 files changed, 42 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index e607cdc..6c7bb3b 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -71,6 +71,7 @@ public class ServerSettings extends ApplicationSettings {
     private static final String MY_PROXY_LIFETIME = "myproxy.life";
     private static final String ACTIVITY_LISTENERS = "activity.listeners";
 
+    private static final String ENABLE_JOB_RESTRICTION_VALIDATION = "enable.job.restriction.validation";
     private static final String GLOBAL_MAX_JOB_COUNT = "global.max.job.count";
 
     private static boolean stopAllThreads = false;
@@ -107,6 +108,10 @@ public class ServerSettings extends ApplicationSettings {
         return getSetting(GLOBAL_MAX_JOB_COUNT);
     }
 
+    public static String getEnableJobRestrictionValidation() {
+        return getSetting(ENABLE_JOB_RESTRICTION_VALIDATION, "false");
+    }
+
     public static String getCredentialStoreDBUser() throws ApplicationSettingsException {
         try {
             return getSetting(CREDENTIAL_STORE_DB_USER);

http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index cd4a2b4..42b1bc8 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -193,12 +193,13 @@ activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdato
 ###---------------------------Orchestrator module Configurations---------------------------###
 #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
 job.submitter=org.apache.airavata.orchestrator.core.impl.GFACServiceJobSubmitter
-job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator,org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator,org.apache.airavata.orchestrator.core.validator.impl.JobCountValidator
 submitter.interval=10000
 threadpool.size=10
 start.submitter=true
 embedded.mode=true
 enable.validation=true
+enable.job.restriction.validation=true
 global.max.job.count=50
 orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
index 6428112..451466d 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
@@ -100,7 +100,9 @@ public class GridPullMonitorHandler extends ThreadedHandler implements
Watcher{
                 e.printStackTrace();
             }
             CommonUtils.addMonitortoQueue(hpcPullMonitor.getQueue(), monitorID);
-            CommonUtils.increaseZkJobCount(monitorID); // update change job count to zookeeper
+            if (ServerSettings.getEnableJobRestrictionValidation().equals("true")) {
+                CommonUtils.increaseZkJobCount(monitorID); // update change job count to
zookeeper
+            }
         } catch (AiravataMonitorException e) {
             logger.error("Error adding monitorID object to the queue with experiment ", monitorID.getExperimentID());
         }

http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
index b0fa11a..25a1ab2 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
@@ -272,17 +272,19 @@ public class HPCPullMonitor extends PullMonitor {
             ZooKeeper zk = null;
             for (MonitorID completedJob : completedJobs) {
                 CommonUtils.removeMonitorFromQueue(queue, completedJob);
-                if (zk == null) {
-                    zk = completedJob.getJobExecutionContext().getZk();
-                }
-                String key = CommonUtils.getJobCountUpdatePath(completedJob);
-                int i = 0;
-                if (jobRemoveCountMap.containsKey(key)) {
-                    i = Integer.valueOf(jobRemoveCountMap.get(key));
+                if (ServerSettings.getEnableJobRestrictionValidation().equals("true")) {
// is job restriction available?
+                    if (zk == null) {
+                        zk = completedJob.getJobExecutionContext().getZk();
+                    }
+                    String key = CommonUtils.getJobCountUpdatePath(completedJob);
+                    int i = 0;
+                    if (jobRemoveCountMap.containsKey(key)) {
+                        i = Integer.valueOf(jobRemoveCountMap.get(key));
+                    }
+                    jobRemoveCountMap.put(key, ++i);
                 }
-                jobRemoveCountMap.put(key, ++i);
             }
-            if (completedJobs.size() > 0) {
+            if (ServerSettings.getEnableJobRestrictionValidation().equals("true") &&
completedJobs.size() > 0) {
                 // reduce completed job count from zookeeper
                 CommonUtils.updateZkWithJobCount(zk, jobRemoveCountMap, false);
             }

http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
index 37890ec..2c66fa2 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java
@@ -1,3 +1,24 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
 package org.apache.airavata.orchestrator.core.validator.impl;
 
 import org.airavata.appcatalog.cpi.AppCatalog;
@@ -10,11 +31,6 @@ import org.apache.airavata.common.utils.AiravataUtils;
 import org.apache.airavata.common.utils.Constants;
 import org.apache.airavata.common.utils.RequestData;
 import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.credential.store.credential.AuditInfo;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.credential.store.store.CredentialReaderFactory;
-import org.apache.airavata.credential.store.store.CredentialStoreException;
-import org.apache.airavata.credential.store.util.TokenizedMyProxyAuthInfo;
 import org.apache.airavata.gfac.core.scheduler.HostScheduler;
 import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
 import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription;
@@ -28,12 +44,10 @@ import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
 import org.apache.airavata.orchestrator.core.context.OrchestratorContext;
 import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
 import org.apache.airavata.persistance.registry.jpa.model.TaskDetail;
-import org.apache.airavata.registry.cpi.RegistryException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;


Mime
View raw message