asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject asterixdb git commit: Add Parameter To Control Job Queue Capacity
Date Thu, 09 Mar 2017 17:30:02 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master df9ccbb79 -> 42fac37dd


Add Parameter To Control Job Queue Capacity

Exposed CC property job.queue.capacity (-job-queue-capacity) to enable
ability to override default capacity of 4k

Change-Id: I36d0727de58dbe9697e3693e49b39f8c8ab32ce8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1562
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/42fac37d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/42fac37d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/42fac37d

Branch: refs/heads/master
Commit: 42fac37dd7727486497def2c1d4f7bee55df1425
Parents: df9ccbb
Author: Michael Blow <mblow@apache.org>
Authored: Wed Mar 8 20:51:45 2017 -0500
Committer: Michael Blow <mblow@apache.org>
Committed: Thu Mar 9 09:28:38 2017 -0800

----------------------------------------------------------------------
 .../java/org/apache/hyracks/control/cc/job/IJobManager.java  | 4 ++++
 .../java/org/apache/hyracks/control/cc/job/JobManager.java   | 6 +++++-
 .../apache/hyracks/control/cc/scheduler/FIFOJobQueue.java    | 7 ++++---
 .../org/apache/hyracks/control/cc/job/JobManagerTest.java    | 1 +
 .../apache/hyracks/control/common/controllers/CCConfig.java  | 8 ++++++++
 5 files changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42fac37d/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
index 21fc08f..8fe542f 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/IJobManager.java
@@ -118,4 +118,8 @@ public interface IJobManager {
      */
     Collection<JobRun> getArchivedJobs();
 
+    /**
+     * @return the maximum number of jobs to queue before rejecting new jobs
+     */
+    int getJobQueueCapacity();
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42fac37d/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
index b35de3d..649487b 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
@@ -283,6 +283,11 @@ public class JobManager implements IJobManager {
         return runMapHistory.get(jobId);
     }
 
+    @Override
+    public int getJobQueueCapacity() {
+        return ccs.getCCConfig().getJobQueueCapacity();
+    }
+
     private void pickJobsToRun() throws HyracksException {
         List<JobRun> selectedRuns = jobQueue.pull();
         for (JobRun run : selectedRuns) {
@@ -335,5 +340,4 @@ public class JobManager implements IJobManager {
             throw HyracksException.create(ErrorCode.INVALID_INPUT_PARAMETER);
         }
     }
-
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42fac37d/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/scheduler/FIFOJobQueue.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/scheduler/FIFOJobQueue.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/scheduler/FIFOJobQueue.java
index 0377692..833baac 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/scheduler/FIFOJobQueue.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/scheduler/FIFOJobQueue.java
@@ -44,22 +44,23 @@ import org.apache.hyracks.control.cc.job.JobRun;
 public class FIFOJobQueue implements IJobQueue {
 
     private static final Logger LOGGER = Logger.getLogger(FIFOJobQueue.class.getName());
-    private static final int CAPACITY = 4096;
 
     private final Map<JobId, JobRun> jobListMap = new LinkedHashMap<>();
     private final IJobManager jobManager;
     private final IJobCapacityController jobCapacityController;
+    private final int jobQueueCapacity;
 
     public FIFOJobQueue(IJobManager jobManager, IJobCapacityController jobCapacityController)
{
         this.jobManager = jobManager;
         this.jobCapacityController = jobCapacityController;
+        this.jobQueueCapacity = jobManager.getJobQueueCapacity();
     }
 
     @Override
     public void add(JobRun run) throws HyracksException {
         int size = jobListMap.size();
-        if (size >= CAPACITY) {
-            throw HyracksException.create(ErrorCode.JOB_QUEUE_FULL, CAPACITY);
+        if (size >= jobQueueCapacity) {
+            throw HyracksException.create(ErrorCode.JOB_QUEUE_FULL, jobQueueCapacity);
         }
         jobListMap.put(run.getJobId(), run);
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42fac37d/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
index 97b05e7..1628248 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
@@ -285,6 +285,7 @@ public class JobManagerTest {
         when(ccs.getApplicationContext()).thenReturn(appCtx);
         when(ccs.getJobLogFile()).thenReturn(logFile);
         when(ccs.getNodeManager()).thenReturn(nodeManager);
+        when(ccs.getCCConfig()).thenReturn(ccConfig);
         return ccs;
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/42fac37d/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
index fbde58c..97438c7 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
@@ -35,6 +35,7 @@ import org.apache.hyracks.control.common.config.ConfigManager;
 import org.apache.hyracks.util.file.FileUtil;
 import org.ini4j.Ini;
 
+@SuppressWarnings("SameParameterValue")
 public class CCConfig extends ControllerConfig {
 
     public static String defaultDir = System.getProperty("java.io.tmpdir");
@@ -60,6 +61,7 @@ public class CCConfig extends ControllerConfig {
         ROOT_DIR(STRING, (Supplier<String>)() -> FileUtil.joinPath(defaultDir, "ClusterControllerService")),
         CLUSTER_TOPOLOGY(STRING),
         JOB_QUEUE_CLASS(STRING, "org.apache.hyracks.control.cc.scheduler.FIFOJobQueue"),
+        JOB_QUEUE_CAPACITY(INTEGER, 4096),
         JOB_MANAGER_CLASS(STRING, "org.apache.hyracks.control.cc.job.JobManager");
 
         private final IOptionType parser;
@@ -143,6 +145,8 @@ public class CCConfig extends ControllerConfig {
                     return "Sets the XML file that defines the cluster topology";
                 case JOB_QUEUE_CLASS:
                     return "Specify the implementation class name for the job queue";
+                case JOB_QUEUE_CAPACITY:
+                    return "The maximum number of jobs to queue before rejecting new jobs";
                 case JOB_MANAGER_CLASS:
                     return "Specify the implementation class name for the job manager";
                 default:
@@ -333,4 +337,8 @@ public class CCConfig extends ControllerConfig {
     public void setJobManagerClass(String jobManagerClass) {
         configManager.set(Option.JOB_MANAGER_CLASS, jobManagerClass);
     }
+
+    public int getJobQueueCapacity() {
+        return getAppConfig().getInt(Option.JOB_QUEUE_CAPACITY);
+    }
 }


Mime
View raw message