hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)
Date Tue, 17 Oct 2017 23:04:32 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk f27a4ad03 -> 46eb1033a


YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46eb1033
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46eb1033
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46eb1033

Branch: refs/heads/trunk
Commit: 46eb1033a86ca53e7b94202567aef8af2417bdf8
Parents: f27a4ad
Author: Haibo Chen <haibochen@apache.org>
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen <haibochen@apache.org>
Committed: Tue Oct 17 16:04:19 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |  1 +
 .../sls/scheduler/CapacitySchedulerMetrics.java |  5 --
 .../sls/scheduler/FairSchedulerMetrics.java     | 17 +----
 .../sls/scheduler/FifoSchedulerMetrics.java     |  5 +-
 .../sls/scheduler/SLSCapacityScheduler.java     | 19 ------
 .../yarn/sls/scheduler/SLSFairScheduler.java    | 19 ------
 .../yarn/sls/scheduler/SchedulerMetrics.java    | 66 +++++++++-----------
 7 files changed, 36 insertions(+), 96 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
     }
 
     queueAppNumMap.put(queueName, appNum);
+    wrapper.getSchedulerMetrics().trackQueue(queueName);
   }
 
   private void runNewAM(String jobType, String user,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends SchedulerMetrics {
   public CapacitySchedulerMetrics() {
     super();
   }
-
-  @Override
-  public void trackQueue(String queueName) {
-    trackedQueues.add(queueName);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   }
 
   @Override
-  public void trackQueue(String queueName) {
-    trackedQueues.add(queueName);
+  protected void registerQueueMetrics(String queueName) {
+    super.registerQueueMetrics(queueName);
+
     FairScheduler fair = (FairScheduler) scheduler;
     final FSQueue queue = fair.getQueueManager().getQueue(queueName);
     registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
       }
     );
   }
-
-  @Override
-  public void untrackQueue(String queueName) {
-    trackedQueues.remove(queueName);
-
-    for (Metric metric: Metric.values()) {
-      metrics.remove("variable.queue." + queueName + "." +
-          metric.value + ".memory");
-      metrics.remove("variable.queue." + queueName + "." +
-          metric.value + ".vcores");
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
index 6ab2e1d..427b597 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
@@ -35,8 +35,9 @@ public class FifoSchedulerMetrics extends SchedulerMetrics {
   }
 
   @Override
-  public void trackQueue(String queueName) {
-    trackedQueues.add(queueName);
+  protected void registerQueueMetrics(String queueName) {
+    super.registerQueueMetrics(queueName);
+
     FifoScheduler fifo = (FifoScheduler) scheduler;
     // for FifoScheduler, only DEFAULT_QUEUE
     // here the three parameters doesn't affect results

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 108c2bc..6848b22 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -323,25 +323,6 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
         queueName);
   }
 
-  private void initQueueMetrics(CSQueue queue) {
-    if (queue instanceof LeafQueue) {
-      schedulerMetrics.initQueueMetric(queue.getQueueName());
-      return;
-    }
-
-    for (CSQueue child : queue.getChildQueues()) {
-      initQueueMetrics(child);
-    }
-  }
-  @Override
-  public void serviceInit(Configuration configuration) throws Exception {
-    super.serviceInit(configuration);
-
-    if (metricsON) {
-      initQueueMetrics(getRootQueue());
-    }
-  }
-
   @Override
   public void serviceStop() throws Exception {
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 81f6648..8e49c51 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -306,25 +306,6 @@ public class SLSFairScheduler extends FairScheduler
         queueName);
   }
 
-  private void initQueueMetrics(FSQueue queue) {
-    if (queue instanceof FSLeafQueue) {
-      schedulerMetrics.initQueueMetric(queue.getQueueName());
-      return;
-    }
-
-    for (FSQueue child : queue.getChildQueues()) {
-      initQueueMetrics(child);
-    }
-  }
-
-  @Override
-  public void serviceInit(Configuration conf) throws Exception {
-    super.serviceInit(conf);
-    if (metricsON) {
-      initQueueMetrics(getQueueManager().getRootQueue());
-    }
-  }
-
   @Override
   public void serviceStop() throws Exception {
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
index cabc94a..9c4d000 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
@@ -234,14 +234,35 @@ public abstract class SchedulerMetrics {
     }
   }
 
-  public abstract void trackQueue(String queueName);
-  
-  public void untrackQueue(String queueName) {
-    for (String m : queueTrackedMetrics) {
-      metrics.remove("variable.queue." + queueName + "." + m);
+  /**
+   * Track a queue by registering its metrics.
+   *
+   * @param queue queue name
+   */
+  public void trackQueue(String queue) {
+    queueLock.lock();
+    try {
+      if (!isTracked(queue)) {
+        trackedQueues.add(queue);
+        registerQueueMetrics(queue);
+      }
+    } finally {
+      queueLock.unlock();
     }
   }
-  
+
+  protected void registerQueueMetrics(String queueName) {
+    SortedMap<String, Counter> counterMap = metrics.getCounters();
+
+    for (QueueMetric queueMetric : QueueMetric.values()) {
+      String metricName = getQueueMetricName(queueName, queueMetric);
+      if (!counterMap.containsKey(metricName)) {
+        metrics.counter(metricName);
+        queueTrackedMetrics.add(metricName);
+      }
+    }
+  }
+
   public boolean isTracked(String queueName) {
     return trackedQueues.contains(queueName);
   }
@@ -547,40 +568,13 @@ public abstract class SchedulerMetrics {
     return "counter.queue." + queue + "." + metric.value;
   }
 
-  private void traceQueueIfNotTraced(String queue) {
-    queueLock.lock();
-    try {
-      if (!isTracked(queue)) {
-        trackQueue(queue);
-      }
-    } finally {
-      queueLock.unlock();
-    }
-  }
-
-  void initQueueMetric(String queueName){
-    SortedMap<String, Counter> counterMap = metrics.getCounters();
-
-    for (QueueMetric queueMetric : QueueMetric.values()) {
-      String metricName = getQueueMetricName(queueName, queueMetric);
-      if (!counterMap.containsKey(metricName)) {
-        metrics.counter(metricName);
-        counterMap = metrics.getCounters();
-      }
-    }
-
-    traceQueueIfNotTraced(queueName);
-  }
-
   void updateQueueMetrics(Resource pendingResource, Resource allocatedResource,
       String queueName) {
+    trackQueue(queueName);
+
     SortedMap<String, Counter> counterMap = metrics.getCounters();
     for(QueueMetric metric : QueueMetric.values()) {
       String metricName = getQueueMetricName(queueName, metric);
-      if (!counterMap.containsKey(metricName)) {
-        metrics.counter(metricName);
-        counterMap = metrics.getCounters();
-      }
 
       if (metric == QueueMetric.PENDING_MEMORY) {
         counterMap.get(metricName).inc(pendingResource.getMemorySize());
@@ -592,8 +586,6 @@ public abstract class SchedulerMetrics {
         counterMap.get(metricName).inc(allocatedResource.getVirtualCores());
       }
     }
-
-    traceQueueIfNotTraced(queueName);
   }
 
   void updateQueueMetricsByRelease(Resource releaseResource, String queue) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message