Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E0B48200BFB for ; Wed, 11 Jan 2017 19:19:22 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id DF779160B53; Wed, 11 Jan 2017 18:19:22 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3B87B160B3B for ; Wed, 11 Jan 2017 19:19:22 +0100 (CET) Received: (qmail 22284 invoked by uid 500); 11 Jan 2017 18:19:18 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 21058 invoked by uid 99); 11 Jan 2017 18:19:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Jan 2017 18:19:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3013ADFDF0; Wed, 11 Jan 2017 18:19:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: stevel@apache.org To: common-commits@hadoop.apache.org Date: Wed, 11 Jan 2017 18:19:22 -0000 Message-Id: <2a413cee4ccd4311872fd383bfc2fa47@git.apache.org> In-Reply-To: <81e84c752ba84f7988aad2243d491ab6@git.apache.org> References: <81e84c752ba84f7988aad2243d491ab6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [06/50] [abbrv] hadoop git commit: YARN-4899. Queue metrics of SLS capacity scheduler only activated after app submit to the queue. (Jonathan Hung via wangda) archived-at: Wed, 11 Jan 2017 18:19:23 -0000 YARN-4899. Queue metrics of SLS capacity scheduler only activated after app submit to the queue. (Jonathan Hung via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ed63e3e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ed63e3e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ed63e3e Branch: refs/heads/HADOOP-13345 Commit: 5ed63e3e9d9937cf7441b7ceb5feafbf486f3387 Parents: a0a2761 Author: Wangda Tan Authored: Wed Jan 4 19:17:11 2017 -0800 Committer: Wangda Tan Committed: Wed Jan 4 19:17:11 2017 -0800 ---------------------------------------------------------------------- .../sls/scheduler/SLSCapacityScheduler.java | 44 ++++++++++++++++++++ 1 file changed, 44 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ed63e3e/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index cf08309..89f9ad3 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -57,7 +57,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -84,6 +86,7 @@ import com.codahale.metrics.Timer; public class SLSCapacityScheduler extends CapacityScheduler implements SchedulerWrapper,Configurable { private static final String EOL = System.getProperty("line.separator"); + private static final String QUEUE_COUNTER_PREFIX = "counter.queue."; private static final int SAMPLING_SIZE = 60; private ScheduledExecutorService pool; // counters for scheduler allocate/handle operations @@ -747,6 +750,47 @@ public class SLSCapacityScheduler extends CapacityScheduler implements } } + private void initQueueMetrics(CSQueue queue) { + if (queue instanceof LeafQueue) { + SortedMap counterMap = metrics.getCounters(); + String queueName = queue.getQueueName(); + String[] names = new String[]{ + QUEUE_COUNTER_PREFIX + queueName + ".pending.memory", + QUEUE_COUNTER_PREFIX + queueName + ".pending.cores", + QUEUE_COUNTER_PREFIX + queueName + ".allocated.memory", + QUEUE_COUNTER_PREFIX + queueName + ".allocated.cores" }; + + for (int i = names.length - 1; i >= 0; i--) { + if (!counterMap.containsKey(names[i])) { + metrics.counter(names[i]); + counterMap = metrics.getCounters(); + } + } + + queueLock.lock(); + try { + if (!schedulerMetrics.isTracked(queueName)) { + schedulerMetrics.trackQueue(queueName); + } + } finally { + queueLock.unlock(); + } + + return; + } + + for (CSQueue child : queue.getChildQueues()) { + initQueueMetrics(child); + } + } + + @Override + public void serviceInit(Configuration configuration) throws Exception { + super.serviceInit(configuration); + + initQueueMetrics(getRootQueue()); + } + public void setQueueSet(Set queues) { this.queueSet = queues; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org