Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6F4C4200C7D for ; Tue, 16 May 2017 18:10:28 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 6DDC2160BC9; Tue, 16 May 2017 16:10:28 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 83659160B9D for ; Tue, 16 May 2017 18:10:27 +0200 (CEST) Received: (qmail 78722 invoked by uid 500); 16 May 2017 16:10:21 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 77537 invoked by uid 99); 16 May 2017 16:10:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 May 2017 16:10:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6F7AAF21A5; Tue, 16 May 2017 16:10:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: curino@apache.org To: common-commits@hadoop.apache.org Date: Tue, 16 May 2017 16:10:51 -0000 Message-Id: <8bf27ff0bdc043e88d2bd0d8fb425006@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [34/50] [abbrv] hadoop git commit: YARN-6247. Share a single instance of SubClusterResolver instead of instantiating one per AM. (Botong Huang via Subru) archived-at: Tue, 16 May 2017 16:10:28 -0000 YARN-6247. Share a single instance of SubClusterResolver instead of instantiating one per AM. (Botong Huang via Subru) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/875e4f2e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/875e4f2e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/875e4f2e Branch: refs/heads/YARN-2915 Commit: 875e4f2ef6d35816fed9a679f03bb37e5c9a241d Parents: 34e4f48 Author: Subru Krishnan Authored: Thu Mar 2 18:54:53 2017 -0800 Committer: Carlo Curino Committed: Tue May 16 08:52:38 2017 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 6 +++ .../src/main/resources/yarn-default.xml | 7 +++ .../resolver/AbstractSubClusterResolver.java | 6 +-- .../federation/resolver/SubClusterResolver.java | 4 +- .../utils/FederationStateStoreFacade.java | 48 +++++++++++++++++--- 5 files changed, 59 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/875e4f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index dd05411..4ba8c85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2572,6 +2572,12 @@ public class YarnConfiguration extends Configuration { public static final String FEDERATION_MACHINE_LIST = FEDERATION_PREFIX + "machine-list"; + public static final String FEDERATION_CLUSTER_RESOLVER_CLASS = + FEDERATION_PREFIX + "subcluster-resolver.class"; + public static final String DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS = + "org.apache.hadoop.yarn.server.federation.resolver." + + "DefaultSubClusterResolverImpl"; + public static final String DEFAULT_FEDERATION_POLICY_KEY = "*"; public static final String FEDERATION_POLICY_MANAGER = FEDERATION_PREFIX http://git-wip-us.apache.org/repos/asf/hadoop/blob/875e4f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0d40482..75b32ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2683,6 +2683,13 @@ yarn.federation.machine-list + + + Class name for SubClusterResolver + + yarn.federation.subcluster-resolver.class + org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl + http://git-wip-us.apache.org/repos/asf/hadoop/blob/875e4f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java index 6b4f60c..bccff2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java @@ -21,8 +21,8 @@ package org.apache.hadoop.yarn.server.federation.resolver; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import java.util.HashMap; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.Map; /** @@ -31,9 +31,9 @@ import java.util.Map; */ public abstract class AbstractSubClusterResolver implements SubClusterResolver { private Map nodeToSubCluster = - new HashMap(); + new ConcurrentHashMap(); private Map> rackToSubClusters = - new HashMap>(); + new ConcurrentHashMap>(); @Override public SubClusterId getSubClusterForNode(String nodename) http://git-wip-us.apache.org/repos/asf/hadoop/blob/875e4f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java index c6adfa6..612d396 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java @@ -25,8 +25,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; /** - * An utility that helps to determine the sub-cluster that a specified node - * belongs to. + * An utility that helps to determine the sub-cluster that a specified node or + * rack belongs to. All implementing classes should be thread-safe. */ public interface SubClusterResolver extends Configurable { http://git-wip-us.apache.org/repos/asf/hadoop/blob/875e4f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 9b794de..e8f245e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; @@ -90,6 +91,7 @@ public final class FederationStateStoreFacade { private int cacheTimeToLive; private Configuration conf; private Cache cache; + private SubClusterResolver subclusterResolver; private FederationStateStoreFacade() { initializeFacadeInternal(new Configuration()); @@ -104,6 +106,12 @@ public final class FederationStateStoreFacade { FederationStateStore.class, createRetryPolicy(conf)); this.stateStore.init(conf); + this.subclusterResolver = createInstance(conf, + YarnConfiguration.FEDERATION_CLUSTER_RESOLVER_CLASS, + YarnConfiguration.DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS, + SubClusterResolver.class); + this.subclusterResolver.load(); + initCache(); } catch (YarnException ex) { @@ -348,6 +356,15 @@ public final class FederationStateStoreFacade { } /** + * Get the singleton instance of SubClusterResolver. + * + * @return SubClusterResolver instance + */ + public SubClusterResolver getSubClusterResolver() { + return this.subclusterResolver; + } + + /** * Helper method to create instances of Object using the class name defined in * the configuration object. The instances creates {@link RetryProxy} using * the specific {@link RetryPolicy}. @@ -359,23 +376,40 @@ public final class FederationStateStoreFacade { * @param retryPolicy the policy for retrying method call failures * @return a retry proxy for the specified interface */ - @SuppressWarnings("unchecked") public static Object createRetryInstance(Configuration conf, String configuredClassName, String defaultValue, Class type, RetryPolicy retryPolicy) { + return RetryProxy.create(type, + createInstance(conf, configuredClassName, defaultValue, type), + retryPolicy); + } + + /** + * Helper method to create instances of Object using the class name specified + * in the configuration object. + * + * @param conf the yarn configuration + * @param configuredClassName the configuration provider key + * @param defaultValue the default implementation class + * @param type the required interface/base class + * @param The type of the instance to create + * @return the instances created + */ + @SuppressWarnings("unchecked") + public static T createInstance(Configuration conf, + String configuredClassName, String defaultValue, Class type) { + String className = conf.get(configuredClassName, defaultValue); try { Class clusterResolverClass = conf.getClassByName(className); if (type.isAssignableFrom(clusterResolverClass)) { - return RetryProxy.create(type, - (T) ReflectionUtils.newInstance(clusterResolverClass, conf), - retryPolicy); + return (T) ReflectionUtils.newInstance(clusterResolverClass, conf); } else { - throw new YarnRuntimeException( - "Class: " + className + " not instance of " + type.getSimpleName()); + throw new YarnRuntimeException("Class: " + className + + " not instance of " + type.getCanonicalName()); } - } catch (Exception e) { + } catch (ClassNotFoundException e) { throw new YarnRuntimeException("Could not instantiate : " + className, e); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org