Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3A240174B2 for ; Fri, 27 Feb 2015 00:02:08 +0000 (UTC) Received: (qmail 44901 invoked by uid 500); 27 Feb 2015 00:02:08 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 44866 invoked by uid 500); 27 Feb 2015 00:02:08 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 44836 invoked by uid 99); 27 Feb 2015 00:02:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Feb 2015 00:02:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D7B70E07EB; Fri, 27 Feb 2015 00:02:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: accumulo git commit: ACCUMULO-3629 Use ClientConf from job, not default. Date: Fri, 27 Feb 2015 00:02:07 +0000 (UTC) Repository: accumulo Updated Branches: refs/heads/master 4b68322ff -> bac5bc5b2 ACCUMULO-3629 Use ClientConf from job, not default. Noticed as a part of ACCUMULO-3599. AccumuloInputFormatIT was failing when run with kerberos on. It hung indefinitely because it constructed a default client conf which did not have instance.rpc.sasl.enabled=true. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bac5bc5b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bac5bc5b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bac5bc5b Branch: refs/heads/master Commit: bac5bc5b29ee5412d687eaa18899de38c489b310 Parents: 4b68322 Author: Josh Elser Authored: Thu Feb 26 18:39:44 2015 -0500 Committer: Josh Elser Committed: Thu Feb 26 19:01:56 2015 -0500 ---------------------------------------------------------------------- .../core/client/mapred/AbstractInputFormat.java | 14 +++++++- .../client/mapreduce/AbstractInputFormat.java | 14 +++++++- .../mapreduce/lib/impl/ConfiguratorBase.java | 38 +++++++++++++++----- 3 files changed, 56 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java index 0ce05d7..ebfbe1b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java @@ -305,6 +305,18 @@ public abstract class AbstractInputFormat implements InputFormat { return InputConfigurator.getTabletLocator(CLASS, job, tableId); } + /** + * Fetch the client configuration from the job. + * + * @param job + * The job + * @return The client configuration for the job + * @since 1.7.0 + */ + protected static ClientConfiguration getClientConfiguration(JobConf job) { + return InputConfigurator.getClientConfiguration(CLASS, job); + } + // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job) /** * Check whether a configuration is fully configured to be used with an Accumulo {@link InputFormat}. @@ -578,7 +590,7 @@ public abstract class AbstractInputFormat implements InputFormat { tl.invalidateCache(); ClientContext context = new ClientContext(getInstance(job), new Credentials(getPrincipal(job), getAuthenticationToken(job)), - ClientConfiguration.loadDefault()); + getClientConfiguration(job)); while (!tl.binRanges(context, ranges, binnedRanges).isEmpty()) { if (!(instance instanceof MockInstance)) { if (!Tables.exists(instance, tableId)) http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java index e1b35b2..255e555 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java @@ -388,6 +388,18 @@ public abstract class AbstractInputFormat extends InputFormat { } /** + * Construct the {@link ClientConfiguration} given the provided context. + * + * @param context + * The Job + * @return The ClientConfiguration + * @since 1.7.0 + */ + protected static ClientConfiguration getClientConfiguration(JobContext context) { + return InputConfigurator.getClientConfiguration(CLASS, context.getConfiguration()); + } + + /** * An abstract base class to be used to create {@link org.apache.hadoop.mapreduce.RecordReader} instances that convert from Accumulo * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to the user's K/V types. * @@ -634,7 +646,7 @@ public abstract class AbstractInputFormat extends InputFormat { tl.invalidateCache(); ClientContext clientContext = new ClientContext(getInstance(context), new Credentials(getPrincipal(context), getAuthenticationToken(context)), - ClientConfiguration.loadDefault()); + getClientConfiguration(context)); while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) { if (!(instance instanceof MockInstance)) { if (!Tables.exists(instance, tableId)) http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java index 3b5fa3a..b32d1b3 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java @@ -358,14 +358,7 @@ public class ConfiguratorBase { if ("MockInstance".equals(instanceType)) return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME))); else if ("ZooKeeperInstance".equals(instanceType)) { - String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG)); - if (clientConfigString == null) { - String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)); - String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS)); - return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers)); - } else { - return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString)); - } + return new ZooKeeperInstance(getClientConfiguration(implementingClass, conf)); } else if (instanceType.isEmpty()) throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName()); else @@ -373,6 +366,35 @@ public class ConfiguratorBase { } /** + * Obtain a {@link ClientConfiguration} based on the configuration. + * + * @param implementingClass + * the class whose name will be used as a prefix for the property configuration key + * @param conf + * the Hadoop configuration object to configure + * + * @return A {@link ClientConfiguration} + * @since 1.7.0 + */ + public static ClientConfiguration getClientConfiguration(Class implementingClass, Configuration conf) { + String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG)); + if (null != clientConfigString) { + return ClientConfiguration.deserialize(clientConfigString); + } + + String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)); + String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS)); + ClientConfiguration clientConf = ClientConfiguration.loadDefault(); + if (null != instanceName) { + clientConf.withInstance(instanceName); + } + if (null != zookeepers) { + clientConf.withZkHosts(zookeepers); + } + return clientConf; + } + + /** * Sets the log level for this job. * * @param implementingClass