accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject accumulo git commit: ACCUMULO-3629 Use ClientConf from job, not default.
Date Fri, 27 Feb 2015 00:02:07 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 4b68322ff -> bac5bc5b2


ACCUMULO-3629 Use ClientConf from job, not default.

Noticed as a part of ACCUMULO-3599. AccumuloInputFormatIT was
failing when run with kerberos on. It hung indefinitely because
it constructed a default client conf which did not have
instance.rpc.sasl.enabled=true.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bac5bc5b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bac5bc5b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bac5bc5b

Branch: refs/heads/master
Commit: bac5bc5b29ee5412d687eaa18899de38c489b310
Parents: 4b68322
Author: Josh Elser <elserj@apache.org>
Authored: Thu Feb 26 18:39:44 2015 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Thu Feb 26 19:01:56 2015 -0500

----------------------------------------------------------------------
 .../core/client/mapred/AbstractInputFormat.java | 14 +++++++-
 .../client/mapreduce/AbstractInputFormat.java   | 14 +++++++-
 .../mapreduce/lib/impl/ConfiguratorBase.java    | 38 +++++++++++++++-----
 3 files changed, 56 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
index 0ce05d7..ebfbe1b 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
@@ -305,6 +305,18 @@ public abstract class AbstractInputFormat<K,V> implements InputFormat<K,V>
{
     return InputConfigurator.getTabletLocator(CLASS, job, tableId);
   }
 
+  /**
+   * Fetch the client configuration from the job.
+   *
+   * @param job
+   *          The job
+   * @return The client configuration for the job
+   * @since 1.7.0
+   */
+  protected static ClientConfiguration getClientConfiguration(JobConf job) {
+    return InputConfigurator.getClientConfiguration(CLASS, job);
+  }
+
   // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext
job)
   /**
    * Check whether a configuration is fully configured to be used with an Accumulo {@link
InputFormat}.
@@ -578,7 +590,7 @@ public abstract class AbstractInputFormat<K,V> implements InputFormat<K,V>
{
           tl.invalidateCache();
 
           ClientContext context = new ClientContext(getInstance(job), new Credentials(getPrincipal(job),
getAuthenticationToken(job)),
-              ClientConfiguration.loadDefault());
+              getClientConfiguration(job));
           while (!tl.binRanges(context, ranges, binnedRanges).isEmpty()) {
             if (!(instance instanceof MockInstance)) {
               if (!Tables.exists(instance, tableId))

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
index e1b35b2..255e555 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
@@ -388,6 +388,18 @@ public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V>
{
   }
 
   /**
+   * Construct the {@link ClientConfiguration} given the provided context.
+   *
+   * @param context
+   *          The Job
+   * @return The ClientConfiguration
+   * @since 1.7.0
+   */
+  protected static ClientConfiguration getClientConfiguration(JobContext context) {
+    return InputConfigurator.getClientConfiguration(CLASS, context.getConfiguration());
+  }
+
+  /**
    * An abstract base class to be used to create {@link org.apache.hadoop.mapreduce.RecordReader}
instances that convert from Accumulo
    * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value}
pairs to the user's K/V types.
    *
@@ -634,7 +646,7 @@ public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V>
{
           tl.invalidateCache();
 
           ClientContext clientContext = new ClientContext(getInstance(context), new Credentials(getPrincipal(context),
getAuthenticationToken(context)),
-              ClientConfiguration.loadDefault());
+              getClientConfiguration(context));
           while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) {
             if (!(instance instanceof MockInstance)) {
               if (!Tables.exists(instance, tableId))

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index 3b5fa3a..b32d1b3 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@ -358,14 +358,7 @@ public class ConfiguratorBase {
     if ("MockInstance".equals(instanceType))
       return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
     else if ("ZooKeeperInstance".equals(instanceType)) {
-      String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
-      if (clientConfigString == null) {
-        String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
-        String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
-        return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
-      } else {
-        return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
-      }
+      return new ZooKeeperInstance(getClientConfiguration(implementingClass, conf));
     } else if (instanceType.isEmpty())
       throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
     else
@@ -373,6 +366,35 @@ public class ConfiguratorBase {
   }
 
   /**
+   * Obtain a {@link ClientConfiguration} based on the configuration.
+   *
+   * @param implementingClass
+   *          the class whose name will be used as a prefix for the property configuration
key
+   * @param conf
+   *          the Hadoop configuration object to configure
+   *
+   * @return A {@link ClientConfiguration}
+   * @since 1.7.0
+   */
+  public static ClientConfiguration getClientConfiguration(Class<?> implementingClass,
Configuration conf) {
+    String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
+    if (null != clientConfigString) {
+      return ClientConfiguration.deserialize(clientConfigString);
+    }
+
+    String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
+    String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
+    ClientConfiguration clientConf = ClientConfiguration.loadDefault();
+    if (null != instanceName) {
+      clientConf.withInstance(instanceName);
+    }
+    if (null != zookeepers) {
+      clientConf.withZkHosts(zookeepers);
+    }
+    return clientConf;
+  }
+
+  /**
    * Sets the log level for this job.
    *
    * @param implementingClass


Mime
View raw message