accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] 02/02: ACCUMULO-4779 made getting vfs config more efficient
Date Fri, 26 Jan 2018 01:00:29 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 1.7
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 7bc630cdce4a0aa02be4873e2873ff1036a94e14
Author: Keith Turner <kturner@apache.org>
AuthorDate: Thu Jan 25 19:44:05 2018 -0500

    ACCUMULO-4779 made getting vfs config more efficient
---
 .../accumulo/core/client/impl/ClientContext.java   |  6 +++++
 .../core/client/mock/MockConfiguration.java        |  5 ++++
 .../accumulo/core/conf/AccumuloConfiguration.java  | 27 ++++++++++++++++++++++
 .../accumulo/core/conf/ConfigurationCopy.java      |  5 ++++
 .../accumulo/core/conf/DefaultConfiguration.java   |  5 ++++
 .../accumulo/core/conf/SiteConfiguration.java      |  9 ++++++++
 .../accumulo/server/conf/ZooConfiguration.java     | 15 ++++++++++--
 .../java/org/apache/accumulo/master/Master.java    |  4 ++--
 .../org/apache/accumulo/tserver/TabletServer.java  |  4 ++--
 .../main/java/org/apache/accumulo/shell/Shell.java |  4 ++--
 .../start/classloader/vfs/ContextManager.java      | 11 +++++----
 11 files changed, 82 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ClientContext.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ClientContext.java
index 6828174..cb1ccb9 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ClientContext.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ClientContext.java
@@ -176,6 +176,12 @@ public class ClientContext {
     final AccumuloConfiguration defaults = DefaultConfiguration.getInstance();
 
     return new AccumuloConfiguration() {
+
+      @Override
+      protected String getArbitrarySystemPropertyImpl(String property) {
+        return config.getString(property, null);
+      }
+
       @Override
       public String get(Property property) {
         final String key = property.getKey();
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/client/mock/MockConfiguration.java
index 410105b..f642770 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockConfiguration.java
@@ -36,6 +36,11 @@ class MockConfiguration extends AccumuloConfiguration {
   }
 
   @Override
+  protected String getArbitrarySystemPropertyImpl(String property) {
+    return map.get(property);
+  }
+
+  @Override
   public String get(Property property) {
     return map.get(property.getKey());
   }
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
index 25d4c60..008418e 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
@@ -33,6 +33,7 @@ import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 
@@ -106,6 +107,32 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str
 
   private static final Logger log = LoggerFactory.getLogger(AccumuloConfiguration.class);
 
+  protected String getArbitrarySystemPropertyImpl(AccumuloConfiguration parent, String property)
{
+    return parent.getArbitrarySystemPropertyImpl(property);
+  }
+
+  /**
+   * This method is not called with sensitive or per table properties.
+   */
+  protected String getArbitrarySystemPropertyImpl(String property) {
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * This method was created because {@link #get(String)} is very slow. However this method
does not properly handle everything that {@link #get(String)} does.
+   * For example it does not properly handle table or sensitive properties.
+   *
+   * <p>
+   * This method has a whitelist of prefixes it handles. To see the whitelist, check the
implementation. When adding to the whitelist, ensure that all
+   * configurations can properly handle.
+   */
+  public String getArbitrarySystemProperty(Property prefix, String property) {
+    Preconditions.checkArgument(prefix == Property.VFS_CONTEXT_CLASSPATH_PROPERTY);
+
+    String key = prefix.getKey() + property;
+    return getArbitrarySystemPropertyImpl(key);
+  }
+
   /**
    * Gets a property value from this configuration.
    *
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
index 28b188f..df43557 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
@@ -64,6 +64,11 @@ public class ConfigurationCopy extends AccumuloConfiguration {
   }
 
   @Override
+  protected String getArbitrarySystemPropertyImpl(String property) {
+    return copy.get(property);
+  }
+
+  @Override
   public void getProperties(Map<String,String> props, Predicate<String> filter)
{
     for (Entry<String,String> entry : copy.entrySet()) {
       if (filter.apply(entry.getKey())) {
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
index e1ff7e1..994d960 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
@@ -58,4 +58,9 @@ public class DefaultConfiguration extends AccumuloConfiguration {
       if (filter.apply(entry.getKey()))
         props.put(entry.getKey(), entry.getValue());
   }
+
+  @Override
+  protected String getArbitrarySystemPropertyImpl(String property) {
+    return null;
+  }
 }
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
index 9f047e2..32b2564 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
@@ -129,6 +129,15 @@ public class SiteConfiguration extends AccumuloConfiguration {
   }
 
   @Override
+  protected String getArbitrarySystemPropertyImpl(String property) {
+    String val = staticConfigs.get(property);
+    if (val == null) {
+      val = parent.getArbitrarySystemPropertyImpl(property);
+    }
+    return val;
+  }
+
+  @Override
   public void getProperties(Map<String,String> props, Predicate<String> filter)
{
     parent.getProperties(props, filter);
 
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index f178ff1..2817ba0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -40,11 +40,13 @@ public class ZooConfiguration extends AccumuloConfiguration {
   private final ZooCache propCache;
   private final AccumuloConfiguration parent;
   private final Map<String,String> fixedProps = Collections.synchronizedMap(new HashMap<String,String>());
+  private final String propPathPrefix;
 
   protected ZooConfiguration(String instanceId, ZooCache propCache, AccumuloConfiguration
parent) {
     this.instanceId = instanceId;
     this.propCache = propCache;
     this.parent = parent;
+    this.propPathPrefix = ZooUtil.getRoot(instanceId) + Constants.ZCONFIG;
   }
 
   @Override
@@ -79,6 +81,15 @@ public class ZooConfiguration extends AccumuloConfiguration {
   }
 
   @Override
+  protected String getArbitrarySystemPropertyImpl(String property) {
+    String val = getRaw(property);
+    if (val == null) {
+      val = getArbitrarySystemPropertyImpl(parent, property);
+    }
+    return val;
+  }
+
+  @Override
   public String get(Property property) {
     if (Property.isFixedZooPropertyKey(property)) {
       if (fixedProps.containsKey(property.getKey())) {
@@ -97,7 +108,7 @@ public class ZooConfiguration extends AccumuloConfiguration {
   }
 
   private String getRaw(String key) {
-    String zPath = ZooUtil.getRoot(instanceId) + Constants.ZCONFIG + "/" + key;
+    String zPath = propPathPrefix + "/" + key;
     byte[] v = propCache.get(zPath);
     String value = null;
     if (v != null)
@@ -109,7 +120,7 @@ public class ZooConfiguration extends AccumuloConfiguration {
   public void getProperties(Map<String,String> props, Predicate<String> filter)
{
     parent.getProperties(props, filter);
 
-    List<String> children = propCache.getChildren(ZooUtil.getRoot(instanceId) + Constants.ZCONFIG);
+    List<String> children = propCache.getChildren(propPathPrefix);
     if (children != null) {
       for (String child : children) {
         if (child != null && filter.apply(child)) {
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 984fb60..740e05f 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -591,8 +591,8 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
     try {
       AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig()
{
         @Override
-        public String getProperty(String key) {
-          return getConfiguration().get(key);
+        public String getVfsContextClasspathProperty(String key) {
+          return getConfiguration().getArbitrarySystemProperty(Property.VFS_CONTEXT_CLASSPATH_PROPERTY,
key);
         }
       });
     } catch (IOException e) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index a277714..f20e145 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -2815,8 +2815,8 @@ public class TabletServer extends AccumuloServerContext implements Runnable
{
     try {
       AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig()
{
         @Override
-        public String getProperty(String key) {
-          return getConfiguration().get(key);
+        public String getVfsContextClasspathProperty(String key) {
+          return getConfiguration().getArbitrarySystemProperty(Property.VFS_CONTEXT_CLASSPATH_PROPERTY,
key);
         }
       });
     } catch (IOException e) {
diff --git a/shell/src/main/java/org/apache/accumulo/shell/Shell.java b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
index b5e5afb..b1e473d 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
@@ -550,8 +550,8 @@ public class Shell extends ShellOptions implements KeywordExecutable {
 
         AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig()
{
           @Override
-          public String getProperty(String key) {
-            return systemConfig.get(key);
+          public String getVfsContextClasspathProperty(String key) {
+            return systemConfig.get(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + key);
           }
 
         });
diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
index a3d51fc..e155b3d 100644
--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
@@ -99,20 +99,21 @@ public class ContextManager {
 
   public static abstract class DefaultContextsConfig implements ContextsConfig {
 
-    public abstract String getProperty(String key);
+    /**
+     * Implementations should prepend {@link AccumuloVFSClassLoader#VFS_CONTEXT_CLASSPATH_PROPERTY}
to the given key.
+     */
+    public abstract String getVfsContextClasspathProperty(String key);
 
     @Override
     public ContextConfig getContextConfig(String context) {
 
-      String key = AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY + context;
-
-      String uris = getProperty(key);
+      String uris = getVfsContextClasspathProperty(context);
 
       if (uris == null) {
         return null;
       }
 
-      String delegate = getProperty(key + ".delegation");
+      String delegate = getVfsContextClasspathProperty(context + ".delegation");
 
       boolean preDelegate = true;
 

-- 
To stop receiving notification emails like this one, please contact
kturner@apache.org.

Mime
View raw message