accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [4/6] git commit: ACCUMULO-1919 Size of visibility cache in AccumuloFileOutputFormat should be configurable.
Date Mon, 09 Jun 2014 14:42:54 GMT
ACCUMULO-1919 Size of visibility cache in AccumuloFileOutputFormat should be configurable.

Signed-off-by: Josh Elser <elserj@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ae1682d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ae1682d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ae1682d4

Branch: refs/heads/master
Commit: ae1682d4ae7a3031598748f676c90829d0b2a978
Parents: 73e0f9c
Author: dallaybatta <vickykak@gmail.com>
Authored: Mon Jun 9 11:29:54 2014 +0530
Committer: Josh Elser <elserj@apache.org>
Committed: Mon Jun 9 10:27:51 2014 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/core/Constants.java     |  2 ++
 .../client/mapred/AccumuloFileOutputFormat.java |  3 +-
 .../mapreduce/lib/impl/ConfiguratorBase.java    | 37 +++++++++++++++++++-
 .../mapred/AccumuloFileOutputFormatTest.java    |  3 ++
 .../lib/impl/ConfiguratorBaseTest.java          |  8 +++++
 5 files changed, 51 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java
index e0e88eb..c9a51a9 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -115,4 +115,6 @@ public class Constants {
   public static final String[] PATH_PROPERTY_ENV_VARS = new String[] {"ACCUMULO_HOME", "ACCUMULO_CONF_DIR"};
 
   public static final String HDFS_TABLES_DIR = "/tables";
+
+  public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000;
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
index 8a1d6df..cfaaa58 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.mapred;
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
 import org.apache.accumulo.core.client.mapreduce.lib.impl.FileOutputConfigurator;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
@@ -145,7 +146,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value>
{
     final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
     final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." +
extension);
     
-    final LRUMap validVisibilities = new LRUMap(1000);
+    final LRUMap validVisibilities = new LRUMap(ConfiguratorBase.getVisibilityCacheSize(conf));
     
     return new RecordWriter<Key,Value>() {
       FileSKVWriter out = null;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index e87d43b..cf1f0ea 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@ -82,7 +82,8 @@ public class ConfiguratorBase {
    * @since 1.6.0
    */
   public static enum GeneralOpts {
-    LOG_LEVEL
+    LOG_LEVEL,
+    VISIBILITY_CACHE_SIZE
   }
 
   /**
@@ -100,6 +101,17 @@ public class ConfiguratorBase {
   }
 
   /**
+  * Provides a configuration key for a given feature enum.
+  * 
+  * @param e
+  *          the enum used to provide the unique part of the configuration key
+  * @return the configuration key
+  */
+  protected static String enumToConfKey(Enum<?> e) {
+	  return  e.getDeclaringClass().getSimpleName() + "." +  StringUtils.camelize(e.name().toLowerCase());
+  }
+
+  /**
    * Sets the connector information needed to communicate with Accumulo in this job.
    * 
    * <p>
@@ -369,4 +381,27 @@ public class ConfiguratorBase {
     return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL),
Level.INFO.toInt()));
   }
 
+  /**
+   * Sets the valid visibility count for this job.
+   * 
+   * @param conf
+   *          the Hadoop configuration object to configure
+   * @param visibilityCacheSize
+   *          the LRU cache size
+   */
+  public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize)
{
+    conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+  }
+
+  /**
+   * Gets the valid visibility count for this job.
+   * 
+   * @param conf
+   *          the Hadoop configuration object to configure
+   * @return the valid visibility count
+   */
+  public static int getVisibilityCacheSize(Configuration conf) {
+    return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
index aad544b..3a4d641 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
@@ -28,6 +28,7 @@ import java.io.IOException;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
 import org.apache.accumulo.core.client.mock.MockInstance;
 import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -53,6 +54,7 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 public class AccumuloFileOutputFormatTest {
+  private static final int JOB_VISIBILITY_CACHE_SIZE = 3000;
   private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName();
   private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
   private static final String BAD_TABLE = PREFIX + "_mapred_bad_table";
@@ -145,6 +147,7 @@ public class AccumuloFileOutputFormatTest {
 
       JobConf job = new JobConf(getConf());
       job.setJarByClass(this.getClass());
+      ConfiguratorBase.setVisibilityCacheSize(job, JOB_VISIBILITY_CACHE_SIZE);
 
       job.setInputFormat(AccumuloInputFormat.class);
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
index d5ebb22..7c1f98b 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ClientConfiguration;
 import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
@@ -126,4 +127,11 @@ public class ConfiguratorBaseTest {
     assertEquals(Level.FATAL, ConfiguratorBase.getLogLevel(this.getClass(), conf));
   }
 
+  @Test
+  public void testSetVisibiltyCacheSize(){
+	 Configuration conf = new Configuration();
+	 assertEquals(Constants.DEFAULT_VISIBILITY_CACHE_SIZE,ConfiguratorBase.getVisibilityCacheSize(conf));
+	 ConfiguratorBase.setVisibilityCacheSize(conf, 2000);
+	 assertEquals(2000,ConfiguratorBase.getVisibilityCacheSize(conf));
+  }
 }


Mime
View raw message