accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject accumulo-testing git commit: ACCUMULO-4579 Fixed hadoop config bug in accumulo-testing
Date Wed, 01 Feb 2017 21:12:21 GMT
Repository: accumulo-testing
Updated Branches:
  refs/heads/master bef8ed208 -> 28ca75a36


ACCUMULO-4579 Fixed hadoop config bug in accumulo-testing

* Bugs were causing tests to not work on cluster
* TestEnv now returns Hadoop configuration that is loaded with configuration
  retrieved from accumulo-testing.properties
* Fixed bug where Twill was being improperly configured to look in wrong
  location for shaded jar when running test application in YARN.
* Updated accumulo-testing.properites by adding accumulo to several property names


Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/28ca75a3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/28ca75a3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/28ca75a3

Branch: refs/heads/master
Commit: 28ca75a368168a035f1936fcca8d3a4a520e6eb3
Parents: bef8ed2
Author: Mike Walch <mwalch@apache.org>
Authored: Mon Jan 30 15:03:21 2017 -0500
Committer: Mike Walch <mwalch@apache.org>
Committed: Wed Feb 1 16:07:35 2017 -0500

----------------------------------------------------------------------
 conf/accumulo-testing.properties.example        | 28 ++++++++------
 .../apache/accumulo/testing/core/TestEnv.java   | 39 +++++++++++++-------
 .../apache/accumulo/testing/core/TestProps.java | 25 ++++++++-----
 .../core/continuous/ContinuousBatchWalker.java  |  2 +-
 .../testing/yarn/YarnAccumuloTestRunner.java    | 19 ++++++----
 5 files changed, 68 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/conf/accumulo-testing.properties.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example
index 1d43e12..ce868be 100644
--- a/conf/accumulo-testing.properties.example
+++ b/conf/accumulo-testing.properties.example
@@ -23,24 +23,28 @@ test.common.accumulo.instance=instance
 test.common.accumulo.username=root
 # Accumulo password
 test.common.accumulo.password=secret
+# Max memory (in bytes) each batch writer will use to buffer writes
+test.common.accumulo.bw.max.memory.bytes=100000000
+# Max latency (in milliseconds) that each batch writer will buffer data
+test.common.accumulo.bw.max.latency.ms=600000
+# Number of write thread for each batch writer
+test.common.accumulo.bw.num.threads=4
+# Number of threads used by batch scanner
+test.common.accumulo.bs.num.threads=8
+# Number of key/value entries to pull during scan
+test.common.accumulo.scanner.batch.size=1000
 # Accumulo keytab
 #test.common.accumulo.keytab=
-# Zookeeper connection string
-test.common.zookeepers=localhost:2181
+# HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
+test.common.hdfs.root=hdfs://localhost:10000
+# YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in
Hadoop's yarn-site.xml
+test.common.yarn.resource.manager=localhost
 # Memory (in MB) given to each container (if running in YARN)
 test.common.yarn.container.memory.mb=1024
 # Number of cores given to each container (if running in YARN)
 test.common.yarn.container.cores=1
-# Max memory (in bytes) each batch writer will use to buffer writes
-test.common.bw.max.memory.bytes=100000000
-# Max latency (in milliseconds) that each batch writer will buffer data
-test.common.bw.max.latency.ms=600000
-# Number of write thread for each batch writer
-test.common.bw.num.threads=4
-# Number of threads used by batch scanner
-test.common.bs.num.threads=8
-# Number of key/value entries to pull during scan
-test.common.scanner.batch.size=1000
+# Zookeeper connection string
+test.common.zookeepers=localhost:2181
 
 ###################################
 # Continuous ingest test properties

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
index cd1f1dc..48266f5 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java
@@ -19,7 +19,7 @@ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 import org.apache.accumulo.core.client.security.tokens.KerberosToken;
 import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.tools.CLI;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.UserGroupInformation;
 
 public class TestEnv {
@@ -27,6 +27,7 @@ public class TestEnv {
   protected final Properties p;
   private Instance instance = null;
   private Connector connector = null;
+  private Configuration hadoopConfig = null;
 
   /**
    * Creates new test environment using provided properties
@@ -96,15 +97,17 @@ public class TestEnv {
   }
 
   public Configuration getHadoopConfiguration() {
-    Configuration config = new Configuration();
-    config.set("mapreduce.framework.name", "yarn");
-    // Setting below are required due to bundled jar breaking default
-    // config.
-    // See
-    // http://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file
-    config.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
-    config.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
-    return config;
+    if (hadoopConfig == null) {
+      hadoopConfig = new Configuration();
+      hadoopConfig.set("fs.defaultFS", getHdfsRoot());
+      // Below is required due to bundled jar breaking default config.
+      // See http://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file
+      hadoopConfig.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
+      hadoopConfig.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
+      hadoopConfig.set("mapreduce.framework.name", "yarn");
+      hadoopConfig.set("yarn.resourcemanager.hostname", getYarnResourceManager());
+    }
+    return hadoopConfig;
   }
 
   /**
@@ -135,6 +138,14 @@ public class TestEnv {
     return p.getProperty(TestProps.ACCUMULO_INSTANCE);
   }
 
+  public String getHdfsRoot() {
+    return p.getProperty(TestProps.HDFS_ROOT);
+  }
+
+  public String getYarnResourceManager() {
+    return p.getProperty(TestProps.YARN_RESOURCE_MANAGER);
+  }
+
   public String getZookeepers() {
     return p.getProperty(TestProps.ZOOKEEPERS);
   }
@@ -164,9 +175,9 @@ public class TestEnv {
   }
 
   public BatchWriterConfig getBatchWriterConfig() {
-    int numThreads = Integer.parseInt(p.getProperty(TestProps.BW_NUM_THREADS));
-    long maxLatency = Long.parseLong(p.getProperty(TestProps.BW_MAX_LATENCY_MS));
-    long maxMemory = Long.parseLong(p.getProperty(TestProps.BW_MAX_MEM_BYTES));
+    int numThreads = Integer.parseInt(p.getProperty(TestProps.ACCUMULO_BW_NUM_THREADS));
+    long maxLatency = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_LATENCY_MS));
+    long maxMemory = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_MEM_BYTES));
 
     BatchWriterConfig config = new BatchWriterConfig();
     config.setMaxWriteThreads(numThreads);
@@ -176,6 +187,6 @@ public class TestEnv {
   }
 
   public int getScannerBatchSize() {
-    return Integer.parseInt(p.getProperty(TestProps.SCANNER_BATCH_SIZE));
+    return Integer.parseInt(p.getProperty(TestProps.ACCUMULO_SCANNER_BATCH_SIZE));
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
index bdc4901..b0927db 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
@@ -42,22 +42,27 @@ public class TestProps {
   public static final String ACCUMULO_USERNAME = COMMON + "accumulo.username";
   // Accumulo password
   public static final String ACCUMULO_PASSWORD = COMMON + "accumulo.password";
+  // Max memory (in bytes) each batch writer will use to buffer writes
+  public static final String ACCUMULO_BW_MAX_MEM_BYTES = COMMON + "accumulo.bw.max.memory.bytes";
+  // Max the maximum time (in ms) each batch writer will buffer data
+  public static final String ACCUMULO_BW_MAX_LATENCY_MS = COMMON + "accumulo.bw.max.latency.ms";
+  // Number of threads each batch writer will use to write data
+  public static final String ACCUMULO_BW_NUM_THREADS = COMMON + "accumulo.bw.num.threads";
+  // Number of thread for each batch scanner
+  public static final String ACCUMULO_BS_NUM_THREADS = COMMON + "accumulo.bw.num.threads";
+  // Number of key/value entries to pull during scan
+  public static final String ACCUMULO_SCANNER_BATCH_SIZE = COMMON + "accumulo.scanner.batch.size";
   // Accumulo keytab
   public static final String ACCUMULO_KEYTAB = COMMON + "accumulo.keytab";
+  // HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml
+  public static final String HDFS_ROOT = COMMON + "hdfs.root";
+  // YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property
in
+  // Hadoop's yarn-site.xml
+  public static final String YARN_RESOURCE_MANAGER = COMMON + "yarn.resource.manager";
   // Memory (in MB) given to each YARN container
   public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb";
   // Number of cores given to each YARN container
   public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores";
-  // Max memory (in bytes) each batch writer will use to buffer writes
-  public static final String BW_MAX_MEM_BYTES = COMMON + "bw.max.memory.bytes";
-  // Max the maximum time (in ms) each batch writer will buffer data
-  public static final String BW_MAX_LATENCY_MS = COMMON + "bw.max.latency.ms";
-  // Number of threads each batch writer will use to write data
-  public static final String BW_NUM_THREADS = COMMON + "bw.num.threads";
-  // Number of thread for each batch scanner
-  public static final String BS_NUM_THREADS = COMMON + "bw.num.threads";
-  // Number of key/value entries to pull during scan
-  public static final String SCANNER_BATCH_SIZE = COMMON + "scanner.batch.size";
 
   /** Continuous ingest test properties **/
   /** Common **/

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
index 0282c2b..88134a0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java
@@ -54,7 +54,7 @@ public class ContinuousBatchWalker {
 
     Random r = new Random();
 
-    int scanThreads = Integer.parseInt(props.getProperty(TestProps.BS_NUM_THREADS));
+    int scanThreads = Integer.parseInt(props.getProperty(TestProps.ACCUMULO_BS_NUM_THREADS));
 
     while (true) {
       BatchScanner bs = conn.createBatchScanner(env.getAccumuloTableName(), auths, scanThreads);

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/28ca75a3/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
index 8c147a7..30b2191 100644
--- a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.testing.yarn;
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import com.google.common.base.Preconditions;
+import org.apache.accumulo.testing.core.TestEnv;
 import org.apache.accumulo.testing.core.TestProps;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.twill.api.ResourceReport;
@@ -36,7 +37,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.io.FileInputStream;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -125,16 +126,18 @@ public class YarnAccumuloTestRunner {
     verifyPath(opts.testProps);
     verifyPath(opts.logProps);
 
+    String jarFileName = Paths.get(opts.jarPath).getFileName().toString();
+
     String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new);
-    BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(opts.jarPath, "/lib",
opts.mainClass, mainArgs);
+    BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(jarFileName, "/lib",
+                                                                          opts.mainClass,
mainArgs);
+
+    Properties props = TestProps.loadFromFile(opts.testProps);
+    TestEnv env = new TestEnv(props);
 
-    Properties props = new Properties();
-    FileInputStream fis = new FileInputStream(opts.testProps);
-    props.load(fis);
-    fis.close();
-    String zookeepers = props.getProperty(TestProps.ZOOKEEPERS);
+    YarnConfiguration yarnConfig = new YarnConfiguration(env.getHadoopConfiguration());
 
-    final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(),
zookeepers);
+    TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getZookeepers());
     twillRunner.start();
 
     TwillController controller = twillRunner.prepare(new YarnTestApp(opts, props))


Mime
View raw message