accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [accumulo-testing] branch master updated: Updates for starting MapReduce jobs from randomwalk (#49)
Date Thu, 10 Jan 2019 17:39:01 GMT
This is an automated email from the ASF dual-hosted git repository.

mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git


The following commit(s) were added to refs/heads/master by this push:
     new a324da8  Updates for starting MapReduce jobs from randomwalk (#49)
a324da8 is described below

commit a324da87f21e1df4f12c7cf0ed4a94cb7c5dc83c
Author: Mike Walch <mwalch@apache.org>
AuthorDate: Thu Jan 10 12:38:56 2019 -0500

    Updates for starting MapReduce jobs from randomwalk (#49)
    
    * Set additional MapReduce configuration and Hadoop username
    * These settings work if running randomwalk on user machine
      or in Docker
---
 Dockerfile                                             | 5 +++++
 README.md                                              | 5 +++--
 bin/rwalk                                              | 2 +-
 conf/env.sh.example                                    | 2 --
 src/main/docker/docker-entry                           | 5 +++++
 src/main/java/org/apache/accumulo/testing/TestEnv.java | 7 +++++++
 6 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4038764..d543d8e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,6 +15,11 @@
 
 FROM centos:7
 
+ARG HADOOP_HOME
+ARG HADOOP_USER_NAME
+ENV HADOOP_HOME ${HADOOP_HOME}
+ENV HADOOP_USER_NAME ${HADOOP_USER_NAME:-hadoop}
+
 RUN yum install -y java-1.8.0-openjdk-devel
 ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk
 
diff --git a/README.md b/README.md
index d659f8d..1a758a1 100644
--- a/README.md
+++ b/README.md
@@ -50,10 +50,11 @@ run in Docker:
     * `conf/accumulo-testing.properties` - Copy this from the example file and configure
it
     * `target/accumulo-testing-2.0.0-SNAPSHOT-shaded.jar` - Can be created using `./bin/build`
 
-   Run the following command to create the image:
+   Run the following command to create the image. `HADOOP_HOME` should be where Hadoop is
installed on your cluster.
+   `HADOOP_USER_NAME` should match the user running Hadoop on your cluster.
 
    ```
-   docker build -t accumulo-testing .
+   docker build --build-arg HADOOP_HOME=$HADOOP_HOME --build-arg HADOOP_USER_NAME=`whoami`
-t accumulo-testing .
    ```
 
 2. The `accumulo-testing` image can run a single command:
diff --git a/bin/rwalk b/bin/rwalk
index 6bd7299..30cf0a7 100755
--- a/bin/rwalk
+++ b/bin/rwalk
@@ -34,7 +34,7 @@ else
   . "$at_home"/conf/env.sh.example
 fi
 
-export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$HADOOP_CONF_DIR:$CLASSPATH"
+export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
 
 randomwalk_main="org.apache.accumulo.testing.randomwalk.Framework"
 
diff --git a/conf/env.sh.example b/conf/env.sh.example
index a48451c..bd372c3 100644
--- a/conf/env.sh.example
+++ b/conf/env.sh.example
@@ -18,8 +18,6 @@
 
 ## Hadoop installation
 export HADOOP_HOME="${HADOOP_HOME:-/path/to/hadoop}"
-## Hadoop configuration
-export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_HOME}/etc/hadoop}"
 ## Accumulo installation
 export ACCUMULO_HOME="${ACCUMULO_HOME:-/path/to/accumulo}"
 ## Path to Accumulo client properties
diff --git a/src/main/docker/docker-entry b/src/main/docker/docker-entry
index 5a98525..6f23e00 100755
--- a/src/main/docker/docker-entry
+++ b/src/main/docker/docker-entry
@@ -36,6 +36,11 @@ if [ -z "$1" ]; then
   exit 1
 fi
 
+if [ -z "$HADOOP_HOME" ]; then
+  echo "HADOOP_HOME must be set!"
+  exit 1
+fi
+
 case "$1" in
   cingest|rwalk)
     "${at_home}"/bin/"$1" "${@:2}"
diff --git a/src/main/java/org/apache/accumulo/testing/TestEnv.java b/src/main/java/org/apache/accumulo/testing/TestEnv.java
index e5ffa1a..601db3c 100644
--- a/src/main/java/org/apache/accumulo/testing/TestEnv.java
+++ b/src/main/java/org/apache/accumulo/testing/TestEnv.java
@@ -97,6 +97,13 @@ public class TestEnv implements AutoCloseable {
       hadoopConfig.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
       hadoopConfig.set("mapreduce.framework.name", "yarn");
       hadoopConfig.set("yarn.resourcemanager.hostname", getYarnResourceManager());
+      String hadoopHome = System.getenv("HADOOP_HOME");
+      if (hadoopHome == null) {
+        throw new IllegalArgumentException("HADOOP_HOME must be set in env");
+      }
+      hadoopConfig.set("yarn.app.mapreduce.am.env", "HADOOP_MAPRED_HOME=" + hadoopHome);
+      hadoopConfig.set("mapreduce.map.env", "HADOOP_MAPRED_HOME=" + hadoopHome);
+      hadoopConfig.set("mapreduce.reduce.env", "HADOOP_MAPRED_HOME=" + hadoopHome);
     }
     return hadoopConfig;
   }


Mime
View raw message