accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [accumulo-testing] branch master updated: Build shaded jar without Hadoop for M/R (#44)
Date Wed, 28 Nov 2018 19:12:32 GMT
This is an automated email from the ASF dual-hosted git repository.

mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git


The following commit(s) were added to refs/heads/master by this push:
     new 8cb9cbb  Build shaded jar without Hadoop for M/R (#44)
8cb9cbb is described below

commit 8cb9cbbf08b173ed3e9ff188c6516def2eb1bc5b
Author: Mike Walch <mwalch@apache.org>
AuthorDate: Wed Nov 28 14:12:28 2018 -0500

    Build shaded jar without Hadoop for M/R (#44)
    
    * Hadoop jars get added to CLASSPATH when needed
---
 Dockerfile          | 46 ++++++++++++++++++++++++++--------------------
 bin/cingest         |  3 +--
 bin/rwalk           |  2 +-
 conf/env.sh.example | 14 ++++++++++----
 pom.xml             |  4 ++--
 5 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 85cce6e..4038764 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,24 +18,30 @@ FROM centos:7
 RUN yum install -y java-1.8.0-openjdk-devel
 ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk
 
-ARG TEST_JAR_VERSION
-ENV TEST_JAR_VERSION 2.0.0-SNAPSHOT
-ENV TEST_JAR_PATH /opt/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar
-ENV ACCUMULO_CLIENT_PROPS /opt/conf/accumulo-client.properties
-ENV TEST_PROPS /opt/conf/accumulo-testing.properties
-ENV TEST_LOG4J /opt/conf/log4j.properties.example
-
-RUN mkdir /opt/bin
-RUN mkdir /opt/conf
-RUN touch /opt/conf/env.sh
-
-ADD ./conf/accumulo-client.properties /opt/conf/
-ADD ./conf/accumulo-testing.properties /opt/conf/
-ADD ./conf/log4j.properties.example /opt/conf/
-ADD ./bin/cingest /opt/bin
-ADD ./bin/rwalk /opt/bin
-ADD ./src/main/docker/docker-entry /opt/bin
-ADD ./target/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar /opt/
-
-ENTRYPOINT ["/opt/bin/docker-entry"]
+ENV HADOOP_API_JAR /opt/at/hadoop-client-api.jar
+ENV HADOOP_RUNTIME_JAR /opt/at/hadoop-client-runtime.jar
+ENV TEST_JAR_PATH /opt/at/accumulo-testing-shaded.jar
+ENV ACCUMULO_CLIENT_PROPS /opt/at/conf/accumulo-client.properties
+ENV TEST_PROPS /opt/at/conf/accumulo-testing.properties
+ENV TEST_LOG4J /opt/at/conf/log4j.properties
+
+RUN mkdir /opt/at
+RUN mkdir /opt/at/bin
+RUN mkdir /opt/at/conf
+
+COPY ./conf/accumulo-client.properties /opt/at/conf/
+COPY ./conf/accumulo-testing.properties /opt/at/conf/
+COPY ./conf/log4j.properties* /opt/at/conf/
+RUN if [[ ! -f /opt/at/conf/log4j.properties ]]; then mv /opt/at/conf/log4j.properties.example
/opt/at/conf/log4j.properties; fi
+RUN touch /opt/at/conf/env.sh
+
+COPY ./bin/cingest /opt/at/bin
+COPY ./bin/rwalk /opt/at/bin
+COPY ./src/main/docker/docker-entry /opt/at/bin
+
+COPY ./target/accumulo-testing-shaded.jar /opt/at/
+COPY ./target/dependency/hadoop-client-api.jar /opt/at/
+COPY ./target/dependency/hadoop-client-runtime.jar /opt/at/
+
+ENTRYPOINT ["/opt/at/bin/docker-entry"]
 CMD ["help"]
diff --git a/bin/cingest b/bin/cingest
index 5801a12..6cbbb40 100755
--- a/bin/cingest
+++ b/bin/cingest
@@ -77,12 +77,11 @@ case "$1" in
     exit 1
 esac
 
-export CLASSPATH="$TEST_JAR_PATH:$CLASSPATH"
+export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
 
 case "$1" in
   verify|moru)
     if [ ! -z $HADOOP_HOME ]; then
-      export CLASSPATH=$HADOOP_HOME/etc/hadoop:$CLASSPATH
       export HADOOP_USE_CLIENT_CLASSLOADER=true
       "$HADOOP_HOME"/bin/yarn jar "$TEST_JAR_PATH" "$ci_main" "$TEST_PROPS" "$ACCUMULO_CLIENT_PROPS"
     else
diff --git a/bin/rwalk b/bin/rwalk
index 8a17b4e..30cf0a7 100755
--- a/bin/rwalk
+++ b/bin/rwalk
@@ -34,7 +34,7 @@ else
   . "$at_home"/conf/env.sh.example
 fi
 
-export CLASSPATH="$TEST_JAR_PATH:$CLASSPATH"
+export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
 
 randomwalk_main="org.apache.accumulo.testing.randomwalk.Framework"
 
diff --git a/conf/env.sh.example b/conf/env.sh.example
index a3368fc..0a2cdbe 100644
--- a/conf/env.sh.example
+++ b/conf/env.sh.example
@@ -42,21 +42,27 @@ fi
 
 # Shaded test jar
 # ===============
-# Version of shaded test jar
-export TEST_JAR_VERSION=2.0.0-SNAPSHOT
 # Versions set below will be what is included in the shaded jar
 export ACCUMULO_VERSION="`$ACCUMULO_HOME/bin/accumulo version`"
 export HADOOP_VERSION="`hadoop version | head -n1 | awk '{print $2}'`"
 export ZOOKEEPER_VERSION=3.4.9
 # Path to shaded test jar
 at_home=$( cd "$( dirname "$conf_dir" )" && pwd )
-export TEST_JAR_PATH="${at_home}/target/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar"
-if [ ! -f "$TEST_JAR_PATH" ]; then
+export TEST_JAR_PATH="${at_home}/target/accumulo-testing-shaded.jar"
+if [[ ! -f "$TEST_JAR_PATH" ]]; then
   echo "Building $TEST_JAR_PATH"
   cd "${at_home}" || exit 1
   mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION"
-D hadoop.version="$HADOOP_VERSION" -D zookeeper.version="$ZOOKEEPER_VERSION"
 fi
 
+# Hadoop jars
+# ===========
+export HADOOP_API_JAR="${at_home}"/target/dependency/hadoop-client-api.jar
+export HADOOP_RUNTIME_JAR="${at_home}"/target/dependency/hadoop-client-runtime.jar
+if [[ ! -f "$HADOOP_API_JAR" || ! -f "$HADOOP_RUNTIME_JAR" ]]; then
+  mvn dependency:copy-dependencies -Dmdep.stripVersion=true -DincludeArtifactIds=hadoop-client-api,hadoop-client-runtime
-Dhadoop.version="$HADOOP_VERSION"
+fi
+
 # Agitator
 # ========
 # Accumulo user
diff --git a/pom.xml b/pom.xml
index 8d4c940..a9c534f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -208,11 +208,13 @@
                 </goals>
                 <phase>package</phase>
                 <configuration>
+                  <finalName>${project.artifactId}-shaded</finalName>
                   <shadedArtifactAttached>true</shadedArtifactAttached>
                   <shadedClassifierName>shaded</shadedClassifierName>
                   <artifactSet>
                     <excludes>
                       <exclude>org.apache.accumulo:accumulo-native</exclude>
+                      <exclude>org.apache.hadoop:*</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>
@@ -222,8 +224,6 @@
                         <exclude>META-INF/*.SF</exclude>
                         <exclude>META-INF/*.DSA</exclude>
                         <exclude>META-INF/*.RSA</exclude>
-                        <!-- Twill 0.13 fails if shaded jar includes this directory !-->
-                        <exclude>META-INF/license/*</exclude>
                       </excludes>
                     </filter>
                   </filters>


Mime
View raw message