accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [accumulo] branch master updated: Remove 'accumulo-util hadoop-jar' command (#872)
Date Sat, 05 Jan 2019 22:54:21 GMT
This is an automated email from the ASF dual-hosted git repository.

mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new c265ea5  Remove 'accumulo-util hadoop-jar' command (#872)
c265ea5 is described below

commit c265ea5b16171032419b164e809f5478f70bbba8
Author: Mike Walch <mwalch@apache.org>
AuthorDate: Sat Jan 5 17:54:17 2019 -0500

    Remove 'accumulo-util hadoop-jar' command (#872)
    
    * Command doesn't work well now that Accumulo does
      not match Hadoop's dependenices
    * Users should create shaded jar instead and submit
      using Hadoop's 'yarn jar' command
    * Command was used by StandaloneClusterControl but only
      by unit test
---
 assemble/bin/accumulo-util                         | 69 ----------------------
 .../standalone/StandaloneClusterControl.java       | 37 +-----------
 .../standalone/StandaloneClusterControlTest.java   | 49 ---------------
 3 files changed, 2 insertions(+), 153 deletions(-)

diff --git a/assemble/bin/accumulo-util b/assemble/bin/accumulo-util
index d243cd4..13eb802 100755
--- a/assemble/bin/accumulo-util
+++ b/assemble/bin/accumulo-util
@@ -22,7 +22,6 @@ Usage: accumulo-util <command> (<argument> ...)
 Commands:
   build-native        Builds Accumulo native libraries
   dump-zoo            Dumps data in ZooKeeper
-  hadoop-jar          Runs 'hadoop jar' command with Accumulo jars
   gen-monitor-cert    Generates Accumulo monitor certficate
   load-jars-hdfs      Loads Accumulo jars in lib/ to HDFS for VFS classloader
   
@@ -182,71 +181,6 @@ function load_jars_hdfs() {
   "$HADOOP" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar"  > /dev/null
 }
 
-function hadoop_jar() {
-  if [[ -x "$HADOOP_HOME/bin/hadoop" ]]; then
-    HADOOP="$HADOOP_HOME/bin/hadoop"
-  else
-    HADOOP=$(which hadoop)
-  fi
-  if [[ ! -x "$HADOOP" ]]; then
-    echo "Could not find 'hadoop' command. Please set hadoop on your PATH or set HADOOP_HOME"
-    exit 1
-  fi
-  if [[ -z "$ZOOKEEPER_HOME" ]]; then
-     echo "ZOOKEEPER_HOME must be set!"
-     exit 1
-  fi
-
-  ZOOKEEPER_CMD="ls -1 $ZOOKEEPER_HOME/zookeeper-[0-9]*[^csn].jar "
-  if [[ $(eval "$ZOOKEEPER_CMD" | wc -l) -ne 1 ]] ; then
-     echo "Not exactly one zookeeper jar in $ZOOKEEPER_HOME"
-     exit 1
-  fi
-  ZOOKEEPER_LIB=$(eval "$ZOOKEEPER_CMD")
-
-  CORE_LIB="${lib}/accumulo-core.jar"
-  THRIFT_LIB="${lib}/libthrift.jar"
-  JCOMMANDER_LIB="${lib}/jcommander.jar"
-  COMMONS_VFS_LIB="${lib}/commons-vfs2.jar"
-  GUAVA_LIB="${lib}/guava.jar"
-  HTRACE_LIB="${lib}/htrace-core.jar"
-
-  USERJARS=" "
-  for arg in "$@"; do
-      if [[ "$arg" != "-libjars" ]] && [[ -z "$TOOLJAR" ]]; then
-        TOOLJAR="$arg"
-        shift
-     elif [[ "$arg" != "-libjars" ]] && [[ -z "$CLASSNAME" ]]; then
-        CLASSNAME="$arg"
-        shift
-     elif [[ -z "$USERJARS" ]]; then
-        USERJARS=$(echo "$arg" | tr "," " ")
-        shift
-     elif [[ "$arg" = "-libjars" ]]; then
-        USERJARS=""
-        shift
-     else
-        break
-     fi
-  done
-
-  LIB_JARS="$THRIFT_LIB,$CORE_LIB,$ZOOKEEPER_LIB,$JCOMMANDER_LIB,$COMMONS_VFS_LIB,$GUAVA_LIB,$HTRACE_LIB"
-  H_JARS="$THRIFT_LIB:$CORE_LIB:$ZOOKEEPER_LIB:$JCOMMANDER_LIB:$COMMONS_VFS_LIB:$GUAVA_LIB:$HTRACE_LIB"
-
-  for jar in $USERJARS; do
-     LIB_JARS="$LIB_JARS,$jar"
-     H_JARS="$H_JARS:$jar"
-  done
-  export HADOOP_CLASSPATH="$H_JARS:$HADOOP_CLASSPATH"
-
-  if [[ -z "$CLASSNAME" || -z "$TOOLJAR" ]]; then
-     echo "Usage: accumulo-util hadoop-jar path/to/myTool.jar my.tool.class.Name [-libjars
my1.jar,my2.jar]" 1>&2
-     exit 1
-  fi
-
-  exec "$HADOOP" jar "$TOOLJAR" "$CLASSNAME" -libjars "$LIB_JARS" "$@"
-}
-
 function main() {
   SOURCE="${BASH_SOURCE[0]}"
   while [ -h "${SOURCE}" ]; do
@@ -266,9 +200,6 @@ function main() {
     dump-zoo)
       "$bin"/accumulo org.apache.accumulo.server.util.DumpZookeeper "${@:2}"
       ;;
-    hadoop-jar)
-      hadoop_jar "${@:2}"
-      ;;
     gen-monitor-cert)
       gen_monitor_cert
       ;;
diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
index c374e86..13c33f7 100644
--- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
@@ -53,7 +53,7 @@ public class StandaloneClusterControl implements ClusterControl {
   private static final Logger log = LoggerFactory.getLogger(StandaloneClusterControl.class);
 
   private static final String ACCUMULO_SERVICE_SCRIPT = "accumulo-service",
-      ACCUMULO_SCRIPT = "accumulo", ACCUMULO_UTIL_SCRIPT = "accumulo-util";
+      ACCUMULO_SCRIPT = "accumulo";
   private static final String MASTER_HOSTS_FILE = "masters", GC_HOSTS_FILE = "gc",
       TSERVER_HOSTS_FILE = "tservers", TRACER_HOSTS_FILE = "tracers",
       MONITOR_HOSTS_FILE = "monitor";
@@ -65,7 +65,7 @@ public class StandaloneClusterControl implements ClusterControl {
   private String serverCmdPrefix;
   protected RemoteShellOptions options;
 
-  protected String accumuloServicePath, accumuloPath, accumuloUtilPath;
+  protected String accumuloServicePath, accumuloPath;
 
   @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN",
       justification = "code runs in same security context as user who provided input file
name")
@@ -81,11 +81,6 @@ public class StandaloneClusterControl implements ClusterControl {
     File bin = new File(accumuloHome, "bin");
     this.accumuloServicePath = new File(bin, ACCUMULO_SERVICE_SCRIPT).getAbsolutePath();
     this.accumuloPath = new File(bin, ACCUMULO_SCRIPT).getAbsolutePath();
-    this.accumuloUtilPath = new File(bin, ACCUMULO_UTIL_SCRIPT).getAbsolutePath();
-  }
-
-  String getAccumuloUtilPath() {
-    return this.accumuloUtilPath;
   }
 
   protected Entry<Integer,String> exec(String hostname, String[] command) throws IOException
{
@@ -133,34 +128,6 @@ public class StandaloneClusterControl implements ClusterControl {
     return msg.replaceAll("[\r\n]", "");
   }
 
-  public Entry<Integer,String> execMapreduceWithStdout(Class<?> clz, String[]
args)
-      throws IOException {
-    String host = "localhost";
-    List<String> cmd = new ArrayList<>();
-    cmd.add(getAccumuloUtilPath());
-    cmd.add("hadoop-jar");
-    cmd.add(getJarFromClass(clz));
-    cmd.add(clz.getName());
-    for (String arg : args) {
-      cmd.add("'" + arg + "'");
-    }
-    log.info("Running: '{}' on {}", StringUtils.join(cmd, " "), host);
-    return exec(host, cmd.toArray(new String[cmd.size()]));
-  }
-
-  String getJarFromClass(Class<?> clz) {
-    CodeSource source = clz.getProtectionDomain().getCodeSource();
-    if (source == null) {
-      throw new RuntimeException("Could not get CodeSource for class");
-    }
-    URL jarUrl = source.getLocation();
-    String jar = jarUrl.getPath();
-    if (!jar.endsWith(".jar")) {
-      throw new RuntimeException("Need to have a jar to run mapreduce: " + jar);
-    }
-    return jar;
-  }
-
   @Override
   public void adminStopAll() throws IOException {
     String master = getHosts(MASTER_HOSTS_FILE).get(0);
diff --git a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
index 1c31f89..5e23b83 100644
--- a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
+++ b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
@@ -16,19 +16,10 @@
  */
 package org.apache.accumulo.cluster.standalone;
 
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.aryEq;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
 import static org.junit.Assert.assertEquals;
 
-import org.easymock.EasyMock;
 import org.junit.Test;
 
-import com.google.common.collect.Maps;
-
 public class StandaloneClusterControlTest {
 
   @Test
@@ -46,44 +37,4 @@ public class StandaloneClusterControlTest {
     assertEquals(accumuloHome + "/bin/accumulo", control.accumuloPath);
     assertEquals(accumuloHome + "/bin/accumulo-service", control.accumuloServicePath);
   }
-
-  @Test
-  public void mapreduceLaunchesLocally() throws Exception {
-    final String accumuloUtilPath = "/usr/lib/accumulo/bin/accumulo-util";
-    final String jar = "/home/user/my_project.jar";
-    final Class<?> clz = Object.class;
-    final String myClass = clz.getName();
-    StandaloneClusterControl control = EasyMock.createMockBuilder(StandaloneClusterControl.class)
-        .addMockedMethod("exec", String.class, String[].class)
-        .addMockedMethod("getAccumuloUtilPath").addMockedMethod("getJarFromClass", Class.class)
-        .createMock();
-
-    final String[] toolArgs = {"-u", "user", "-p", "password"};
-    final String[] expectedCommands = new String[4 + toolArgs.length];
-
-    int i = 0;
-    expectedCommands[i++] = accumuloUtilPath;
-    expectedCommands[i++] = "hadoop-jar";
-    expectedCommands[i++] = jar;
-    expectedCommands[i++] = myClass;
-    for (int j = 0; j < toolArgs.length; j++) {
-      expectedCommands[i + j] = quote(toolArgs[j]);
-    }
-
-    expect(control.getAccumuloUtilPath()).andReturn(accumuloUtilPath);
-    expect(control.getJarFromClass(anyObject(Class.class))).andReturn(jar);
-    expect(control.exec(eq("localhost"), aryEq(expectedCommands)))
-        .andReturn(Maps.immutableEntry(0, ""));
-
-    replay(control);
-
-    // Give a fake Class -- we aren't verifying the actual class passed in
-    control.execMapreduceWithStdout(clz, toolArgs);
-
-    verify(control);
-  }
-
-  private String quote(String word) {
-    return "'" + word + "'";
-  }
 }


Mime
View raw message