hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-16076 : LLAP packaging - include aux libs (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Date Fri, 10 Mar 2017 01:23:47 GMT
Repository: hive
Updated Branches:
  refs/heads/master 9d3ff496a -> 039fee7f9


HIVE-16076 : LLAP packaging - include aux libs (Sergey Shelukhin, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/039fee7f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/039fee7f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/039fee7f

Branch: refs/heads/master
Commit: 039fee7f944199eece949e24951e1f7be9c879d4
Parents: 9d3ff49
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Thu Mar 9 17:19:23 2017 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Thu Mar 9 17:19:23 2017 -0800

----------------------------------------------------------------------
 .../hive/llap/cli/LlapOptionsProcessor.java     | 22 ++++++++++++++----
 .../hadoop/hive/llap/cli/LlapServiceDriver.java | 24 ++++++++++++++++----
 2 files changed, 37 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/039fee7f/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index 6463b30..aa09083 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -48,6 +48,7 @@ public class LlapOptionsProcessor {
   public static final String OPTION_SIZE = "size"; // forward via config.json
   public static final String OPTION_XMX = "xmx"; // forward as arg
   public static final String OPTION_AUXJARS = "auxjars"; // used to localize jars
+  public static final String OPTION_AUXHIVE = "auxhive"; // used to localize jars
   public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars
   public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json
   public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant
parameter
@@ -86,11 +87,12 @@ public class LlapOptionsProcessor {
     private final String logger;
     private final boolean isStarting;
     private final String output;
+    private final boolean isHiveAux;
 
     public LlapOptions(String name, int instances, String directory, int executors, int ioThreads,
-                       long cache, long size, long xmx, String jars, boolean isHbase,
-                       @Nonnull Properties hiveconf, String javaPath, String llapQueueName,
-                       String logger, boolean isStarting, String output) throws ParseException
{
+        long cache, long size, long xmx, String jars, boolean isHbase,
+        @Nonnull Properties hiveconf, String javaPath, String llapQueueName, String logger,
+        boolean isStarting, String output, boolean isHiveAux) throws ParseException {
       if (instances <= 0) {
         throw new ParseException("Invalid configuration: " + instances
             + " (should be greater than 0)");
@@ -105,6 +107,7 @@ public class LlapOptionsProcessor {
       this.xmx = xmx;
       this.jars = jars;
       this.isHbase = isHbase;
+      this.isHiveAux = isHiveAux;
       this.conf = hiveconf;
       this.javaPath = javaPath;
       this.llapQueueName = llapQueueName;
@@ -157,6 +160,10 @@ public class LlapOptionsProcessor {
       return isHbase;
     }
 
+    public boolean getIsHiveAux() {
+      return isHiveAux;
+    }
+
     public Properties getConfig() {
       return conf;
     }
@@ -253,6 +260,9 @@ public class LlapOptionsProcessor {
     options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXHBASE).withLongOpt(OPTION_AUXHBASE)
         .withDescription("whether to package the HBase jars (true by default)").create('h'));
 
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXHIVE).withLongOpt(OPTION_AUXHIVE)
+        .withDescription("whether to package the Hive aux jars (true by default)").create(OPTION_AUXHIVE));
+
     options.addOption(OptionBuilder.hasArg().withArgName(OPTION_JAVA_HOME).withLongOpt(OPTION_JAVA_HOME)
         .withDescription(
             "Path to the JRE/JDK. This should be installed at the same location on all cluster
nodes ($JAVA_HOME, java.home by default)")
@@ -310,6 +320,8 @@ public class LlapOptionsProcessor {
     final long xmx = parseSuffixed(commandLine.getOptionValue(OPTION_XMX, "-1"));
     final boolean isHbase = Boolean.parseBoolean(
         commandLine.getOptionValue(OPTION_AUXHBASE, "true"));
+    final boolean isHiveAux = Boolean.parseBoolean(
+        commandLine.getOptionValue(OPTION_AUXHIVE, "true"));
     final boolean doStart = commandLine.hasOption(OPTION_START);
     final String output = commandLine.getOptionValue(OPTION_OUTPUT_DIR, null);
 
@@ -340,8 +352,8 @@ public class LlapOptionsProcessor {
 
     // loglevel, chaosmonkey & args are parsed by the python processor
 
-    return new LlapOptions(name, instances, directory, executors, ioThreads, cache,
-        size, xmx, jars, isHbase, hiveconf, javaHome, queueName, logger, doStart, output);
+    return new LlapOptions(name, instances, directory, executors, ioThreads, cache, size,
xmx,
+        jars, isHbase, hiveconf, javaHome, queueName, logger, doStart, output, isHiveAux);
   }
 
   private void printUsage() {

http://git-wip-us.apache.org/repos/asf/hive/blob/039fee7f/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index e8517ab..22e5ee8 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -102,7 +102,7 @@ public class LlapServiceDriver {
    * This is a working configuration for the instance to merge various variables.
    * It is not written out for llap server usage
    */
-  private final Configuration conf;
+  private final HiveConf conf;
 
   public LlapServiceDriver() {
     SessionState ss = SessionState.get();
@@ -435,17 +435,33 @@ public class LlapServiceDriver {
             }
           }
 
-          String auxJars = options.getAuxJars();
+          HashSet<String> auxJars = new HashSet<>();
+          // There are many ways to have AUX jars in Hive... sigh
+          if (options.getIsHiveAux()) {
+            // Note: we don't add ADDED jars, RELOADABLE jars, etc. That is by design; there
are too many ways
+            // to add jars in Hive, some of which are session/etc. specific. Env + conf +
arg should be enough.
+            addAuxJarsToSet(auxJars, conf.getAuxJars());
+            addAuxJarsToSet(auxJars, System.getenv("HIVE_AUX_JARS_PATH"));
+            LOG.info("Adding the following aux jars from the environment and configs: " +
auxJars);
+          }
+
+          addAuxJarsToSet(auxJars, options.getAuxJars());
+          for (String jarPath : auxJars) {
+            lfs.copyFromLocalFile(new Path(jarPath), libDir);
+          }
+          return null;
+        }
+
+        private void addAuxJarsToSet(HashSet<String> auxJarSet, String auxJars) {
           if (auxJars != null && !auxJars.isEmpty()) {
             // TODO: transitive dependencies warning?
             String[] jarPaths = auxJars.split(",");
             for (String jarPath : jarPaths) {
               if (!jarPath.isEmpty()) {
-                lfs.copyFromLocalFile(new Path(jarPath), libDir);
+                auxJarSet.add(jarPath);
               }
             }
           }
-          return null;
         }
       };
 


Mime
View raw message