Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E5FF3181C2 for ; Fri, 18 Dec 2015 23:36:52 +0000 (UTC) Received: (qmail 97909 invoked by uid 500); 18 Dec 2015 23:36:52 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 97865 invoked by uid 500); 18 Dec 2015 23:36:52 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 97849 invoked by uid 99); 18 Dec 2015 23:36:52 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Dec 2015 23:36:52 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 60E93DF99E; Fri, 18 Dec 2015 23:36:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Date: Fri, 18 Dec 2015 23:36:53 -0000 Message-Id: <1462d9f3073e480d9386e149136d2a19@git.apache.org> In-Reply-To: <299def9dad284e828a5c7dfd834051a4@git.apache.org> References: <299def9dad284e828a5c7dfd834051a4@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] hive git commit: HIVE-12633 : LLAP: package included serde jars (Sergey Shelukhin/Gopal V, reviewed by Gopal V) HIVE-12633 : LLAP: package included serde jars (Sergey Shelukhin/Gopal V, reviewed by Gopal V) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4da7ed91 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4da7ed91 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4da7ed91 Branch: refs/heads/branch-2.0 Commit: 4da7ed913918792bd212109d8c4f72678136d979 Parents: 1d5e9c9 Author: Sergey Shelukhin Authored: Fri Dec 18 15:36:19 2015 -0800 Committer: Sergey Shelukhin Committed: Fri Dec 18 15:36:35 2015 -0800 ---------------------------------------------------------------------- .../hive/llap/cli/LlapOptionsProcessor.java | 16 +++++++- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 43 +++++++++++++++++++- 2 files changed, 55 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4da7ed91/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java index 8fd615c..58ef472 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java @@ -44,10 +44,11 @@ public class LlapOptionsProcessor { private final long cache; private final long size; private final long xmx; + private final String jars; private final Properties conf; public LlapOptions(String name, int instances, String directory, int executors, long cache, - long size, long xmx, @Nonnull Properties hiveconf) throws ParseException { + long size, long xmx, String jars, @Nonnull Properties hiveconf) throws ParseException { if (instances <= 0) { throw new ParseException("Invalid configuration: " + instances + " (should be greater than 0)"); @@ -59,6 +60,7 @@ public class LlapOptionsProcessor { this.cache = cache; this.size = size; this.xmx = xmx; + this.jars = jars; this.conf = hiveconf; } @@ -90,6 +92,10 @@ public class LlapOptionsProcessor { return xmx; } + public String getAuxJars() { + return jars; + } + public Properties getConfig() { return conf; } @@ -134,6 +140,10 @@ public class LlapOptionsProcessor { options.addOption(OptionBuilder.hasArg().withArgName("xmx").withLongOpt("xmx") .withDescription("working memory size").create('w')); + options.addOption(OptionBuilder.hasArg().withArgName("auxjars").withLongOpt("auxjars") + .withDescription("additional jars to package (by default, JSON and HBase SerDe jars" + + " are packaged if available)").create('j')); + // -hiveconf x=y options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value") .withLongOpt("hiveconf").withDescription("Use value for given property").create()); @@ -156,6 +166,7 @@ public class LlapOptionsProcessor { int instances = Integer.parseInt(commandLine.getOptionValue("instances")); String directory = commandLine.getOptionValue("directory"); + String jars = commandLine.getOptionValue("auxjars"); String name = commandLine.getOptionValue("name", null); @@ -174,7 +185,8 @@ public class LlapOptionsProcessor { // loglevel, chaosmonkey & args are parsed by the python processor - return new LlapOptions(name, instances, directory, executors, cache, size, xmx, hiveconf); + return new LlapOptions( + name, instances, directory, executors, cache, size, xmx, jars, hiveconf); } http://git-wip-us.apache.org/repos/asf/hive/blob/4da7ed91/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 08d573b..8e5377f 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hive.llap.cli; import java.io.OutputStreamWriter; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -45,6 +47,9 @@ import com.google.common.base.Preconditions; public class LlapServiceDriver { protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName()); + private static final String[] DEFAULT_AUX_CLASSES = new String[] { + "org.apache.hive.hcatalog.data.JsonSerDe", "org.apache.hadoop.hive.hbase.HBaseSerDe" }; + private final Configuration conf; public LlapServiceDriver() { @@ -204,11 +209,45 @@ public class LlapServiceDriver { CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), libDir.toString(), true); lfs.delete(new Path(libDir, "tez.tar.gz"), false); - // TODO: aux jars (like compression libs) - lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapInputFormat.class)), libDir); lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(HiveInputFormat.class)), libDir); + // copy default aux classes (json/hbase) + + for (String className : DEFAULT_AUX_CLASSES) { + String jarPath = null; + boolean hasException = false; + try { + Class auxClass = Class.forName(className); + jarPath = Utilities.jarFinderGetJar(auxClass); + } catch (Throwable t) { + hasException = true; + String err = + "Cannot find a jar for [" + className + "] due to an exception (" + t.getMessage() + + "); not packaging the jar"; + LOG.error(err, t); + System.err.println(err); + } + if (jarPath != null) { + lfs.copyFromLocalFile(new Path(jarPath), libDir); + } else if (!hasException) { + String err = "Cannot find a jar for [" + className + "]; not packaging the jar"; + LOG.error(err); + System.err.println(err); + } + } + + String auxJars = options.getAuxJars(); + if (auxJars != null && !auxJars.isEmpty()) { + // TODO: transitive dependencies warning? + String[] jarPaths = auxJars.split(","); + for (String jarPath : jarPaths) { + if (!jarPath.isEmpty()) { + lfs.copyFromLocalFile(new Path(jarPath), libDir); + } + } + } + Path confPath = new Path(tmpDir, "conf"); lfs.mkdirs(confPath);