hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject hive git commit: HIVE-10981: LLAP: Accept --hiveconf parameters for the LlapServiceDriver (gopalv)
Date Tue, 30 Jun 2015 05:18:19 GMT
Repository: hive
Updated Branches:
  refs/heads/llap 163cefb5d -> e9b7d7584


HIVE-10981: LLAP: Accept --hiveconf parameters for the LlapServiceDriver (gopalv)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e9b7d758
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e9b7d758
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e9b7d758

Branch: refs/heads/llap
Commit: e9b7d75848b3f117baf5bb4fb0ff99f101798a02
Parents: 163cefb
Author: Gopal V <gopalv@apache.org>
Authored: Mon Jun 29 22:03:45 2015 -0700
Committer: Gopal V <gopalv@apache.org>
Committed: Mon Jun 29 22:17:35 2015 -0700

----------------------------------------------------------------------
 .../hive/llap/cli/LlapOptionsProcessor.java     | 48 ++++++++++++++------
 .../hadoop/hive/llap/cli/LlapServiceDriver.java | 11 +++++
 llap-server/src/main/resources/package.py       |  2 +-
 3 files changed, 47 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e9b7d758/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index 211cd8b..e55b083 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.hive.llap.cli;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
+
+import javax.annotation.Nonnull;
 
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
@@ -34,16 +37,17 @@ import org.apache.hadoop.util.StringUtils;
 public class LlapOptionsProcessor {
 
   public class LlapOptions {
-    private int instances = 0;
-    private String directory = null;
-    private String name;
-    private int executors;
-    private long cache;
-    private long size;
-    private long xmx;
+    private final int instances;
+    private final String directory;
+    private final String name;
+    private final int executors;
+    private final long cache;
+    private final long size;
+    private final long xmx;
+    private final Properties conf;
 
     public LlapOptions(String name, int instances, String directory, int executors, long
cache,
-        long size, long xmx) throws ParseException {
+        long size, long xmx, @Nonnull Properties hiveconf) throws ParseException {
       if (instances <= 0) {
         throw new ParseException("Invalid configuration: " + instances
             + " (should be greater than 0)");
@@ -55,6 +59,7 @@ public class LlapOptionsProcessor {
       this.cache = cache;
       this.size = size;
       this.xmx = xmx;
+      this.conf = hiveconf;
     }
 
     public String getName() {
@@ -84,6 +89,10 @@ public class LlapOptionsProcessor {
     public long getXmx() {
       return xmx;
     }
+
+    public Properties getConfig() {
+      return conf;
+    }
   }
 
   protected static final Log l4j = LogFactory.getLog(LlapOptionsProcessor.class.getName());
@@ -125,6 +134,10 @@ public class LlapOptionsProcessor {
     options.addOption(OptionBuilder.hasArg().withArgName("xmx").withLongOpt("xmx")
         .withDescription("working memory size").create('w'));
 
+    // -hiveconf x=y
+    options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value")
+        .withLongOpt("hiveconf").withDescription("Use value for given property").create());
+
     // [-H|--help]
     options.addOption(new Option("H", "help", false, "Print help information"));
   }
@@ -146,14 +159,23 @@ public class LlapOptionsProcessor {
 
     String name = commandLine.getOptionValue("name", null);
 
-    int executors = Integer.parseInt(commandLine.getOptionValue("executors", "-1"));
-    long cache = parseSuffixed(commandLine.getOptionValue("cache", "-1"));
-    long size = parseSuffixed(commandLine.getOptionValue("size", "-1"));
-    long xmx = parseSuffixed(commandLine.getOptionValue("xmx", "-1"));
+    final int executors = Integer.parseInt(commandLine.getOptionValue("executors", "-1"));
+    final long cache = parseSuffixed(commandLine.getOptionValue("cache", "-1"));
+    final long size = parseSuffixed(commandLine.getOptionValue("size", "-1"));
+    final long xmx = parseSuffixed(commandLine.getOptionValue("xmx", "-1"));
+
+    final Properties hiveconf;
+
+    if (commandLine.hasOption("hiveconf")) {
+      hiveconf = commandLine.getOptionProperties("hiveconf");
+    } else {
+      hiveconf = new Properties();
+    }
 
     // loglevel, chaosmonkey & args are parsed by the python processor
 
-    return new LlapOptions(name, instances, directory, executors, cache, size, xmx);
+    return new LlapOptions(name, instances, directory, executors, cache, size, xmx, hiveconf);
+
   }
 
   private void printUsage() {

http://git-wip-us.apache.org/repos/asf/hive/blob/e9b7d758/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index b3d155b..05fecc7 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.hive.llap.cli;
 
 import java.io.OutputStreamWriter;
 import java.net.URL;
+import java.util.Enumeration;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -168,6 +170,10 @@ public class LlapServiceDriver {
           / (1024 * 1024));
     }
 
+    for (Entry<Object, Object> props : options.getConfig().entrySet()) {
+      conf.set((String) props.getKey(), (String) props.getValue());
+    }
+
     URL logger = conf.getResource("llap-daemon-log4j.properties");
 
     if (null == logger) {
@@ -213,6 +219,11 @@ public class LlapServiceDriver {
 
         Configuration copy = resolve(conf, "llap-daemon-site.xml");
 
+        for (Entry<Object, Object> props : options.getConfig().entrySet()) {
+          // overrides
+          copy.set((String) props.getKey(), (String) props.getValue());
+        }
+
         copy.writeXml(confStream);
         confStream.close();
       } else {

http://git-wip-us.apache.org/repos/asf/hive/blob/e9b7d758/llap-server/src/main/resources/package.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py
index f410730..caf0fb4 100644
--- a/llap-server/src/main/resources/package.py
+++ b/llap-server/src/main/resources/package.py
@@ -48,7 +48,7 @@ def zipdir(path, zip, prefix="."):
 			zip.write(src, dst)
 	
 def main(args):
-	opts, args = getopt(args,"",["instances=","output=", "input=","args=","name=","loglevel=","chaosmonkey=","size=","xmx=",
"cache=", "executors="])
+	opts, args = getopt(args,"",["instances=","output=", "input=","args=","name=","loglevel=","chaosmonkey=","size=","xmx=",
"cache=", "executors=","hiveconf="])
 	version = os.getenv("HIVE_VERSION")
 	if not version:
 		version = strftime("%d%b%Y", gmtime()) 


Mime
View raw message