hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject hive git commit: HIVE-12967: Change LlapServiceDriver to read a properties file instead of llap-daemon-site (Siddarth Seth, via Gopal V)
Date Wed, 17 Feb 2016 01:13:20 GMT
Repository: hive
Updated Branches:
  refs/heads/master 1b99efec6 -> f8f50ab17


HIVE-12967: Change LlapServiceDriver to read a properties file instead of llap-daemon-site (Siddarth Seth, via Gopal V)

Signed-off-by: Gopal V <gopalv@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8f50ab1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8f50ab1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8f50ab1

Branch: refs/heads/master
Commit: f8f50ab17da3155489e96cdf2e9b849aa88b8700
Parents: 1b99efe
Author: Gopal V <gopalv@apache.org>
Authored: Tue Feb 16 16:10:34 2016 -0800
Committer: Gopal V <gopalv@apache.org>
Committed: Tue Feb 16 16:26:04 2016 -0800

----------------------------------------------------------------------
 bin/ext/llap.sh                                 |   1 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  78 +++++++++-
 .../llap/configuration/LlapConfiguration.java   |  41 -----
 .../registry/impl/LlapFixedRegistryImpl.java    |   6 +-
 .../registry/impl/LlapYarnRegistryImpl.java     |   5 +-
 .../hive/llap/cli/LlapOptionsProcessor.java     |  96 ++++++++----
 .../hadoop/hive/llap/cli/LlapServiceDriver.java | 155 ++++++++++++-------
 .../configuration/LlapDaemonConfiguration.java  |  39 +++++
 .../hive/llap/daemon/impl/AMReporter.java       |   1 -
 .../hive/llap/daemon/impl/LlapDaemon.java       |   4 +-
 .../hive/llap/daemon/impl/QueryTracker.java     |   2 -
 .../llap/shufflehandler/ShuffleHandler.java     |  21 ++-
 .../main/resources/llap-cli-log4j2.properties   |  77 +++++++++
 .../hive/llap/daemon/MiniLlapCluster.java       |   1 -
 .../impl/TestLlapDaemonProtocolServerImpl.java  |   4 +-
 15 files changed, 376 insertions(+), 155 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/bin/ext/llap.sh
----------------------------------------------------------------------
diff --git a/bin/ext/llap.sh b/bin/ext/llap.sh
index fa87587..838bb3a 100644
--- a/bin/ext/llap.sh
+++ b/bin/ext/llap.sh
@@ -30,6 +30,7 @@ llap () {
 
   set -e;
 
+  export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=llap-cli-log4j2.properties "
   # hadoop 20 or newer - skip the aux_jars option. picked up from hiveconf
   $HADOOP $CLASS $HIVE_OPTS -directory $TMPDIR "$@"
   

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 3ab1dba..48e8491 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.conf;
 import com.google.common.base.Joiner;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.hive.conf.Validator.PatternSet;
 import org.apache.hadoop.hive.conf.Validator.RangeValidator;
@@ -46,9 +47,11 @@ import java.io.InputStream;
 import java.io.PrintStream;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -109,8 +112,17 @@ public class HiveConf extends Configuration {
     for (ConfVars confVar : ConfVars.values()) {
       vars.put(confVar.varname, confVar);
     }
+
+    Set<String> llapDaemonConfVarsSetLocal = new LinkedHashSet<>();
+    populateLlapDaemonVarsSet(llapDaemonConfVarsSetLocal);
+    llapDaemonVarsSet = Collections.unmodifiableSet(llapDaemonConfVarsSetLocal);
   }
 
+  @InterfaceAudience.Private
+  public static final String PREFIX_LLAP = "llap.";
+  @InterfaceAudience.Private
+  public static final String PREFIX_HIVE_LLAP = "hive.llap.";
+
   /**
    * Metastore related options that the db is initialized against. When a conf
    * var in this is list is changed, the metastore instance for the CLI will
@@ -237,6 +249,70 @@ public class HiveConf extends Configuration {
   };
 
   /**
+   * Variables used by LLAP daemons.
+   * TODO: Eventually auto-populate this based on prefixes. The conf variables
+   * will need to be renamed for this.
+   */
+  private static final Set<String> llapDaemonVarsSet;
+
+  private static void populateLlapDaemonVarsSet(Set<String> llapDaemonVarsSetLocal) {
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_ENABLED.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_MEMORY_MODE.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_MAX_ALLOC.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_ARENA_COUNT.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_DIRECT.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_USE_LRFU.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_LRFU_LAMBDA.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_CACHE_ALLOW_SYNTHETIC_FILEID.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_USE_FILEID_PATH.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ORC_ENABLE_TIME_COUNTERS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_THREADPOOL_SIZE.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_KERBEROS_PRINCIPAL.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_KERBEROS_KEYTAB_FILE.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_KERBEROS_PRINCIPAL.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_KERBEROS_KEYTAB_FILE.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_ZK_CONNECTION_STRING.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_SECURITY_ACL.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_MANAGEMENT_ACL.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DELEGATION_TOKEN_LIFETIME.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_MANAGEMENT_RPC_PORT.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_WEB_AUTO_AUTH.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WORK_DIRS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_TIMEOUT_MS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_SLEEP_BETWEEN_RETRIES_MS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_RPC_PORT.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_NUM_FILE_CLEANER_THREADS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SERVICE_REFRESH_INTERVAL.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WAIT_QUEUE_COMPARATOR_CLASS_NAME.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WEB_PORT.varname);
+    llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WEB_SSL.varname);
+  }
+
+  /**
+   * Get a set containing configuration parameter names used by LLAP Server isntances
+   * @return an unmodifiable set containing llap ConfVars
+   */
+  public static final Set<String> getLlapDaemonConfVars() {
+    return llapDaemonVarsSet;
+  }
+
+
+  /**
    * ConfVars.
    *
    * These are the default configuration properties for Hive. Each HiveConf
@@ -2492,7 +2568,7 @@ public class HiveConf extends Configuration {
     LLAP_DAEMON_YARN_SHUFFLE_PORT("hive.llap.daemon.yarn.shuffle.port", 15551,
       "YARN shuffle port for LLAP-daemon-hosted shuffle.", "llap.daemon.yarn.shuffle.port"),
     LLAP_DAEMON_YARN_CONTAINER_MB("hive.llap.daemon.yarn.container.mb", -1,
-      "TODO doc. Unused?", "llap.daemon.yarn.container.mb"),
+      "llap server yarn container size in MB. Used in LlapServiceDriver and package.py", "llap.daemon.yarn.container.mb"),
     LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED("hive.llap.daemon.shuffle.dir.watcher.enabled", false,
       "TODO doc", "llap.daemon.shuffle.dir-watcher.enabled"),
     LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS(

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
deleted file mode 100644
index abdbc09..0000000
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.configuration;
-
-import java.net.URL;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class LlapConfiguration extends Configuration {
-  public static final String LLAP_PREFIX = "llap.";
-  public static final String LLAP_DAEMON_PREFIX = "llap.daemon.";
-
-  public LlapConfiguration(Configuration conf) {
-    super(conf);
-    addResource(LLAP_DAEMON_SITE);
-  }
-
-  public LlapConfiguration() {
-    super(false);
-    addResource(LLAP_DAEMON_SITE);
-  }
-
-  public LlapConfiguration(Configuration conf, URL llapDaemonConfLocation) {
-    super(conf);
-    addResource(llapDaemonConfLocation);
-  }
-
-  private static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml";
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
index 92044bb..c3c16c4 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
 import org.apache.hadoop.hive.llap.registry.ServiceInstance;
 import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
 import org.apache.hadoop.hive.llap.registry.ServiceRegistry;
@@ -67,9 +66,8 @@ public class LlapFixedRegistryImpl implements ServiceRegistry {
     this.mngPort = HiveConf.getIntVar(conf, ConfVars.LLAP_MANAGEMENT_RPC_PORT);
 
     for (Map.Entry<String, String> kv : conf) {
-      if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX)
-          || kv.getKey().startsWith("hive.llap.")
-          || kv.getKey().startsWith(LlapConfiguration.LLAP_PREFIX)) {
+      if (kv.getKey().startsWith(HiveConf.PREFIX_LLAP)
+          || kv.getKey().startsWith(HiveConf.PREFIX_HIVE_LLAP)) {
         // TODO: read this somewhere useful, like the task scheduler
         srv.put(kv.getKey(), kv.getValue());
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
index efe31cc..c83dd6e 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
@@ -39,7 +39,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
 import org.apache.hadoop.hive.llap.registry.ServiceInstance;
 import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
 import org.apache.hadoop.hive.llap.registry.ServiceRegistry;
@@ -161,8 +160,8 @@ public class LlapYarnRegistryImpl implements ServiceRegistry {
     srv.addExternalEndpoint(getServicesEndpoint());
 
     for (Map.Entry<String, String> kv : this.conf) {
-      if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX)
-          || kv.getKey().startsWith("hive.llap.")) {
+      if (kv.getKey().startsWith(HiveConf.PREFIX_LLAP)
+          || kv.getKey().startsWith(HiveConf.PREFIX_HIVE_LLAP)) {
         // TODO: read this somewhere useful, like the task scheduler
         srv.set(kv.getKey(), kv.getValue());
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index 6d25384..1732df8 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -18,8 +18,9 @@
 
 package org.apache.hadoop.hive.llap.cli;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.Properties;
 
 import javax.annotation.Nonnull;
@@ -36,6 +37,21 @@ import org.apache.hadoop.util.StringUtils;
 
 public class LlapOptionsProcessor {
 
+  public static final String OPTION_INSTANCES = "instances"; //forward as arg
+  public static final String OPTION_NAME = "name"; // forward as arg
+  public static final String OPTION_DIRECTORY = "directory"; // work-dir
+  public static final String OPTION_ARGS = "args"; // forward as arg
+  public static final String OPTION_LOGLEVEL = "loglevel"; // forward as arg
+  public static final String OPTION_CHAOS_MONKEY = "chaosmonkey"; // forward as arg
+  public static final String OPTION_EXECUTORS = "executors"; // llap-daemon-site
+  public static final String OPTION_CACHE = "cache"; // llap-daemon-site
+  public static final String OPTION_SIZE = "size"; // forward via config.json
+  public static final String OPTION_XMX = "xmx"; // forward as arg
+  public static final String OPTION_AUXJARS = "auxjars"; // used to localize jars
+  public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars
+  public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json
+  public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter
+
   public class LlapOptions {
     private final int instances;
     private final String directory;
@@ -47,10 +63,12 @@ public class LlapOptionsProcessor {
     private final String jars;
     private final boolean isHbase;
     private final Properties conf;
+    private final String javaPath;
 
     public LlapOptions(String name, int instances, String directory, int executors, long cache,
-        long size, long xmx, String jars, boolean isHbase, @Nonnull Properties hiveconf)
-            throws ParseException {
+                       long size, long xmx, String jars, boolean isHbase,
+                       @Nonnull Properties hiveconf, String javaPath)
+        throws ParseException {
       if (instances <= 0) {
         throw new ParseException("Invalid configuration: " + instances
             + " (should be greater than 0)");
@@ -65,6 +83,7 @@ public class LlapOptionsProcessor {
       this.jars = jars;
       this.isHbase = isHbase;
       this.conf = hiveconf;
+      this.javaPath = javaPath;
     }
 
     public String getName() {
@@ -106,57 +125,67 @@ public class LlapOptionsProcessor {
     public Properties getConfig() {
       return conf;
     }
+
+    public String getJavaPath() {
+      return javaPath;
+    }
   }
 
   protected static final Logger l4j = LoggerFactory.getLogger(LlapOptionsProcessor.class.getName());
   private final Options options = new Options();
-  Map<String, String> hiveVariables = new HashMap<String, String>();
   private org.apache.commons.cli.CommandLine commandLine;
 
   @SuppressWarnings("static-access")
   public LlapOptionsProcessor() {
 
     // set the number of instances on which llap should run
-    options.addOption(OptionBuilder.hasArg().withArgName("instances").withLongOpt("instances")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_INSTANCES).withLongOpt(OPTION_INSTANCES)
         .withDescription("Specify the number of instances to run this on").create('i'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("name").withLongOpt("name")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_NAME).withLongOpt(OPTION_NAME)
         .withDescription("Cluster name for YARN registry").create('n'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("directory").withLongOpt("directory")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_DIRECTORY).withLongOpt(OPTION_DIRECTORY)
         .withDescription("Temp directory for jars etc.").create('d'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("args").withLongOpt("args")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_ARGS).withLongOpt(OPTION_ARGS)
         .withDescription("java arguments to the llap instance").create('a'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("loglevel").withLongOpt("loglevel")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_LOGLEVEL).withLongOpt(OPTION_LOGLEVEL)
         .withDescription("log levels for the llap instance").create('l'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("chaosmonkey").withLongOpt("chaosmonkey")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CHAOS_MONKEY).withLongOpt(OPTION_CHAOS_MONKEY)
         .withDescription("chaosmonkey interval").create('m'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("executors").withLongOpt("executors")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_EXECUTORS).withLongOpt(OPTION_EXECUTORS)
         .withDescription("executor per instance").create('e'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("cache").withLongOpt("cache")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CACHE).withLongOpt(OPTION_CACHE)
         .withDescription("cache size per instance").create('c'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("size").withLongOpt("size")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SIZE).withLongOpt(OPTION_SIZE)
         .withDescription("container size per instance").create('s'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("xmx").withLongOpt("xmx")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_XMX).withLongOpt(OPTION_XMX)
         .withDescription("working memory size").create('w'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("auxjars").withLongOpt("auxjars")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXJARS).withLongOpt(OPTION_AUXJARS)
         .withDescription("additional jars to package (by default, JSON SerDe jar is packaged"
             + " if available)").create('j'));
 
-    options.addOption(OptionBuilder.hasArg().withArgName("auxhbase").withLongOpt("auxhbase")
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXHBASE).withLongOpt(OPTION_AUXHBASE)
         .withDescription("whether to package the HBase jars (true by default)").create('h'));
 
+    options.addOption(OptionBuilder.hasArg().withArgName(OPTION_JAVA_HOME).withLongOpt(OPTION_JAVA_HOME)
+        .withDescription(
+            "Path to the JRE/JDK. This should be installed at the same location on all cluster nodes ($JAVA_HOME, java.home by default)")
+        .create());
+
     // -hiveconf x=y
     options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value")
-        .withLongOpt("hiveconf").withDescription("Use value for given property").create());
+        .withLongOpt(OPTION_HIVECONF)
+        .withDescription("Use value for given property. Overridden by explicit parameters")
+        .create());
 
     // [-H|--help]
     options.addOption(new Option("H", "help", false, "Print help information"));
@@ -166,7 +195,7 @@ public class LlapOptionsProcessor {
     return StringUtils.TraditionalBinaryPrefix.string2long(value);
   }
 
-  public LlapOptions processOptions(String argv[]) throws ParseException {
+  public LlapOptions processOptions(String argv[]) throws ParseException, IOException {
     commandLine = new GnuParser().parse(options, argv);
     if (commandLine.hasOption('H') || false == commandLine.hasOption("instances")) {
       // needs at least --instances
@@ -174,31 +203,34 @@ public class LlapOptionsProcessor {
       return null;
     }
 
-    int instances = Integer.parseInt(commandLine.getOptionValue("instances"));
-    String directory = commandLine.getOptionValue("directory");
-    String jars = commandLine.getOptionValue("auxjars");
+    int instances = Integer.parseInt(commandLine.getOptionValue(OPTION_INSTANCES));
+    String directory = commandLine.getOptionValue(OPTION_DIRECTORY);
+    String jars = commandLine.getOptionValue(OPTION_AUXJARS);
 
-    String name = commandLine.getOptionValue("name", null);
+    String name = commandLine.getOptionValue(OPTION_NAME, null);
 
-    final int executors = Integer.parseInt(commandLine.getOptionValue("executors", "-1"));
-    final long cache = parseSuffixed(commandLine.getOptionValue("cache", "-1"));
-    final long size = parseSuffixed(commandLine.getOptionValue("size", "-1"));
-    final long xmx = parseSuffixed(commandLine.getOptionValue("xmx", "-1"));
-    final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue("auxhbase", "true"));
+    final int executors = Integer.parseInt(commandLine.getOptionValue(OPTION_EXECUTORS, "-1"));
+    final long cache = parseSuffixed(commandLine.getOptionValue(OPTION_CACHE, "-1"));
+    final long size = parseSuffixed(commandLine.getOptionValue(OPTION_SIZE, "-1"));
+    final long xmx = parseSuffixed(commandLine.getOptionValue(OPTION_XMX, "-1"));
+    final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue(OPTION_AUXHBASE, "true"));
 
     final Properties hiveconf;
 
-    if (commandLine.hasOption("hiveconf")) {
-      hiveconf = commandLine.getOptionProperties("hiveconf");
+    if (commandLine.hasOption(OPTION_HIVECONF)) {
+      hiveconf = commandLine.getOptionProperties(OPTION_HIVECONF);
     } else {
       hiveconf = new Properties();
     }
 
+    String javaHome = null;
+    if (commandLine.hasOption(OPTION_JAVA_HOME)) {
+      javaHome = commandLine.getOptionValue(OPTION_JAVA_HOME);
+    }
     // loglevel, chaosmonkey & args are parsed by the python processor
 
     return new LlapOptions(
-        name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf);
-
+        name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf, javaHome);
   }
 
   private void printUsage() {

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index d01c8ce..32c791f 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -21,12 +21,14 @@ package org.apache.hadoop.hive.llap.cli;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.net.URL;
-import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Collection;
+import java.util.Properties;
 
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
 import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -52,14 +54,18 @@ import com.google.common.base.Preconditions;
 public class LlapServiceDriver {
 
   protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName());
+
   private static final String[] DEFAULT_AUX_CLASSES = new String[] {
   "org.apache.hive.hcatalog.data.JsonSerDe" };
   private static final String HBASE_SERDE_CLASS = "org.apache.hadoop.hive.hbase.HBaseSerDe";
-  private static final String[] NEEDED_CONFIGS = {
-    "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" };
+  private static final String[] NEEDED_CONFIGS = LlapDaemonConfiguration.DAEMON_CONFIGS;
   private static final String[] OPTIONAL_CONFIGS = { "ssl-server.xml" };
 
 
+  /**
+   * This is a working configuration for the instance to merge various variables.
+   * It is not written out for llap server usage
+   */
   private final Configuration conf;
 
   public LlapServiceDriver() {
@@ -82,40 +88,56 @@ public class LlapServiceDriver {
     System.exit(ret);
   }
 
-  /**
-   * Intersect llap-daemon-site.xml configuration properties against an existing Configuration
-   * object, while resolving any ${} parameters that might be present.
-   * 
-   * @param raw
-   * @return configuration object which is a slice of configured
-   */
-  public static Configuration resolve(Configuration configured, String first, String... resources) {
-    Configuration defaults = new Configuration(false);
 
-    defaults.addResource(first);
+  private static Configuration resolve(Configuration configured, Properties direct,
+                                       Properties hiveconf) {
+    Configuration conf = new Configuration(false);
 
-    for (String resource : resources) {
-      defaults.addResource(resource);
-    }
+    populateConf(configured, conf, hiveconf, "CLI hiveconf");
+    populateConf(configured, conf, direct, "CLI direct");
+
+    return conf;
+  }
 
-    Configuration slice = new Configuration(false);
-    // for everything in defaults, slice out those from the configured
-    for (Map.Entry<String, String> kv : defaults) {
-      slice.set(kv.getKey(), configured.get(kv.getKey()));
+  private static void populateConf(Configuration configured, Configuration target,
+                                   Properties properties, String source) {
+    for (Entry<Object, Object> entry : properties.entrySet()) {
+      String key = (String) entry.getKey();
+      String val = configured.get(key);
+      if (val != null) {
+        target.set(key, val, source);
+      }
     }
+  }
 
-    return slice;
+  private static void populateConfWithLlapProperties(Configuration conf, Properties properties) {
+    for(Entry<Object, Object> props : properties.entrySet()) {
+      String key = (String) props.getKey();
+      if (HiveConf.getLlapDaemonConfVars().contains(key)) {
+        conf.set(key, (String) props.getValue());
+      } else {
+        if (key.startsWith(HiveConf.PREFIX_LLAP) || key.startsWith(HiveConf.PREFIX_HIVE_LLAP)) {
+          LOG.warn("Adding key [{}] even though it is not in the set of known llap-server keys");
+          conf.set(key, (String) props.getValue());
+        } else {
+          LOG.warn("Ignoring unknown llap server parameter: [{}]", key);
+        }
+      }
+    }
   }
 
   private void run(String[] args) throws Exception {
     LlapOptionsProcessor optionsProcessor = new LlapOptionsProcessor();
     LlapOptions options = optionsProcessor.processOptions(args);
 
+    Properties propsDirectOptions = new Properties();
+
     if (options == null) {
       // help
       return;
     }
 
+    // Working directory.
     Path tmpDir = new Path(options.getDirectory());
 
     if (conf == null) {
@@ -135,14 +157,19 @@ public class LlapServiceDriver {
     for (String f : OPTIONAL_CONFIGS) {
       conf.addResource(f);
     }
+
     conf.reloadConfiguration();
 
+    populateConfWithLlapProperties(conf, options.getConfig());
+
+
     if (options.getName() != null) {
       // update service registry configs - caveat: this has nothing to do with the actual settings
       // as read by the AM
       // if needed, use --hiveconf llap.daemon.service.hosts=@llap0 to dynamically switch between
       // instances
       conf.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + options.getName());
+      propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + options.getName());
     }
 
     if (options.getSize() != -1) {
@@ -160,35 +187,40 @@ public class LlapServiceDriver {
       }
     }
 
+    // This parameter is read in package.py - and nowhere else. Does not need to be part of HiveConf - that's just confusing.
     final long minAlloc = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1);
+    long containerSize = -1;
     if (options.getSize() != -1) {
-      final long containerSize = options.getSize() / (1024 * 1024);
+      containerSize = options.getSize() / (1024 * 1024);
       Preconditions.checkArgument(containerSize >= minAlloc,
           "Container size should be greater than minimum allocation(%s)", minAlloc + "m");
       conf.setLong(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, containerSize);
+      propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, String.valueOf(containerSize));
     }
 
     if (options.getExecutors() != -1) {
       conf.setLong(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, options.getExecutors());
+      propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, String.valueOf(options.getExecutors()));
       // TODO: vcpu settings - possibly when DRFA works right
     }
 
     if (options.getCache() != -1) {
       conf.set(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
           Long.toString(options.getCache()));
+      propsDirectOptions.setProperty(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
+          Long.toString(options.getCache()));
     }
 
     if (options.getXmx() != -1) {
       // Needs more explanation here
       // Xmx is not the max heap value in JDK8
-      // You need to subtract 50% of the survivor fraction from this, to get actual usable memory before it goes into GC 
-      conf.setLong(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, (long)(options.getXmx())
-          / (1024 * 1024));
+      // You need to subtract 50% of the survivor fraction from this, to get actual usable memory before it goes into GC
+      long xmx = (long) (options.getXmx() / (1024 * 1024));
+      conf.setLong(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, xmx);
+      propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, String.valueOf(xmx));
     }
 
-    for (Entry<Object, Object> props : options.getConfig().entrySet()) {
-      conf.set((String) props.getKey(), (String) props.getValue());
-    }
+
 
     URL logger = conf.getResource("llap-daemon-log4j2.properties");
 
@@ -205,9 +237,10 @@ public class LlapServiceDriver {
       LOG.warn("Unable to find llap scripts:" + scripts);
     }
 
+
     Path libDir = new Path(tmpDir, "lib");
 
-    String tezLibs = conf.get("tez.lib.uris");
+    String tezLibs = conf.get(TezConfiguration.TEZ_LIB_URIS);
     if (tezLibs == null) {
       LOG.warn("Missing tez.lib.uris in tez-site.xml");
     }
@@ -264,31 +297,43 @@ public class LlapServiceDriver {
       }
     }
 
+    String java_home;
+    if (options.getJavaPath() == null || options.getJavaPath().isEmpty()) {
+      java_home = System.getenv("JAVA_HOME");
+      String jre_home = System.getProperty("java.home");
+      if (java_home == null) {
+        java_home = jre_home;
+      } else if (!java_home.equals(jre_home)) {
+        LOG.warn("Java versions might not match : JAVA_HOME=[{}],process jre=[{}]",
+            java_home, jre_home);
+      }
+    } else {
+      java_home = options.getJavaPath();
+    }
+    if (java_home == null || java_home.isEmpty()) {
+      throw new RuntimeException(
+          "Could not determine JAVA_HOME from command line parameters, environment or system properties");
+    }
+    LOG.info("Using [{}] for JAVA_HOME", java_home);
+
     Path confPath = new Path(tmpDir, "conf");
     lfs.mkdirs(confPath);
 
+    // Copy over the mandatory configs for the package.
     for (String f : NEEDED_CONFIGS) {
-      copyConfig(options, lfs, confPath, f);
+      copyConfig(lfs, confPath, f);
     }
     for (String f : OPTIONAL_CONFIGS) {
       try {
-        copyConfig(options, lfs, confPath, f);
+        copyConfig(lfs, confPath, f);
       } catch (Throwable t) {
         LOG.info("Error getting an optional config " + f + "; ignoring: " + t.getMessage());
       }
     }
+    createLlapDaemonConfig(lfs, confPath, conf, propsDirectOptions, options.getConfig());
 
     lfs.copyFromLocalFile(new Path(logger.toString()), confPath);
 
-    String java_home = System.getenv("JAVA_HOME");
-    String jre_home = System.getProperty("java.home");
-    if (java_home == null) {
-      java_home = jre_home;
-    } else if (!java_home.equals(jre_home)) {
-      LOG.warn("Java versions might not match : JAVA_HOME=%s,process jre=%s", 
-          java_home, jre_home);
-    }
-
     // extract configs for processing by the python fragments in Slider
     JSONObject configs = new JSONObject();
 
@@ -296,6 +341,7 @@ public class LlapServiceDriver {
 
     configs.put(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, HiveConf.getIntVar(conf,
         ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB));
+    configs.put(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, containerSize);
 
     configs.put(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
         HiveConf.getSizeVar(conf, HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE));
@@ -362,23 +408,22 @@ public class LlapServiceDriver {
     }
   }
 
-  private void copyConfig(
-      LlapOptions options, FileSystem lfs, Path confPath, String f) throws IOException {
-    if (f.equals("llap-daemon-site.xml")) {
-      FSDataOutputStream confStream = lfs.create(new Path(confPath, f));
+  private void createLlapDaemonConfig(FileSystem lfs, Path confPath, Configuration configured,
+                                      Properties direct, Properties hiveconf) throws IOException {
+    FSDataOutputStream confStream =
+        lfs.create(new Path(confPath, LlapDaemonConfiguration.LLAP_DAEMON_SITE));
 
-      Configuration copy = resolve(conf, "llap-daemon-site.xml");
+    Configuration llapDaemonConf = resolve(configured, direct, hiveconf);
 
-      for (Entry<Object, Object> props : options.getConfig().entrySet()) {
-        // overrides
-        copy.set((String) props.getKey(), (String) props.getValue());
-      }
+    llapDaemonConf.writeXml(confStream);
+    confStream.close();
+  }
 
-      copy.writeXml(confStream);
-      confStream.close();
-    } else {
-      // they will be file:// URLs
-      lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
-    }
+  private void copyConfig(FileSystem lfs, Path confPath, String f) throws IOException {
+    HiveConf.getBoolVar(new Configuration(false), ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS);
+    // they will be file:// URLs
+    lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
   }
+
+
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java b/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
new file mode 100644
index 0000000..72340f5
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.configuration;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
+/**
+ * Configuration for LLAP daemon processes only. This should not be used by any clients.
+ */
+public class LlapDaemonConfiguration extends Configuration {
+
+  @InterfaceAudience.Private
+  public static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml";
+
+  @InterfaceAudience.Private
+  public static final String[] DAEMON_CONFIGS = { /* in specific order */"core-site.xml",
+      "hdfs-site.xml", "yarn-site.xml", "tez-site.xml", "hive-site.xml" };
+  
+  public LlapDaemonConfiguration() {
+    super(false);
+    for (String conf : DAEMON_CONFIGS) {
+      addResource(conf);
+    }
+    addResource(LLAP_DAEMON_SITE);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
index d1ec715..04c28cb 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
 import org.apache.hadoop.hive.llap.daemon.QueryFailedHandler;
 import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
 import org.apache.hadoop.io.Text;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index c0d4690..e066742 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -29,7 +29,7 @@ import javax.management.ObjectName;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
 import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
 import org.apache.hadoop.hive.llap.daemon.QueryFailedHandler;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
@@ -302,7 +302,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
     try {
       // Cache settings will need to be setup in llap-daemon-site.xml - since the daemons don't read hive-site.xml
       // Ideally, these properties should be part of LlapDameonConf rather than HiveConf
-      LlapConfiguration daemonConf = new LlapConfiguration();
+      LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
       int numExecutors = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
 
       String localDirList = HiveConf.getVar(daemonConf, ConfVars.LLAP_DAEMON_WORK_DIRS);

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
index 80264a0..14657e6 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
 import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
@@ -42,7 +41,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantLock;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
index 2c51169..39a1468 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
@@ -119,10 +119,10 @@ public class ShuffleHandler implements AttemptRegistrationListener {
 
   public static final String SHUFFLE_HANDLER_LOCAL_DIRS = "llap.shuffle.handler.local-dirs";
 
-  public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
+  public static final String SHUFFLE_MANAGE_OS_CACHE = "lla[.shuffle.manage.os.cache";
   public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;
 
-  public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes";
+  public static final String SHUFFLE_READAHEAD_BYTES = "llap.shuffle.readahead.bytes";
   public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024;
 
   public static final String SHUFFLE_DIR_WATCHER_ENABLED = "llap.shuffle.dir-watcher.enabled";
@@ -163,40 +163,39 @@ public class ShuffleHandler implements AttemptRegistrationListener {
   public static final String SHUFFLE_PORT_CONFIG_KEY = "llap.shuffle.port";
   public static final int DEFAULT_SHUFFLE_PORT = 15551;
 
-  // TODO Change configs to remove mapreduce references.
   public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
-      "mapreduce.shuffle.connection-keep-alive.enable";
+      "llap.shuffle.connection-keep-alive.enable";
   public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
 
   public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT =
-      "mapreduce.shuffle.connection-keep-alive.timeout";
+      "llap.shuffle.connection-keep-alive.timeout";
   public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT = 5; //seconds
 
   public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
-      "mapreduce.shuffle.mapoutput-info.meta.cache.size";
+      "llap.shuffle.mapoutput-info.meta.cache.size";
   public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
       10000;
 
   public static final String CONNECTION_CLOSE = "close";
 
   public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
-    "mapreduce.shuffle.ssl.file.buffer.size";
+    "llap.shuffle.ssl.file.buffer.size";
 
   public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
 
-  public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections";
+  public static final String MAX_SHUFFLE_CONNECTIONS = "llap.shuffle.max.connections";
   public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit
   
-  public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads";
+  public static final String MAX_SHUFFLE_THREADS = "llap.shuffle.max.threads";
   // 0 implies Netty default of 2 * number of available processors
   public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0;
   
   public static final String SHUFFLE_BUFFER_SIZE = 
-      "mapreduce.shuffle.transfer.buffer.size";
+      "llap.shuffle.transfer.buffer.size";
   public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;
   
   public static final String  SHUFFLE_TRANSFERTO_ALLOWED = 
-      "mapreduce.shuffle.transferTo.allowed";
+      "llap.shuffle.transferTo.allowed";
   public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
   public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = 
       false;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/main/resources/llap-cli-log4j2.properties
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/llap-cli-log4j2.properties b/llap-server/src/main/resources/llap-cli-log4j2.properties
new file mode 100644
index 0000000..7542193
--- /dev/null
+++ b/llap-server/src/main/resources/llap-cli-log4j2.properties
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = WARN
+name = LlapCliLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+property.hive.log.level = INFO
+property.hive.root.logger = console
+property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
+property.hive.log.file = hive.log
+
+# list of all appenders
+appenders = console, DRFA
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+# appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
+appender.console.layout.pattern = %p %c{2}: %m%n
+
+# daily rolling file appender
+appender.DRFA.type = RollingFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}
+# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
+appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = 30
+
+# list of all loggers
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HadoopConf
+
+logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+logger.NIOServerCnxn.level = WARN
+
+logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+logger.ClientCnxnSocketNIO.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.HadoopConf.name = org.apache.hadoop.conf.Configuration
+logger.HadoopConf.level = ERROR
+
+# root logger
+rootLogger.level = ${sys:hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
index deade5f..5fa2bf1 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
 import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.Service;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
index 1cef218..a65bf5c 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
@@ -25,7 +25,7 @@ import com.google.protobuf.ServiceException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
 import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
 import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
@@ -39,7 +39,7 @@ public class TestLlapDaemonProtocolServerImpl {
 
   @Test(timeout = 10000)
   public void test() throws ServiceException, IOException {
-    LlapConfiguration daemonConf = new LlapConfiguration();
+    LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
     int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT);
     int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS);
     ContainerRunner containerRunnerMock = mock(ContainerRunner.class);


Mime
View raw message