ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject [5/5] ambari git commit: AMBARI-18901. LLAP integration enhancements (Swapan Sridhar via smohanty)
Date Mon, 12 Dec 2016 06:02:14 GMT
AMBARI-18901. LLAP integration enhancements (Swapan Sridhar via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ab9acef4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ab9acef4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ab9acef4

Branch: refs/heads/trunk
Commit: ab9acef421af40ddb88fe7589c16d1868fda55f3
Parents: 646cfc6
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Sun Dec 11 22:01:54 2016 -0800
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Sun Dec 11 22:01:54 2016 -0800

----------------------------------------------------------------------
 .../server/configuration/Configuration.java     |    1 -
 .../server/upgrade/UpgradeCatalog250.java       |   74 +-
 .../package/scripts/hive_interactive.py         |   12 +
 .../package/scripts/hive_server_interactive.py  |   48 +-
 .../0.12.0.2.0/package/scripts/params_linux.py  |    9 +
 .../0.4.0.2.1/package/scripts/params_linux.py   |    7 +
 .../stacks/HDP/2.2/services/stack_advisor.py    |   15 +-
 .../stacks/HDP/2.3/services/stack_advisor.py    |    2 +
 .../HIVE/configuration/hive-interactive-env.xml |   80 +-
 .../configuration/hive-interactive-site.xml     |   68 +-
 .../HIVE/configuration/tez-interactive-site.xml |   71 +-
 .../HDP/2.5/services/HIVE/themes/theme.json     |   27 +-
 .../YARN/configuration/capacity-scheduler.xml   |    2 +-
 .../stacks/HDP/2.5/services/stack_advisor.py    |  700 ++--
 .../services/HIVE/configuration/hive-env.xml    |  119 +
 .../HIVE/configuration/hive-interactive-env.xml |   87 +
 .../2.6/services/TEZ/configuration/tez-env.xml  |   56 +
 .../2.6/services/TEZ/configuration/tez-site.xml |   28 +
 .../stacks/HDP/2.6/services/stack_advisor.py    |   25 +-
 .../server/upgrade/UpgradeCatalog250Test.java   |   85 +-
 .../stacks/2.2/common/test_stack_advisor.py     |    7 +-
 .../stacks/2.5/HIVE/test_hive_server_int.py     |    2 +
 .../stacks/2.5/common/test_stack_advisor.py     | 3280 ++----------------
 .../app/mixins/common/configs/configs_saver.js  |    2 +-
 24 files changed, 1458 insertions(+), 3349 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 526e5be..50ec0e2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2531,7 +2531,6 @@ public class Configuration {
           "log4j.monitor.delay", TimeUnit.MINUTES.toMillis(5));
 
   /**
-<<<<<<< a5fdae802210ae1f8d4fed2234f1651cbe61c2b5
    * Indicates whether parallel topology task creation is enabled for blueprint cluster provisioning.
    * Defaults to <code>false</code>.
    * @see #TOPOLOGY_TASK_PARALLEL_CREATION_THREAD_COUNT

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index ff1663e..e148899 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -17,14 +17,8 @@
  */
 package org.apache.ambari.server.upgrade;
 
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.inject.Inject;
+import com.google.inject.Injector;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -38,8 +32,13 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Upgrade catalog for version 2.5.0.
@@ -135,6 +134,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     addNewConfigurationsFromXml();
     updateAMSConfigs();
     updateKafkaConfigs();
+    updateHIVEInteractiveConfigs();
+    updateTEZInteractiveConfigs();
     updateHiveLlapConfigs();
     updateTablesForZeppelinViewRemoval();
     updateAtlasConfigs();
@@ -356,5 +357,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
       }
     }
   }
-}
 
+  /**
+   * Updates Hive Interactive's config in hive-interactive-site.
+   *
+   * @throws AmbariException
+   */
+  protected void updateHIVEInteractiveConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
+          if (hiveInteractiveSite != null) {
+            updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size",
+                "SET_ON_FIRST_INVOCATION"), true, true);
+
+            updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size",
+                "1000000000"), true, true);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Updates Tez for Hive2 Interactive's config in tez-interactive-site.
+   *
+   * @throws AmbariException
+   */
+  protected void updateTEZInteractiveConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site");
+          if (tezInteractiveSite != null) {
+
+            updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb",
"512"), true, true);
+
+            updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb",
+                "100"), true, true);
+          }
+        }
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
index 888b920..af2a05d 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
@@ -57,6 +57,7 @@ Sets up the configs, jdbc connection and tarball copy to HDFS for Hive Server
In
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
 def hive_interactive(name=None):
   import params
+  MB_TO_BYTES = 1048576
 
   # Create Hive User Dir
   params.HdfsResource(params.hive_hdfs_user_dir,
@@ -107,6 +108,17 @@ def hive_interactive(name=None):
       del merged_hive_interactive_site[item]
 
   '''
+  Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now.
We need to
+  convert it to bytes before we write it down to config file.
+  '''
+  if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
+    hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get("hive.llap.io.memory.size")
+    hive_llap_io_mem_size_in_bytes = long(hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
+    merged_hive_interactive_site['hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
+    Logger.info("Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes'
before writing "
+                "it to config file.".format(hive_llap_io_mem_size_in_mb, hive_llap_io_mem_size_in_bytes))
+
+  '''
   Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
   which would have come in config 'hive.exec.post.hooks' during the site merge logic, if
Atlas is installed.
   '''

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index b5edac8..e04c53c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -268,15 +268,29 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
           Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME))
         pass
 
+      # Call for cleaning up the earlier run(s) LLAP package folders.
+      self._cleanup_past_llap_package_dirs()
+
       Logger.info("Starting LLAP")
       LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
 
       unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
 
+      # Figure out the Slider Anti-affinity to be used.
+      # YARN does not support anti-affinity, and therefore Slider implements AA by the means
of exclusion lists, i.e, it
+      # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine).
When the LLAP
+      # container memory size configuration is more than half of YARN node memory, AA is
implicit and should be avoided.
+      slider_placement = 4
+      if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)):
+        slider_placement = 0
+        Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} >
0.5 * "
+                    "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size,
params.yarn_nm_mem))
+
       cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances
{params.num_llap_nodes}"
                    " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m"
                    " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m
--loglevel {params.llap_log_level}"
-                   " {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
+                   " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}"
+                   " {params.llap_extra_slider_opts} --skiphadoopversion --skiphbasecp --output
{LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
       if params.security_enabled:
         llap_keytab_splits = params.hive_llap_keytab_file.split("/")
         Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
@@ -341,6 +355,38 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
         raise
 
     """
+    Checks and deletes previous run 'LLAP package' folders, ignoring three latest packages.
+    Last three are are ignore for debugging/reference purposes.
+    Helps in keeping check on disk space used.
+    """
+    def _cleanup_past_llap_package_dirs(self):
+      try:
+        import params
+        Logger.info("Determining previous run 'LLAP package' folder(s) to be deleted ....")
+        llap_package_folder_name_prefix = "llap-slider" # Package name is like : llap-sliderYYYY-MM-DD-HH:MM:SS
+        num_folders_to_retain = 3  # Hardcoding it as of now, as no considerable use was
found to provide an env param.
+        file_names = [dir_name for dir_name in os.listdir(Script.get_tmp_dir())
+                      if dir_name.startswith(llap_package_folder_name_prefix)]
+
+        file_names.sort()
+        del file_names[-num_folders_to_retain:] # Ignore 'num_folders_to_retain' latest package
folders.
+        Logger.info("Previous run 'LLAP package' folder(s) to be deleted = {0}".format(file_names))
+
+        if file_names:
+          for path in file_names:
+            abs_path = Script.get_tmp_dir()+"/"+path
+            if os.path.isdir(abs_path):
+              shutil.rmtree(abs_path)
+              Logger.info("Deleted previous run 'LLAP package' folder : {0}".format(abs_path))
+        else:
+          Logger.info("No '{0}*' folder deleted.".format(llap_package_folder_name_prefix))
+      except Exception as e:
+        Logger.info("Exception while doing cleanup for past 'LLAP package(s)'.")
+        traceback.print_exc()
+
+
+
+    """
     Does kinit and copies keytab for Hive/LLAP to HDFS.
     """
     def setup_security(self):

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index f1a95ed..33f1056 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -134,6 +134,13 @@ if check_stack_feature(StackFeature.HIVE_SERVER_INTERACTIVE, version_for_stack_f
 hive_interactive_bin = format('{stack_root}/current/{component_directory_interactive}/bin')
 hive_interactive_lib = format('{stack_root}/current/{component_directory_interactive}/lib')
 
+# Heap dump related
+heap_dump_enabled = default('/configurations/hive-env/enable_heap_dump', None)
+heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False.
+if heap_dump_enabled:
+  heap_dump_path = default('/configurations/hive-env/heap_dump_location', "/tmp")
+  heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="+heap_dump_path
+
 # Hive Interactive related paths
 hive_interactive_var_lib = '/var/lib/hive2'
 
@@ -588,6 +595,7 @@ if has_hive_interactive:
   hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content']
   hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive',
False)
   llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch
{% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts
-XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else
%}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{%
endif %}')
+  hive_interactive_heapsize = config['configurations']['hive-interactive-env']['hive_heapsize']
 
   # Service check related
   if hive_transport_mode.lower() == "http":
@@ -599,6 +607,7 @@ if has_hive_interactive:
   tez_interactive_user = config['configurations']['tez-env']['tez_user']
   num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status',
10)
   # Used in LLAP slider package creation
+  yarn_nm_mem = config['configurations']['yarn-site']['yarn.nodemanager.resource.memory-mb']
   num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes']
   llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb']
   llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level']

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
index 6587151..0322406 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
@@ -63,6 +63,13 @@ if stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONIN
   config_path = os.path.join(stack_root, "current/tez-client/conf")
   config_dir = os.path.realpath(config_path)
 
+# Heap dump related
+heap_dump_enabled = default('/configurations/tez-env/enable_heap_dump', None)
+heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False.
+if heap_dump_enabled:
+  heap_dump_path = default('/configurations/tez-env/heap_dump_location', "/tmp")
+  heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="+heap_dump_path
+
 kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 smokeuser = config['configurations']['cluster-env']['smokeuser']

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 100d66e..6ae10b3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -307,6 +307,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putWebhcatSiteProperty = self.putProperty(configurations, "webhcat-site", services)
     putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
     putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env")
+    putHiveServerPropertyAttributes = self.putPropertyAttribute(configurations, "hiveserver2-site")
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
 
     #  Storage
@@ -527,7 +528,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putHiveServerProperty("hive.security.authorization.enabled", "true")
       putHiveServerProperty("hive.security.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory")
       putHiveServerProperty("hive.security.authenticator.manager", "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator")
-      putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role")
+      putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,"
+                                                         "hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled")
       putHiveSiteProperty("hive.security.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory")
       if sqlstdauth_class not in auth_manager_values:
         auth_manager_values.append(sqlstdauth_class)
@@ -543,7 +545,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putHiveServerProperty("hive.security.authorization.enabled", "true")
       putHiveServerProperty("hive.security.authorization.manager", "com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory")
       putHiveServerProperty("hive.security.authenticator.manager", "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator")
-      putHiveServerProperty("hive.conf.restricted.list", "hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager")
+      putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,"
+                                                         "hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled")
+
+    # hive_security_authorization == 'None'
+    if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower()
== "None":
+      putHiveSiteProperty("hive.server2.enable.doAs", "true")
+      putHiveServerProperty("hive.security.authorization.enabled", "false")
+      putHiveServerPropertyAttributes("hive.security.authorization.manager", 'delete', 'true')
+      putHiveServerPropertyAttributes("hive.security.authenticator.manager", 'delete', 'true')
+      putHiveServerPropertyAttributes("hive.conf.restricted.list", 'delete', 'true')
 
     putHiveSiteProperty("hive.server2.use.SSL", "false")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index b6c83bd..d97efe2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -162,6 +162,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and
int(versionSplits[1]) > 7:
           jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
     putTezProperty('tez.am.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps
-XX:+UseNUMA " + jvmGCParams)
+    # Note: Same calculation is done in 2.6/stack_advisor::recommendTezConfigurations() for
'tez.task.launch.cmd-opts',
+    # and along with it, are appended heap dump opts. If something changes here, make sure
to change it in 2.6 stack.
     putTezProperty('tez.task.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps
-XX:+UseNUMA " + jvmGCParams)
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 279c7c4..7b20728 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -53,40 +53,14 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>llap_queue_capacity</name>
-    <value>0</value>
-    <description>Percentage of the cluster dedicated to interactive query.</description>
-    <display-name>% of Cluster Capacity</display-name>
-    <value-attributes>
-      <type>int</type>
-      <minimum>20</minimum>
-      <maximum>100</maximum>
-      <increment-step>1</increment-step>
-    </value-attributes>
-    <depends-on>
-      <property>
-        <type>hive-interactive-env</type>
-        <name>enable_hive_interactive</name>
-      </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.llap.daemon.queue.name</name>
-      </property>
-      <property>
-        <type>capacity-scheduler</type>
-        <name>yarn.scheduler.capacity.root.queues</name>
-      </property>
-    </depends-on>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
     <name>num_llap_nodes</name>
     <value>1</value>
     <description>The number of Hive LLAP daemons to run.</description>
-    <display-name>Number of LLAP Daemons</display-name>
+    <display-name>Number of LLAP Nodes</display-name>
     <value-attributes>
       <type>int</type>
       <minimum>1</minimum>
+      <increment-step>1</increment-step>
     </value-attributes>
     <depends-on>
       <property>
@@ -94,10 +68,6 @@
         <name>enable_hive_interactive</name>
       </property>
       <property>
-        <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
-      </property>
-      <property>
         <type>hive-interactive-site</type>
         <name>hive.llap.daemon.queue.name</name>
       </property>
@@ -105,10 +75,6 @@
         <type>capacity-scheduler</type>
         <name>yarn.scheduler.capacity.root.queues</name>
       </property>
-      <property>
-        <type>hive-interactive-site</type>
-        <name>hive.server2.tez.sessions.per.default.queue</name>
-      </property>
     </depends-on>
     <on-ambari-upgrade add="true"/>
   </property>
@@ -150,7 +116,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -187,7 +153,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -211,7 +177,7 @@
     <display-name>LLAP's reserved headroom for YARN container</display-name>
     <value-attributes>
       <type>int</type>
-      <unit>bytes</unit>
+      <unit>MB</unit>
     </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>
@@ -246,6 +212,40 @@
     <display-name>LLAP app java opts</display-name>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>hive_heapsize</name>
+    <value>512</value>
+    <description>Hive Java heap size</description>
+    <display-name>HiveServer Interactive Heap Size</display-name>
+    <value-attributes>
+      <type>int</type>
+      <minimum>512</minimum>
+      <unit>MB</unit>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+  </property>
 
 
   <!-- hive-env.sh -->
@@ -267,7 +267,7 @@
       if [ "$SERVICE" = "metastore" ]; then
       export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
       else
-      export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
+      export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and
Client
       fi
 
       export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS  -Xmx${HADOOP_HEAPSIZE}m"

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 0207e49..2fb1553 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -67,7 +67,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -411,7 +411,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -447,7 +447,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -478,7 +478,7 @@ limitations under the License.
     <name>hive.llap.daemon.yarn.container.mb</name>
     <display-name>Memory per daemon</display-name>
     <description>Total memory used by individual LLAP daemons. This includes memory
for the cache as well as for the query execution.</description>
-    <value>341</value>
+    <value>0</value>
     <value-attributes>
       <type>int</type>
       <unit>MB</unit>
@@ -491,7 +491,7 @@ limitations under the License.
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>
@@ -646,4 +646,62 @@ limitations under the License.
     </value-attributes>
     <on-ambari-upgrade add="false"/>
   </property>
+  <property>
+    <name>hive.tez.container.size</name>
+    <value>SET_ON_FIRST_INVOCATION</value>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>hive.auto.convert.join.noconditionaltask.size</name>
+    <value>1000000000</value>
+    <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
+      is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
+      converted to a mapjoin(there is no conditional task). The default is 10MB.
+    </description>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
index 9e588e9..b331736 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
@@ -83,11 +83,8 @@
   </property>
   <property>
     <name>tez.am.resource.memory.mb</name>
-    <value>1536</value>
+    <value>0</value>
     <description>The amount of memory to be used by the AppMaster</description>
-    <value-attributes>
-      <type>int</type>
-    </value-attributes>
     <depends-on>
       <property>
         <type>hive-interactive-env</type>
@@ -95,7 +92,71 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.runtime.io.sort.mb</name>
+    <value>512</value>
+    <description>The size of the sort buffer when output needs to be sorted</description>
+    <depends-on>
+      <property>
+        <type>tez-site</type>
+        <name>tez.task.resource.memory.mb</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.queue.name</name>
+      </property>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.server2.tez.sessions.per.default.queue</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>tez.runtime.unordered.output.buffer.size-mb</name>
+    <value>100</value>
+    <description>The size of the buffer when output does not require to be sorted</description>
+    <depends-on>
+      <property>
+        <type>tez-site</type>
+        <name>tez.task.resource.memory.mb</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
       </property>
       <property>
         <type>hive-interactive-site</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 452537d..1d50b6a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -63,11 +63,7 @@
           ]
         },
         {
-          "config": "hive-interactive-env/llap_queue_capacity",
-          "subsection-name": "interactive-query-row1-col1"
-        },
-        {
-          "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
+          "config": "hive-interactive-env/num_llap_nodes",
           "subsection-name": "interactive-query-row1-col1",
           "depends-on": [
             {
@@ -89,12 +85,8 @@
           ]
         },
         {
-          "config": "hive-interactive-env/copy_num_llap_nodes",
+          "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
           "subsection-name": "interactive-query-row1-col1",
-          "property_value_attributes": {
-            "ui_only_property": true,
-            "copy": "hive-interactive-env/num_llap_nodes"
-          },
           "depends-on": [
             {
               "configs":[
@@ -226,23 +218,12 @@
         }
       },
       {
-        "config": "hive-interactive-env/copy_num_llap_nodes",
-        "widget": {
-          "type": "label",
-          "units": [
-            {
-              "unit-name": "int"
-            }
-          ]
-        }
-      },
-      {
-        "config": "hive-interactive-env/llap_queue_capacity",
+        "config": "hive-interactive-env/num_llap_nodes",
         "widget": {
           "type": "slider",
           "units": [
             {
-              "unit-name": "percent"
+              "unit-name": "int"
             }
           ]
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
index 9ff8484..bc0ecc4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
@@ -26,7 +26,7 @@
       </property>
       <property>
         <type>hive-interactive-env</type>
-        <name>llap_queue_capacity</name>
+        <name>num_llap_nodes</name>
       </property>
     </depends-on>
     <on-ambari-upgrade add="false"/>


Mime
View raw message