ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [49/50] ambari git commit: AMBARI-20225. Take care of hive-interactive-site's 'hive.tez.container.size' config during HDP upgrade from 2.5 to 2.6 and update default value for tez-interactive-site's config 'tez.am.resource.memory.mb'.
Date Wed, 01 Mar 2017 20:39:06 GMT
AMBARI-20225. Take care of hive-interactive-site's 'hive.tez.container.size' config during
HDP upgrade from 2.5 to 2.6 and update default value for tez-interactive-site's config 'tez.am.resource.memory.mb'.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/baefbcca
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/baefbcca
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/baefbcca

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: baefbcca0b60caac801f3addf0d015df23a5e96d
Parents: 2072511
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Mon Feb 27 22:56:54 2017 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Wed Mar 1 12:24:06 2017 -0800

----------------------------------------------------------------------
 .../YARN/3.0.0.3.0/service_advisor.py           | 51 ++++++++++++++----
 .../configuration/hive-interactive-site.xml     |  4 +-
 .../HIVE/configuration/tez-interactive-site.xml |  2 +-
 .../stacks/HDP/2.5/services/stack_advisor.py    | 56 +++++++++++++++-----
 .../stacks/HDP/2.5/upgrades/config-upgrade.xml  |  5 ++
 .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml |  4 ++
 .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml     |  1 +
 .../stacks/2.5/common/test_stack_advisor.py     | 44 +++++++--------
 8 files changed, 117 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index e1e03c1..18938a3 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -48,6 +48,8 @@ class YARNServiceAdvisor(service_advisor.ServiceAdvisor):
     self.as_super = super(YARNServiceAdvisor, self)
     self.as_super.__init__(*args, **kwargs)
 
+    self.CLUSTER_CREATE_OPERATION = "ClusterCreate"
+
     # Always call these methods
     self.modifyMastersWithMultipleInstances()
     self.modifyCardinalitiesDict()
@@ -488,6 +490,14 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
       Note: All memory calculations are in MB, unless specified otherwise.
     """
     Logger.info("DBG: Entered updateLlapConfigs")
+
+    # Determine if we entered here during cluster creation.
+    operation = getUserOperationContext(services, "operation")
+    is_cluster_create_opr = False
+    if operation == self.CLUSTER_CREATE_OPERATION:
+      is_cluster_create_opr = True
+    Logger.info("Is cluster create operation ? = {0}".format(is_cluster_create_opr))
+
     putHiveInteractiveSiteProperty = self.putProperty(configurations, YARNRecommender.HIVE_INTERACTIVE_SITE,
services)
     putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, YARNRecommender.HIVE_INTERACTIVE_SITE)
     putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env",
services)
@@ -605,7 +615,8 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     Logger.info("DBG: Calculated total_cluster_capacity : {0}, using following : node_manager_cnt
: {1}, "
                 "yarn_nm_mem_in_mb : {2}".format(total_cluster_capacity, node_manager_cnt,
yarn_nm_mem_in_mb))
     yarn_min_container_size = float(self.get_yarn_min_container_size(services, configurations))
-    tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity))
+    tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity),
is_cluster_create_opr,
+                                                                 changed_configs_has_enable_hive_int)
     normalized_tez_am_container_size = self._normalizeUp(tez_am_container_size, yarn_min_container_size)
 
     if yarn_site and "yarn.nodemanager.resource.cpu-vcores" in yarn_site:
@@ -619,7 +630,8 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
 
     # Calculate the available memory for LLAP app
     yarn_nm_mem_in_mb_normalized = self._normalizeDown(yarn_nm_mem_in_mb, yarn_min_container_size)
-    mem_per_thread_for_llap = self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized,
cpu_per_nm_host)
+    mem_per_thread_for_llap = float(self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized,
cpu_per_nm_host,
+                                                                           is_cluster_create_opr,
changed_configs_has_enable_hive_int))
     Logger.info("DBG: Calculated mem_per_thread_for_llap : {0}, using following: yarn_nm_mem_in_mb_normalized
: {1}, "
                 "cpu_per_nm_host : {2}".format(mem_per_thread_for_llap, yarn_nm_mem_in_mb_normalized,
cpu_per_nm_host))
 
@@ -856,8 +868,10 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     # Done with calculations, updating calculated configs.
     Logger.info("DBG: Applying the calculated values....")
 
-    normalized_tez_am_container_size = long(normalized_tez_am_container_size)
-    putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size)
+    if is_cluster_create_opr or changed_configs_has_enable_hive_int:
+      normalized_tez_am_container_size = long(normalized_tez_am_container_size)
+      putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size)
+      Logger.info("DBG: Setting 'tez.am.resource.memory.mb' config value as : {0}".format(normalized_tez_am_container_size))
 
     if not llap_concurrency_in_changed_configs:
       min_llap_concurrency = 1
@@ -872,18 +886,19 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
     #TODO A single value is not being set for numNodes in case of a custom queue. Also the
attribute is set to non-visible, so the UI likely ends up using an old cached value
     if (num_llap_nodes != num_llap_nodes_requested):
-      Logger.info("User requested num_llap_nodes : {0}, but used/adjusted value for calculations
is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
+      Logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for
calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
     else:
-      Logger.info("Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
+      Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
 
     llap_container_size = long(llap_daemon_mem_per_node)
     putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
 
     # Set 'hive.tez.container.size' only if it is read as "SET_ON_FIRST_INVOCATION", implying
initialization.
     # Else, we don't (1). Override the previous calculated value or (2). User provided value.
-    if self.get_hive_tez_container_size(services) == YARNRecommender.CONFIG_VALUE_UINITIALIZED:
+    if is_cluster_create_opr or changed_configs_has_enable_hive_int:
       mem_per_thread_for_llap = long(mem_per_thread_for_llap)
       putHiveInteractiveSiteProperty('hive.tez.container.size', mem_per_thread_for_llap)
+      Logger.info("DBG: Setting 'hive.tez.container.size' config value as : {0}".format(mem_per_thread_for_llap))
 
     putTezInteractiveSiteProperty('tez.runtime.io.sort.mb', tez_runtime_io_sort_mb)
     if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]:
@@ -970,13 +985,14 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     # TODO: This potentially takes up the entire node leaving no space for AMs.
     return min(floor(nm_mem_per_node_normalized / mem_per_thread), nm_cpus_per_node)
 
-  def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host):
+  def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host,
is_cluster_create_opr=False,
+                                        enable_hive_interactive_1st_invocation=False):
     """
     Calculates 'mem_per_thread_for_llap' for 1st time initialization. Else returns 'hive.tez.container.size'
read value.
     """
     hive_tez_container_size = self.get_hive_tez_container_size(services)
 
-    if hive_tez_container_size == self.CONFIG_VALUE_UINITIALIZED:
+    if is_cluster_create_opr or enable_hive_interactive_1st_invocation:
       if nm_mem_per_node_normalized <= 1024:
         calculated_hive_tez_container_size = min(512, nm_mem_per_node_normalized)
       elif nm_mem_per_node_normalized <= 4096:
@@ -1003,6 +1019,18 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     if hsi_site and 'hive.tez.container.size' in hsi_site:
       hive_container_size = hsi_site['hive.tez.container.size']
 
+    if not hive_container_size:
+      # This can happen (1). If config is missing in hive-interactive-site or (2). its an
+      # upgrade scenario from Ambari 2.4 to Ambari 2.5 with HDP 2.5 installed. Read it
+      # from hive-site.
+      #
+      # If Ambari 2.5 after upgrade from 2.4 is managing HDP 2.6 here, this config would
have
+      # already been added in hive-interactive-site as part of HDP upgrade from 2.5 to 2.6,
+      # and we wont end up in this block to look up in hive-site.
+      hive_site = self.getServicesSiteProperties(services, "hive-site")
+      if hive_site and 'hive.tez.container.size' in hive_site:
+        hive_container_size = hive_site['hive.tez.container.size']
+
     return hive_container_size
 
   def get_llap_headroom_space(self, services, configurations):
@@ -1383,14 +1411,15 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
 
     return yarn_nm_mem_in_mb
 
-  def calculate_tez_am_container_size(self, services, total_cluster_capacity):
+  def calculate_tez_am_container_size(self, services, total_cluster_capacity, is_cluster_create_opr=False,
+                                      enable_hive_interactive_1st_invocation=False):
     """
     Calculates Tez App Master container size (tez.am.resource.memory.mb) for tez_hive2/tez-site
on initialization if values read is 0.
     Else returns the read value.
     """
     tez_am_resource_memory_mb = self.get_tez_am_resource_memory_mb(services)
     calculated_tez_am_resource_memory_mb = None
-    if tez_am_resource_memory_mb == YARNRecommender.CONFIG_VALUE_UINITIALIZED:
+    if is_cluster_create_opr or enable_hive_interactive_1st_invocation:
       if total_cluster_capacity <= 4096:
         calculated_tez_am_resource_memory_mb = 256
       elif total_cluster_capacity > 4096 and total_cluster_capacity <= 73728:

http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 8008bef..e11bb18 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -647,7 +647,7 @@ limitations under the License.
   </property>
   <property>
     <name>hive.tez.container.size</name>
-    <value>SET_ON_FIRST_INVOCATION</value>
+    <value>682</value>
     <depends-on>
       <property>
         <type>hive-interactive-env</type>
@@ -670,7 +670,7 @@ limitations under the License.
         <name>hive.server2.tez.sessions.per.default.queue</name>
       </property>
     </depends-on>
-    <on-ambari-upgrade add="true"/>
+    <on-ambari-upgrade add="false"/>
   </property>
   <property>
     <name>hive.auto.convert.join.noconditionaltask.size</name>

http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
index 1c5117e..e914d69 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
@@ -83,7 +83,7 @@
   </property>
   <property>
     <name>tez.am.resource.memory.mb</name>
-    <value>SET_ON_FIRST_INVOCATION</value>
+    <value>1536</value>
     <description>The amount of memory to be used by the AppMaster</description>
     <depends-on>
       <property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 6f3dfa7..a450d4d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -34,6 +34,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     self.YARN_ROOT_DEFAULT_QUEUE_NAME = 'default'
     self.AMBARI_MANAGED_LLAP_QUEUE_NAME = 'llap'
     self.CONFIG_VALUE_UINITIALIZED = 'SET_ON_FIRST_INVOCATION'
+    self.CLUSTER_CREATE_OPERATION = "ClusterCreate"
+
 
   def recommendOozieConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor,self).recommendOozieConfigurations(configurations, clusterData,
services, hosts)
@@ -780,6 +782,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       Note: All memory calculations are in MB, unless specified otherwise.
     """
     Logger.info("DBG: Entered updateLlapConfigs");
+
+    # Determine if we entered here during cluster creation.
+    operation = getUserOperationContext(services, "operation")
+    is_cluster_create_opr = False
+    if operation == self.CLUSTER_CREATE_OPERATION:
+      is_cluster_create_opr = True
+    Logger.info("Is cluster create operation ? = {0}".format(is_cluster_create_opr))
+
     putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE,
services)
     putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, self.HIVE_INTERACTIVE_SITE)
     putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env",
services)
@@ -901,7 +911,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     Logger.info("DBG: Calculated total_cluster_capacity : {0}, using following : node_manager_cnt
: {1}, "
                 "yarn_nm_mem_in_mb : {2}".format(total_cluster_capacity, node_manager_cnt,
yarn_nm_mem_in_mb))
     yarn_min_container_size = float(self.get_yarn_min_container_size(services, configurations))
-    tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity))
+    tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity),
is_cluster_create_opr,
+                                                                 changed_configs_has_enable_hive_int)
     normalized_tez_am_container_size = self._normalizeUp(tez_am_container_size, yarn_min_container_size)
 
     if yarn_site and "yarn.nodemanager.resource.cpu-vcores" in yarn_site:
@@ -915,7 +926,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     # Calculate the available memory for LLAP app
     yarn_nm_mem_in_mb_normalized = self._normalizeDown(yarn_nm_mem_in_mb, yarn_min_container_size)
-    mem_per_thread_for_llap = float(self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized,
cpu_per_nm_host))
+    mem_per_thread_for_llap = float(self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized,
cpu_per_nm_host,
+                                                                           is_cluster_create_opr,
changed_configs_has_enable_hive_int))
     Logger.info("DBG: Calculated mem_per_thread_for_llap : {0}, using following: yarn_nm_mem_in_mb_normalized
: {1}, "
                   "cpu_per_nm_host : {2}".format(mem_per_thread_for_llap, yarn_nm_mem_in_mb_normalized,
cpu_per_nm_host))
 
@@ -939,7 +951,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     Logger.info("DBG: Calculated 'min_node_required': {0}, using following : min_memory_required
: {1}, yarn_nm_mem_in_mb_normalized "
                 ": {2}".format(min_nodes_required, min_memory_required, yarn_nm_mem_in_mb_normalized))
     if min_nodes_required > node_manager_cnt:
-      Logger.warn("ERROR: Not enough memory/nodes to run LLAP");
+      Logger.warning("ERROR: Not enough memory/nodes to run LLAP");
       self.recommendDefaultLlapConfiguration(configurations, services, hosts)
       return
 
@@ -1176,8 +1188,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     # Done with calculations, updating calculated configs.
     Logger.info("DBG: Applying the calculated values....")
 
-    normalized_tez_am_container_size = long(normalized_tez_am_container_size)
-    putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size)
+    if is_cluster_create_opr or changed_configs_has_enable_hive_int:
+      normalized_tez_am_container_size = long(normalized_tez_am_container_size)
+      putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size)
+      Logger.info("DBG: Setting 'tez.am.resource.memory.mb' config value as : {0}".format(normalized_tez_am_container_size))
 
     if not llap_concurrency_in_changed_configs:
       min_llap_concurrency = 1
@@ -1193,20 +1207,22 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
     #TODO A single value is not being set for numNodes in case of a custom queue. Also the
attribute is set to non-visible, so the UI likely ends up using an old cached value
     if (num_llap_nodes != num_llap_nodes_requested):
-      Logger.info("User requested num_llap_nodes : {0}, but used/adjusted value for calculations
is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
+      Logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for
calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
     else:
-      Logger.info("Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
+      Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
     putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes)
-    Logger.info("Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
+    Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
 
     llap_container_size = long(llap_daemon_mem_per_node)
     putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
 
     # Set 'hive.tez.container.size' only if it is read as "SET_ON_FIRST_INVOCATION", implying
initialization.
     # Else, we don't (1). Override the previous calculated value or (2). User provided value.
-    if self.get_hive_tez_container_size(services) == self.CONFIG_VALUE_UINITIALIZED:
+    if is_cluster_create_opr or changed_configs_has_enable_hive_int:
       mem_per_thread_for_llap = long(mem_per_thread_for_llap)
       putHiveInteractiveSiteProperty('hive.tez.container.size', mem_per_thread_for_llap)
+      Logger.info("DBG: Setting 'hive.tez.container.size' config value as : {0}".format(mem_per_thread_for_llap))
+
 
     putTezInteractiveSiteProperty('tez.runtime.io.sort.mb', tez_runtime_io_sort_mb)
     if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]:
@@ -1296,13 +1312,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     # TODO: This potentially takes up the entire node leaving no space for AMs.
     return min(math.floor(nm_mem_per_node_normalized / mem_per_thread), nm_cpus_per_node)
 
-  def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host):
+  def calculate_mem_per_thread_for_llap(self, services, nm_mem_per_node_normalized, cpu_per_nm_host,
is_cluster_create_opr=False,
+                                        enable_hive_interactive_1st_invocation=False):
     """
     Calculates 'mem_per_thread_for_llap' for 1st time initialization. Else returns 'hive.tez.container.size'
read value.
     """
     hive_tez_container_size = self.get_hive_tez_container_size(services)
-
-    if hive_tez_container_size == self.CONFIG_VALUE_UINITIALIZED:
+    if is_cluster_create_opr or enable_hive_interactive_1st_invocation:
       if nm_mem_per_node_normalized <= 1024:
         calculated_hive_tez_container_size = min(512, nm_mem_per_node_normalized)
       elif nm_mem_per_node_normalized <= 4096:
@@ -1328,6 +1344,18 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     hsi_site = self.getServicesSiteProperties(services, self.HIVE_INTERACTIVE_SITE)
     if hsi_site and 'hive.tez.container.size' in hsi_site:
       hive_container_size = hsi_site['hive.tez.container.size']
+
+    if not hive_container_size:
+      # This can happen (1). If config is missing in hive-interactive-site or (2). its an
+      # upgrade scenario from Ambari 2.4 to Ambari 2.5 with HDP 2.5 installed. Read it
+      # from hive-site.
+      #
+      # If Ambari 2.5 after upgrade from 2.4 is managing HDP 2.6 here, this config would
have
+      # already been added in hive-interactive-site as part of HDP upgrade from 2.5 to 2.6,
+      # and we wont end up in this block to look up in hive-site.
+      hive_site = self.getServicesSiteProperties(services, "hive-site")
+      if hive_site and 'hive.tez.container.size' in hive_site:
+        hive_container_size = hive_site['hive.tez.container.size']
     return hive_container_size
 
   def get_llap_headroom_space(self, services, configurations):
@@ -1401,14 +1429,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     if yarn_min_container_size < 256:
       return 256
 
-  def calculate_tez_am_container_size(self, services, total_cluster_capacity):
+  def calculate_tez_am_container_size(self, services, total_cluster_capacity, is_cluster_create_opr=False,
enable_hive_interactive_1st_invocation=False):
     """
     Calculates Tez App Master container size (tez.am.resource.memory.mb) for tez_hive2/tez-site
on initialization if values read is 0.
     Else returns the read value.
     """
     tez_am_resource_memory_mb = self.get_tez_am_resource_memory_mb(services)
     calculated_tez_am_resource_memory_mb = None
-    if tez_am_resource_memory_mb == self.CONFIG_VALUE_UINITIALIZED:
+    if is_cluster_create_opr or enable_hive_interactive_1st_invocation:
       if total_cluster_capacity <= 4096:
         calculated_tez_am_resource_memory_mb = 256
       elif total_cluster_capacity > 4096 and total_cluster_capacity <= 73728:

http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index da334be..e5f07ba 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -444,6 +444,11 @@
             <set key="llap_headroom_space" value="12288"/>
           </definition>
 
+          <definition xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive">
+            <type>hive-interactive-site</type>
+            <transfer operation="copy" from-type="hive-site" from-key="hive.tez.container.size"
to-key="hive.tez.container.size" default-value="682"  if-type="hive-interactive-site" if-key="hive.tez.container.size"
if-key-state="absent"/>
+          </definition>
+
         </changes>
 
       </component>

http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 8e98ba8..0d2f1bc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -504,6 +504,10 @@
         <task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options_for_tez_am"/>
       </execute-stage>
 
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Copying hive.tez.container.size
to HIVE SERVER INTERACTIVE's hive-site">
+        <task xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive"/>
+      </execute-stage>
+
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing
property">
         <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
           <summary>Updating hive.enforce.bucketing property to true value</summary>

http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index c1e8f8e..f0a4f05 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -798,6 +798,7 @@
           <task xsi:type="configure" id="hive_log4j_parameterize" />
           <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
           <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"
/>
+          <task xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive"
/>
         </pre-upgrade>
         
         <pre-downgrade />

http://git-wip-us.apache.org/repos/asf/ambari/blob/baefbcca/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index cf1c0ee..2d7bcd3 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1214,7 +1214,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'286261248')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'512')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -1414,7 +1414,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'],
'341')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'286261248')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'341')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -1612,7 +1612,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'1024')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -1817,7 +1817,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'2048')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -2002,23 +2002,23 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
 
     self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'],
3)
-    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'10571')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'11253')
 
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'],
'3')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'],
'3')
 
-    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'],
'4427')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'],
'2037')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'],
'true')
 
-    self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'],
'4915')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'],
'7372')
     self.assertEqual(configurations['hive-interactive-env']['properties']['hive_heapsize'],
'2048')
     self.assertEqual(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'],
{'maximum': '3', 'minimum': '1', 'read_only': 'false'})
 
     self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'],
'341')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
-    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'858783744')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'1364')
+    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'682')
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -2417,7 +2417,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'2048')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -2624,7 +2624,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'3072')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -2809,23 +2809,23 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'],
3)
     self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
 
-    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'203918')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'202554')
 
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'],
'10')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'],
'10')
 
-    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'],
'183438')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'],
'161594')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'],
'true')
 
-    self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'],
'16384')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'],
'34816')
     self.assertEqual(configurations['hive-interactive-env']['properties']['hive_heapsize'],
'2048')
     self.assertEqual(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'],
{'maximum': '5', 'minimum': '1', 'read_only': 'false'})
 
     self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'],
'341')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
-    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'1145044992')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'341')
+    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'1705')
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -3022,7 +3022,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'2048')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}, {'value': 'llap', 'label': 'llap'}]})
 
 
@@ -3214,7 +3214,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'],
'2048')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'2048')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}]})
 
 
@@ -3438,7 +3438,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'],
'512')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'1024')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'a1', 'label': 'a1'}, {'value': 'b', 'label': 'b'}, {'value': 'llap',
'label': 'llap'}]})
 
 
@@ -3845,8 +3845,8 @@ class TestHDP25StackAdvisor(TestCase):
                       {'maximum': '1', 'minimum': '1', 'read_only': 'false'})
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'],
'3')
 
-    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'],
'197632')
-    self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'],
'2457')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'],
'188416')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'],
'9830')
 
 
 
@@ -4043,7 +4043,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'default')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'],
'572522496')
 
-    self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'],
'2048')
+    self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
{'entries': [{'value': 'default', 'label': 'default'}]})
 
 


Mime
View raw message