ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-17103. Incorporating the inheritance logic for tez_hive2/tez-interactive-site.xml using tez/tez-site.xml. Also, removing 'hive.tez.java.opts' config from hive2/hive-site.xml as it would be got via inheritance from hive/hive-site
Date Thu, 09 Jun 2016 21:08:49 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 4e2043de5 -> 66decdeb2


AMBARI-17103. Incorporating the inheritance logic for tez_hive2/tez-interactive-site.xml using
tez/tez-site.xml. Also, removing 'hive.tez.java.opts' config from hive2/hive-site.xml as it
would be got via inheritance from hive/hive-site.xml.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/66decdeb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/66decdeb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/66decdeb

Branch: refs/heads/trunk
Commit: 66decdeb25e0ca8666ba85a541c8a85b71fe4147
Parents: 4e2043d
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Thu Jun 9 12:32:48 2016 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Thu Jun 9 14:07:03 2016 -0700

----------------------------------------------------------------------
 .../package/scripts/hive_interactive.py         | 15 +++++-
 .../HIVE/configuration/hive-interactive-env.xml |  4 ++
 .../configuration/hive-interactive-site.xml     | 17 ++++---
 .../stacks/HDP/2.5/services/stack_advisor.py    | 33 +++++++------
 .../stacks/2.5/common/test_stack_advisor.py     | 50 ++++++++++----------
 5 files changed, 69 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/66decdeb/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
index b6b963b..631b5a8 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
@@ -101,10 +101,21 @@ def hive_interactive(name=None):
   '''
   remove_atlas_hook_if_exists(merged_hive_interactive_site)
 
-  # Merge tez-interactive with tez-site
+  '''
+  As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
+  we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
+  '''
+  merged_tez_interactive_site = {}
+  if 'tez-site' in params.config['configurations']:
+    merged_tez_interactive_site.update(params.config['configurations']['tez-site'])
+    Logger.info("Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'.")
+  else:
+    Logger.error("Tez's 'tez-site' couldn't be retrieved from passed-in configurations.")
+
+  merged_tez_interactive_site.update(params.config['configurations']['tez-interactive-site'])
   XmlConfig("tez-site.xml",
             conf_dir = params.tez_interactive_config_dir,
-            configurations = params.config['configurations']['tez-interactive-site'],
+            configurations = merged_tez_interactive_site,
             configuration_attributes=params.config['configuration_attributes']['tez-interactive-site'],
             owner = params.tez_interactive_user,
             group = params.user_group,

http://git-wip-us.apache.org/repos/asf/ambari/blob/66decdeb/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index c34351d..6f806d2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -153,6 +153,10 @@
         <type>hive-interactive-site</type>
         <name>hive.server2.tez.sessions.per.default.queue</name>
       </property>
+      <property>
+        <type>tez-site</type>
+        <name>tez.am.resource.memory.mb</name>
+      </property>
     </depends-on>
     <on-ambari-upgrade add="true"/>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/66decdeb/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 3227478..ac92b86 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -131,15 +131,6 @@ limitations under the License.
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>hive.tez.java.opts</name>
-    <value/>
-    <description>Java command line options for Tez. The -Xmx parameter value is generally
80% of hive.tez.container.size.</description>
-    <value-attributes>
-      <empty-value-valid>true</empty-value-valid>
-    </value-attributes>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
     <name>hive.vectorized.execution.reduce.enabled</name>
     <value>true</value>
     <description>
@@ -389,6 +380,10 @@ limitations under the License.
         <type>hive-interactive-site</type>
         <name>hive.server2.tez.sessions.per.default.queue</name>
       </property>
+      <property>
+        <type>tez-site</type>
+        <name>tez.am.resource.memory.mb</name>
+      </property>
     </depends-on>
     <on-ambari-upgrade add="true"/>
   </property>
@@ -421,6 +416,10 @@ limitations under the License.
         <type>hive-interactive-site</type>
         <name>hive.server2.tez.sessions.per.default.queue</name>
       </property>
+      <property>
+        <type>tez-site</type>
+        <name>tez.am.resource.memory.mb</name>
+      </property>
     </depends-on>
     <on-ambari-upgrade add="true"/>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/66decdeb/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 6b5e2a1..df6c65c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -816,27 +816,26 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     return yarn_nm_mem_in_mb
 
   """
-  Gets Tez App Master container size (tez.am.resource.memory.mb). Takes into account if it
has been calculated
+  Gets Tez App Master container size (tez.am.resource.memory.mb) from tez-site. Takes into
account if it has been calculated
   as part of current Stack Advisor invocation.
   """
   def get_tez_am_container_size(self, services, configurations):
     llap_daemon_container_size = None
-
     # Check if 'tez.am.resource.memory.mb' is modified in current ST invocation.
-    if 'tez-interactive-site' in configurations and 'tez.am.resource.memory.mb' in configurations['tez-interactive-site']['properties']:
-      llap_daemon_container_size = float(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'])
+    if 'tez-site' in configurations and 'tez.am.resource.memory.mb' in configurations['tez-site']['properties']:
+      llap_daemon_container_size = float(configurations['tez-site']['properties']['tez.am.resource.memory.mb'])
       Logger.info("'tez.am.resource.memory.mb' read from configurations as : {0}".format(llap_daemon_container_size))
 
     if not llap_daemon_container_size:
       # Check if 'tez.am.resource.memory.mb' is input in services array.
-      if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
-          'tez.am.resource.memory.mb' in services['configurations']['tez-interactive-site']['properties']:
-        llap_daemon_container_size = float(services['configurations']['tez-interactive-site']['properties']['tez.am.resource.memory.mb'])
+      if 'tez-site' in services['configurations'] and \
+          'tez.am.resource.memory.mb' in services['configurations']['tez-site']['properties']:
+        llap_daemon_container_size = float(services['configurations']['tez-site']['properties']['tez.am.resource.memory.mb'])
         Logger.info("'tez.am.resource.memory.mb' read from services as : {0}".format(llap_daemon_container_size))
 
 
     if not llap_daemon_container_size:
-      raise Fail("Couldn't retrieve Hive Server interactive's 'tez.am.resource.memory.mb'
config.")
+      raise Fail("Couldn't retrieve Hive Server's 'tez.am.resource.memory.mb' config.")
 
     assert (llap_daemon_container_size > 0), "'tez.am.resource.memory.mb' current value
: {0}. " \
                                              "Expected value : > 0".format(llap_daemon_container_size)
@@ -865,9 +864,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     min_required_perc = min_required * 100 / total_cluster_cap
     Logger.info("Calculated min_llap_queue_perc_required_in_cluster : {0} and min_llap_queue_cap_required_in_cluster:
{1} "
-                "using following : yarn_min_container_size : {2}, ""slider_am_size : {3},
hive_tez_container_size : {4}, "
-                "hive_am_container_size : {5}".format(min_required_perc, min_required, yarn_min_container_size,
-                slider_am_size, hive_tez_container_size, tez_am_container_size))
+                "using following : yarn_min_container_size : {2}, slider_am_size : {3}, hive_tez_container_size
: {4}, "
+                "hive_am_container_size : {5}, total_cluster_cap : {6}, yarn_rm_mem_in_mb
: {7}"
+                "".format(min_required_perc, min_required, yarn_min_container_size, slider_am_size,
hive_tez_container_size,
+                          tez_am_container_size, total_cluster_cap, yarn_rm_mem_in_mb))
     return int(math.ceil(min_required_perc))
 
   """
@@ -903,9 +903,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       if 'llap_queue_capacity' in services['configurations']['hive-interactive-env']['properties']:
         llap_slider_cap_percentage = int(
           services['configurations']['hive-interactive-env']['properties']['llap_queue_capacity'])
-        llap_min_reqd_cap_percentage = self.min_llap_queue_perc_required_in_cluster(services,
hosts, configurations)
+        llap_min_reqd_cap_percentage = math.ceil(self.min_llap_queue_perc_required_in_cluster(services,
hosts, configurations))
+        # Checks for upper bound for the minimum llap queue calculated capacity
+        if llap_min_reqd_cap_percentage > 100:
+          Logger.error("Minimum calculated 'llap' queue capacity ({0}%) is more than what
the cluster has in totality. "
+                       "Not creating/adjusting 'llap' queue. Increase the overall cluster
capacity.".format(llap_min_reqd_cap_percentage))
+          return
         if llap_slider_cap_percentage <= 0 or llap_slider_cap_percentage > 100:
-          Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage,
llap_min_reqd_cap_percentage))
+          Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% (invalid size) to {1}%".format(llap_slider_cap_percentage,
llap_min_reqd_cap_percentage))
           putHiveInteractiveEnvProperty('llap_queue_capacity', llap_min_reqd_cap_percentage)
           llap_slider_cap_percentage = llap_min_reqd_cap_percentage
       else:
@@ -926,7 +931,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       # Get 'llap' queue capacity
       currLlapQueueCap = -1
       if 'yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity' in cap_sched_config_keys:
-        currLlapQueueCap = int(capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity'))
+        currLlapQueueCap = int(float(capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity')))
 
       if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
           'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:

http://git-wip-us.apache.org/repos/asf/ambari/blob/66decdeb/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 22b3c52..5a54548 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -146,7 +146,7 @@ class TestHDP25StackAdvisor(TestCase):
     # Expected capacity-scheduler with 'llap' (size:20) and 'default' queue at root level.
     self.expected_capacity_scheduler_llap_queue_size_20 = {
       "properties": {
-        "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=80\n'
+        "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=80.0\n'
                               'yarn.scheduler.capacity.root.accessible-node-labels=*\n'
                               'yarn.scheduler.capacity.root.capacity=100\n'
                               'yarn.scheduler.capacity.root.queues=default,llap\n'
@@ -155,7 +155,7 @@ class TestHDP25StackAdvisor(TestCase):
                               'yarn.scheduler.capacity.root.default.state=RUNNING\n'
                               'yarn.scheduler.capacity.maximum-am-resource-percent=1\n'
                               'yarn.scheduler.capacity.root.default.acl_submit_applications=*\n'
-                              'yarn.scheduler.capacity.root.default.capacity=80\n'
+                              'yarn.scheduler.capacity.root.default.capacity=80.0\n'
                               'yarn.scheduler.capacity.root.acl_administer_queue=*\n'
                               'yarn.scheduler.capacity.node-locality-delay=40\n'
                               'yarn.scheduler.capacity.queue-mappings-override.enable=false\n'
@@ -163,8 +163,8 @@ class TestHDP25StackAdvisor(TestCase):
                               'yarn.scheduler.capacity.root.llap.state=RUNNING\n'
                               'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n'
                               'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n'
-                              'yarn.scheduler.capacity.root.llap.maximum-capacity=20\n'
-                              'yarn.scheduler.capacity.root.llap.capacity=20\n'
+                              'yarn.scheduler.capacity.root.llap.maximum-capacity=20.0\n'
+                              'yarn.scheduler.capacity.root.llap.capacity=20.0\n'
                               'yarn.scheduler.capacity.root.llap.acl_submit_applications=hive\n'
                               'yarn.scheduler.capacity.root.llap.acl_administer_queue=hive\n'
                               'yarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'
@@ -351,7 +351,7 @@ class TestHDP25StackAdvisor(TestCase):
     }
 
     # Expected 'hive_interactive_env' with 'llap_queue_capacity' set to 20.
-    self.expected_llap_queue_capacity_20 = '20'
+    self.expected_llap_queue_capacity_20 = '20.0'
 
     # Expected 'hive_interactive_env' with 'llap_queue_capacity' set to 40.
     self.expected_llap_queue_capacity_40 = '40'
@@ -614,7 +614,7 @@ class TestHDP25StackAdvisor(TestCase):
               'hive.llap.daemon.queue.name':'default'
             }
           },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -768,7 +768,7 @@ class TestHDP25StackAdvisor(TestCase):
               'hive.llap.daemon.queue.name':'default'
             }
           },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -920,7 +920,7 @@ class TestHDP25StackAdvisor(TestCase):
               'hive.llap.daemon.queue.name':'default',
             }
           },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -1072,7 +1072,7 @@ class TestHDP25StackAdvisor(TestCase):
               'hive.llap.daemon.queue.name':'default',
             }
           },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -1235,7 +1235,7 @@ class TestHDP25StackAdvisor(TestCase):
               'hive.server2.tez.sessions.per.default.queue' : '1'
             }
           },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -1392,7 +1392,7 @@ class TestHDP25StackAdvisor(TestCase):
               'hive.llap.daemon.queue.name':'llap'
             }
           },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -1556,7 +1556,7 @@ class TestHDP25StackAdvisor(TestCase):
               'hive.llap.daemon.queue.name':'llap'
             }
           },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -3236,7 +3236,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '1'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "512"
           }
@@ -3421,7 +3421,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '1'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -3614,7 +3614,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '1'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -3808,7 +3808,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '4'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -3997,7 +3997,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '3'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -4186,7 +4186,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '3'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -4377,7 +4377,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '4'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -4568,7 +4568,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '10'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "341"
           }
@@ -4757,7 +4757,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '3'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -4933,7 +4933,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '3'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -5112,7 +5112,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '3'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }
@@ -5465,7 +5465,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '3'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024",
           }
@@ -5635,7 +5635,7 @@ class TestHDP25StackAdvisor(TestCase):
             "yarn.nodemanager.resource.cpu-vcores": '3'
           }
         },
-        "tez-interactive-site": {
+        "tez-site": {
           "properties": {
             "tez.am.resource.memory.mb": "1024"
           }


Mime
View raw message