ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [07/28] ambari git commit: AMBARI-20279. HiveServerInteractive. Set value for num_llap_nodes_for_llap_daemons only if it exists in hive-interactive-site.
Date Fri, 03 Mar 2017 18:03:54 GMT
AMBARI-20279. HiveServerInteractive. Set value for num_llap_nodes_for_llap_daemons only if
it exists in hive-interactive-site.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1142b526
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1142b526
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1142b526

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 1142b526195feced7b42c82119220e065d56e24b
Parents: 7e2eb29
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Thu Mar 2 14:25:12 2017 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Thu Mar 2 17:19:33 2017 -0800

----------------------------------------------------------------------
 .../YARN/3.0.0.3.0/service_advisor.py           | 15 ++++++++--
 .../stacks/HDP/2.5/services/stack_advisor.py    | 17 +++++++++--
 .../stacks/2.5/common/test_stack_advisor.py     | 30 +++++++++++++-------
 3 files changed, 45 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1142b526/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index 3629f30..3d77a26 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -918,8 +918,12 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     else:
       Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
 
-    putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes)
-    Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
+    # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site.
+    # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this
config is from 2.6 stack onwards only.
+    if "hive-interactive-env" in services["configurations"] and \
+        "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]:
+      putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes)
+      Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
 
     llap_container_size = long(llap_daemon_mem_per_node)
     putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
@@ -980,7 +984,12 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue',
"minimum", 1)
     putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue',
"maximum", 1)
     putHiveInteractiveEnvProperty('num_llap_nodes', 0)
-    putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0)
+
+    # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site.
+    # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this
config is from 2.6 stack onwards only.
+    if "hive-interactive-env" in services["configurations"] and \
+        "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]:
+      putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0)
     putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1)
     putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
     putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size)

http://git-wip-us.apache.org/repos/asf/ambari/blob/1142b526/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index a450d4d..85e632d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -1210,8 +1210,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       Logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for
calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
     else:
       Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
-    putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes)
-    Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
+
+    # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site.
+    # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this
config is from 2.6 stack onwards only.
+    if "hive-interactive-env" in services["configurations"] and \
+        "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]:
+      putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes)
+      Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
 
     llap_container_size = long(llap_daemon_mem_per_node)
     putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
@@ -1274,7 +1279,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue',
"minimum", 1)
     putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue',
"maximum", 1)
     putHiveInteractiveEnvProperty('num_llap_nodes', 0)
-    putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0)
+
+    # Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site.
+    # This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this
config is from 2.6 stack onwards only.
+    if "hive-interactive-env" in services["configurations"] and \
+        "num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]:
+      putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0)
+
     putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1)
     putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
     putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size)

http://git-wip-us.apache.org/repos/asf/ambari/blob/1142b526/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 2d7bcd3..a913b35 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1134,7 +1134,8 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'num_llap_nodes':'1'
+              'num_llap_nodes':'1',
+              'num_llap_nodes_for_llap_daemons': '1'
             }
           },
         "hive-interactive-site":
@@ -1336,7 +1337,8 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'num_llap_nodes':'1'
+              'num_llap_nodes':'1',
+              'num_llap_nodes_for_llap_daemons': '1'
             }
           },
         "hive-interactive-site":
@@ -1532,7 +1534,8 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'num_llap_nodes': 1
+              'num_llap_nodes': 1,
+              'num_llap_nodes_for_llap_daemons': '1'
             }
           },
         "hive-interactive-site":
@@ -1737,7 +1740,8 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'num_llap_nodes':'3'
+              'num_llap_nodes':'3',
+              'num_llap_nodes_for_llap_daemons': '1'
             }
           },
         "hive-interactive-site":
@@ -1938,6 +1942,7 @@ class TestHDP25StackAdvisor(TestCase):
             'properties': {
               'enable_hive_interactive': 'true',
               'num_llap_nodes':'3',
+              'num_llap_nodes_for_llap_daemons': '1',
              }
           },
         "hive-interactive-site":
@@ -2337,7 +2342,8 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'num_llap_nodes':'3'
+              'num_llap_nodes':'3',
+              'num_llap_nodes_for_llap_daemons': '1'
             }
           },
         "hive-interactive-site":
@@ -2605,7 +2611,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'],
'1.0')
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'],
{'maximum': '4', 'minimum': '1'})
 
-    self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'],
3)
+    self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties'])
     self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
 
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'36864')
@@ -2806,7 +2812,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'],
'1')
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'],
{'maximum': '4', 'minimum': '1'})
 
-    self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'],
3)
+    self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties'])
     self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
 
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'202554')
@@ -3003,7 +3009,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler':
'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.roo
 t.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'})
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'],
{'maximum': '4'})
 
-    self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'],
3)
+    self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties'])
     self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
 
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'202752')
@@ -3197,7 +3203,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'],
{'maximum': '4'})
 
     self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'],
3)
-    self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'],
3)
+    self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties'])
 
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'202752')
 
@@ -3421,7 +3427,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'],
{'maximum': '4', 'minimum': '1'})
 
     self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'],
3)
-    self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'],
3)
+    self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties'])
 
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'],
'204288')
 
@@ -3588,6 +3594,7 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
+              'num_llap_nodes_for_llap_daemons': '1'
             }
           },
         "hive-interactive-site":
@@ -3960,7 +3967,8 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'num_llap_nodes': '1'
+              'num_llap_nodes': '1',
+              'num_llap_nodes_for_llap_daemons': '0'
             }
           },
         "hive-interactive-site":


Mime
View raw message