ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-19737. Add a validation check in HSI about Capacity Scheduler preemption requirements.
Date Fri, 27 Jan 2017 19:04:51 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 f353d013a -> 72cff6c68


AMBARI-19737. Add a validation check in HSI about Capacity Scheduler preemption requirements.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/72cff6c6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/72cff6c6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/72cff6c6

Branch: refs/heads/branch-2.5
Commit: 72cff6c680d35bd12903d49fdb9f47b5c1163790
Parents: f353d01
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Fri Jan 27 11:02:48 2017 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Fri Jan 27 11:04:37 2017 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.5/services/stack_advisor.py      | 12 ++++++++++++
 .../2.5/common/services-normal-his-2-hosts.json   |  8 +++++++-
 .../2.5/common/services-normal-his-valid.json     |  8 +++++++-
 .../stacks/2.5/common/test_stack_advisor.py       | 18 +++++++++++-------
 4 files changed, 37 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/72cff6c6/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 70da914..71bde8a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -388,8 +388,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
   def validateHiveInteractiveEnvConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
     hive_site_env_properties = self.getSiteProperties(configurations, "hive-interactive-env")
+    yarn_site_properties = self.getSiteProperties(configurations, "yarn-site")
     validationItems = []
     hsi_hosts = self.getHostsForComponent(services, "HIVE", "HIVE_SERVER_INTERACTIVE")
+
+    # Check for expecting 'enable_hive_interactive' is ON given that there is HSI on atleast
one host present.
     if len(hsi_hosts) > 0:
       # HIVE_SERVER_INTERACTIVE is mapped to a host
       if 'enable_hive_interactive' not in hive_site_env_properties or (
@@ -407,6 +410,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
                                 "item": self.getErrorItem(
                                   "enable_hive_interactive in hive-interactive-env should
be set to false.")})
 
+    # Check for 'yarn.resourcemanager.scheduler.monitor.enable' config to be true if HSI
is ON.
+    if yarn_site_properties and 'yarn.resourcemanager.scheduler.monitor.enable' in yarn_site_properties:
+      scheduler_monitor_enabled = yarn_site_properties['yarn.resourcemanager.scheduler.monitor.enable']
+      if scheduler_monitor_enabled.lower() == 'false' and hive_site_env_properties and 'enable_hive_interactive'
in hive_site_env_properties and \
+        hive_site_env_properties['enable_hive_interactive'].lower() == 'true':
+        validationItems.append({"config-name": "enable_hive_interactive",
+                                "item": self.getWarnItem(
+                                  "When enabling LLAP, set 'yarn.resourcemanager.scheduler.monitor.enable'
to true to ensure that LLAP gets the full allocated capacity.")})
+
     validationProblems = self.toConfigurationValidationProblems(validationItems, "hive-interactive-env")
     return validationProblems
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/72cff6c6/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
index 46a6f4d..1c3d8b0 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
+++ b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
@@ -1052,7 +1052,8 @@
         "hive.llap.daemon.queue.name": "llap",
         "hive.server2.enable.doAs": "true",
         "hive.llap.daemon.num.executors": "0",
-        "hive.server2.tez.sessions.per.default.queue": "32"
+        "hive.server2.tez.sessions.per.default.queue": "32",
+        "hive.tez.container.size" : "341"
       }
     },
     "hive-interactive-env" : {
@@ -1103,6 +1104,11 @@
       "properties": {
         "tez.am.resource.memory.mb" : "341"
       }
+    },
+    "tez-interactive-site": {
+      "properties": {
+        "tez.am.resource.memory.mb" : "341"
+      }
     }
   },
   "changed-configurations" : [ ],

http://git-wip-us.apache.org/repos/asf/ambari/blob/72cff6c6/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
index 7955333..fad4a55 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
+++ b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
@@ -1049,7 +1049,8 @@
   "configurations" : {
     "hive-interactive-site" : {
       "properties": {
-        "hive.llap.daemon.queue.name": "llap"
+        "hive.llap.daemon.queue.name": "llap",
+        "hive.tez.container.size" : "341"
       }
     },
     "capacity-scheduler" : {
@@ -1095,6 +1096,11 @@
       "properties": {
         "tez.am.resource.memory.mb" : "341"
       }
+    },
+    "tez-interactive-site": {
+      "properties": {
+        "tez.am.resource.memory.mb" : "341"
+      }
     }
   },
   "changed-configurations" : [ ],

http://git-wip-us.apache.org/repos/asf/ambari/blob/72cff6c6/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index a53cb25..ac00bab 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -492,7 +492,8 @@ class TestHDP25StackAdvisor(TestCase):
         "properties": {"hive_security_authorization": "None"}
       },
       "yarn-site": {
-        "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"}
+        "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+                       "yarn.resourcemanager.scheduler.monitor.enable": "false"}
       }
     }
     configurations2 = {
@@ -506,17 +507,22 @@ class TestHDP25StackAdvisor(TestCase):
         "properties": {"hive_security_authorization": "None"}
       },
       "yarn-site": {
-        "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"}
+        "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+                       "yarn.resourcemanager.scheduler.monitor.enable": "true"}
       }
     }
     services = self.load_json("services-normal-his-valid.json")
 
+    # Checks for WARN message that 'yarn.resourcemanager.scheduler.monitor.enable' should
be true.
     res_expected = [
+      {'config-type': 'hive-interactive-env', 'message': "When enabling LLAP, set 'yarn.resourcemanager.scheduler.monitor.enable'
to true to ensure that LLAP gets the full allocated capacity.", 'type': 'configuration', 'config-name':
'enable_hive_interactive', 'level': 'WARN'}
     ]
     # the above error is not what we are checking for - just to keep test happy without having
to test
     res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults,
configurations, services, {})
     self.assertEquals(res, res_expected)
 
+    # (1). Checks for ERROR message for 'enable_hive_interactive' to be true.
+    # (2). Further, no message regarding 'yarn.resourcemanager.scheduler.monitor.enable'
as it is true already.
     res_expected = [
       {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires
enable_hive_interactive in hive-interactive-env set to true.', 'type': 'configuration', 'config-name':
'enable_hive_interactive', 'level': 'ERROR'}
     ]
@@ -525,7 +531,6 @@ class TestHDP25StackAdvisor(TestCase):
     pass
 
 
-  ''' TODO: Commenting, Need to fix validations in 2.5/stack_advisor and then fix the test
code
   """
   Tests validations for Hive Server Interactive site.
   """
@@ -558,7 +563,7 @@ class TestHDP25StackAdvisor(TestCase):
     # Expected : Error telling about the current size compared to minimum required size.
     services1 = self.load_json("services-normal-his-valid.json")
     res_expected1 = [
-      {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity
(49%) is less than minimum required "
+      {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity
(49.0%) is less than minimum required "
         "capacity (50%) for LLAP app to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name',
'level': 'ERROR'},
     ]
     res1 = self.stackAdvisor.validateHiveInteractiveSiteConfigurations({}, {}, {}, services1,
hosts)
@@ -575,7 +580,7 @@ class TestHDP25StackAdvisor(TestCase):
     #               than 50% of queue capacity for LLAP.
     services2 = self.load_json("services-normal-his-2-hosts.json")
     res_expected2 = [
-      {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity
(49%) is less than minimum required "
+      {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity
(49.0%) is less than minimum required "
                                                           "capacity (50%) for LLAP app to
run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
       {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' current
state is : 'STOPPED'. It is required to be in "
                                                           "'RUNNING' state for LLAP to run",
'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
@@ -636,7 +641,7 @@ class TestHDP25StackAdvisor(TestCase):
       },
     }
     res_expected4 = [
-      {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity
(49%) is less than minimum required capacity (200%) for LLAP app to run",
+      {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity
(49.0%) is less than minimum required capacity (200%) for LLAP app to run",
        'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
       {'config-type': 'hive-interactive-site', 'message': "Capacity used by 'llap' queue
is '250.88'. Service checks may not run as remaining available capacity "
                                                           "(261.12) in cluster is less than
512 MB.", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level':
'WARN'}]
@@ -645,7 +650,6 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(res4, res_expected4)
     pass
 
-  '''
 
 
 


Mime
View raw message