ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-17540. Add a validation check for HiveServer Interactive if total remaining capacity in cluster is less than 512 MB (a safeguard for Service Checks). Also following fixes : (1). Setting visibility to FALSE for 'LLAP queue capaci
Date Sun, 03 Jul 2016 10:39:55 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 ed3fd0389 -> 2f4df2261


AMBARI-17540. Add a validation check for HiveServer Interactive if total remaining capacity
in cluster is less than 512 MB (a safeguard for Service Checks). Also following fixes : (1).
Setting visibility to FALSE for 'LLAP queue capacity' slider if 'llap' queue is in STOPPED
state. and (2). security_status fn. for Hive Server interactive.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2f4df226
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2f4df226
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2f4df226

Branch: refs/heads/branch-2.4
Commit: 2f4df2261b7d6fa7b813e52c28fe0e119003bffb
Parents: ed3fd03
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Sun Jul 3 03:39:21 2016 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Sun Jul 3 03:39:34 2016 -0700

----------------------------------------------------------------------
 .../package/scripts/hive_server_interactive.py  | 63 +++++++++++++++++++-
 .../stacks/HDP/2.5/services/stack_advisor.py    | 36 +++++++++--
 .../stacks/2.5/common/test_stack_advisor.py     | 34 ++++++++++-
 3 files changed, 125 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/2f4df226/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 8fb91d6..1bdca08 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -48,6 +48,9 @@ from ambari_commons.os_family_impl import OsFamilyImpl
 
 from resource_management.core.exceptions import ComponentIsNotRunning
 from resource_management.libraries.functions.decorator import retry
+from resource_management.libraries.functions.security_commons import build_expectations,
\
+  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties,
\
+  FILE_TYPE_XML
 
 # Local Imports
 from setup_ranger_hive import setup_ranger_hive
@@ -148,7 +151,65 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
       check_process_status(pid_file)
 
     def security_status(self, env):
-      HiveServerDefault.security_status(env)
+      import status_params
+      env.set_params(status_params)
+
+      if status_params.security_enabled:
+        props_value_check = {"hive.server2.authentication": "KERBEROS",
+                             "hive.metastore.sasl.enabled": "true",
+                             "hive.security.authorization.enabled": "true"}
+        props_empty_check = ["hive.server2.authentication.kerberos.keytab",
+                             "hive.server2.authentication.kerberos.principal",
+                             "hive.server2.authentication.spnego.principal",
+                             "hive.server2.authentication.spnego.keytab"]
+
+        props_read_check = ["hive.server2.authentication.kerberos.keytab",
+                            "hive.server2.authentication.spnego.keytab"]
+        hive_site_props = build_expectations('hive-site', props_value_check, props_empty_check,
+                                             props_read_check)
+
+        hive_expectations ={}
+        hive_expectations.update(hive_site_props)
+
+        security_params = get_params_from_filesystem(status_params.hive_server_interactive_conf_dir,
+                                                     {'hive-site.xml': FILE_TYPE_XML})
+        result_issues = validate_security_config_properties(security_params, hive_expectations)
+        if not result_issues: # If all validations passed successfully
+          try:
+            # Double check the dict before calling execute
+            if 'hive-site' not in security_params \
+              or 'hive.server2.authentication.kerberos.keytab' not in security_params['hive-site']
\
+              or 'hive.server2.authentication.kerberos.principal' not in security_params['hive-site']
\
+              or 'hive.server2.authentication.spnego.keytab' not in security_params['hive-site']
\
+              or 'hive.server2.authentication.spnego.principal' not in security_params['hive-site']:
+              self.put_structured_out({"securityState": "UNSECURED"})
+              self.put_structured_out({"securityIssuesFound": "Keytab file or principal are
not set property."})
+              return
+
+            cached_kinit_executor(status_params.kinit_path_local,
+                                  status_params.hive_user,
+                                  security_params['hive-site']['hive.server2.authentication.kerberos.keytab'],
+                                  security_params['hive-site']['hive.server2.authentication.kerberos.principal'],
+                                  status_params.hostname,
+                                  status_params.tmp_dir)
+            cached_kinit_executor(status_params.kinit_path_local,
+                                  status_params.hive_user,
+                                  security_params['hive-site']['hive.server2.authentication.spnego.keytab'],
+                                  security_params['hive-site']['hive.server2.authentication.spnego.principal'],
+                                  status_params.hostname,
+                                  status_params.tmp_dir)
+            self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+          except Exception as e:
+            self.put_structured_out({"securityState": "ERROR"})
+            self.put_structured_out({"securityStateErrorInfo": str(e)})
+        else:
+          issues = []
+          for cf in result_issues:
+            issues.append("Configuration file %s did not pass the validation. Reason: %s"
% (cf, result_issues[cf]))
+          self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+          self.put_structured_out({"securityState": "UNSECURED"})
+      else:
+        self.put_structured_out({"securityState": "UNSECURED"})
 
     def restart_llap(self, env):
       """

http://git-wip-us.apache.org/repos/asf/ambari/blob/2f4df226/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 9426571..1d092cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -30,6 +30,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     Logger.initialize_logger()
     self.HIVE_INTERACTIVE_SITE = 'hive-interactive-site'
     self.YARN_ROOT_DEFAULT_QUEUE_NAME = 'default'
+    self.AMBARI_MANAGED_LLAP_QUEUE_NAME = 'llap'
 
   def createComponentLayoutRecommendations(self, services, hosts):
     parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
@@ -167,10 +168,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
          and Hive2 app.
       2. Queue selected in 'hive.llap.daemon.queue.name' config state should not be 'STOPPED'.
       3. 'hive.server2.enable.doAs' config should be set to 'false' for Hive2.
+      4. if 'llap' queue is selected, in order to run Service Checks, 'remaining available
capacity' in cluster is atleast 512 MB.
   """
   def validateHiveInteractiveSiteConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
     validationItems = []
     hsi_hosts = self.__getHostsForComponent(services, "HIVE", "HIVE_SERVER_INTERACTIVE")
+    curr_selected_queue_for_llap = None
+    curr_selected_queue_for_llap_cap_perc =  None
+    MIN_ASSUMED_CAP_REQUIRED_FOR_SERVICE_CHECKS = 512
     if len(hsi_hosts) > 0:
       capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
       if capacity_scheduler_properties:
@@ -178,16 +183,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
             'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
           curr_selected_queue_for_llap = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.queue.name']
           if curr_selected_queue_for_llap:
-            curr_selected_queue_for_llap_cap = capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+curr_selected_queue_for_llap+'.capacity')
-            if curr_selected_queue_for_llap_cap:
-              curr_selected_queue_for_llap_cap = int(float(curr_selected_queue_for_llap_cap))
+            curr_selected_queue_for_llap_cap_perc = capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+curr_selected_queue_for_llap+'.capacity')
+            if curr_selected_queue_for_llap_cap_perc:
+              curr_selected_queue_for_llap_cap_perc = int(float(curr_selected_queue_for_llap_cap_perc))
               min_reqd_queue_cap_perc = self.min_queue_perc_reqd_for_llap_and_hive_app(services,
hosts, configurations)
 
               # Validate that the selected queue in 'hive.llap.daemon.queue.name' should
be sized >= to minimum required
               # to run LLAP and Hive2 app.
-              if curr_selected_queue_for_llap_cap < min_reqd_queue_cap_perc:
+              if curr_selected_queue_for_llap_cap_perc < min_reqd_queue_cap_perc:
                 errMsg1 =  "Selected queue '{0}' capacity ({1}%) is less than minimum required
capacity ({2}%) for LLAP " \
-                          "app to run".format(curr_selected_queue_for_llap, curr_selected_queue_for_llap_cap,
min_reqd_queue_cap_perc)
+                          "app to run".format(curr_selected_queue_for_llap, curr_selected_queue_for_llap_cap_perc,
min_reqd_queue_cap_perc)
                 validationItems.append({"config-name": "hive.llap.daemon.queue.name","item":
self.getErrorItem(errMsg1)})
             else:
               Logger.error("Couldn't retrieve '{0}' queue's capacity from 'capacity-scheduler'
while doing validation checks for "
@@ -221,6 +226,23 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
           if hive2_enable_do_as == 'true':
             validationItems.append({"config-name": "hive.server2.enable.doAs","item": self.getErrorItem("Value
should be set to 'false' for Hive2.")})
 
+      # Validate that 'remaining available capacity' in cluster is atleast 512 MB, after
'llap' queue is selected,
+      # in order to run Service Checks.
+      if curr_selected_queue_for_llap and curr_selected_queue_for_llap_cap_perc and \
+          curr_selected_queue_for_llap == self.AMBARI_MANAGED_LLAP_QUEUE_NAME:
+        # Get total cluster capacity
+        node_manager_host_list = self.get_node_manager_hosts(services, hosts)
+        node_manager_cnt = len(node_manager_host_list)
+        yarn_nm_mem_in_mb = self.get_yarn_nm_mem_in_mb(services, configurations)
+        total_cluster_capacity = node_manager_cnt * yarn_nm_mem_in_mb
+        curr_selected_queue_for_llap_cap = float(curr_selected_queue_for_llap_cap_perc) /
100 * total_cluster_capacity
+        available_cap_in_cluster = total_cluster_capacity - curr_selected_queue_for_llap_cap
+        if available_cap_in_cluster < MIN_ASSUMED_CAP_REQUIRED_FOR_SERVICE_CHECKS:
+          errMsg3 =  "Capacity used by '{0}' queue is '{1}'. Service checks may not run as
remaining available capacity " \
+                     "({2}) in cluster is less than 512 MB.".format(self.AMBARI_MANAGED_LLAP_QUEUE_NAME,
curr_selected_queue_for_llap_cap, available_cap_in_cluster)
+          validationItems.append({"config-name": "hive.llap.daemon.queue.name","item": self.getWarnItem(errMsg3)})
+
+
     validationProblems = self.toConfigurationValidationProblems(validationItems, "hive-interactive-site")
     return validationProblems
 
@@ -525,7 +547,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
         if llap_daemon_selected_queue_name:
           llap_selected_queue_state = capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+llap_daemon_selected_queue_name+".state")
           if llap_selected_queue_state == None or llap_selected_queue_state == "STOPPED":
-            raise Fail("Selected LLAP app queue '{0}' current state is : '{1}'. Setting LLAP
configs to default values."
+            putHiveInteractiveEnvPropertyAttribute("llap_queue_capacity", "visible", "false")
+            raise Fail("Selected LLAP app queue '{0}' current state is : '{1}'. Setting LLAP
configs to default values "
+                       "and 'llap' queue capacity slider visibility to 'False'."
                        .format(llap_daemon_selected_queue_name, llap_selected_queue_state))
         else:
           raise Fail("Retrieved LLAP app queue name is : '{0}'. Setting LLAP configs to default
values."

http://git-wip-us.apache.org/repos/asf/ambari/blob/2f4df226/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index a5bfefc..4c65792 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -592,6 +592,38 @@ class TestHDP25StackAdvisor(TestCase):
     res_expected3 = []
     res3 = self.stackAdvisor.validateHiveInteractiveSiteConfigurations({}, {}, configurations,
services1, hosts)
     self.assertEquals(res3, res_expected3)
+
+
+    # Test D : When remaining available capacity is less than 512M (designated for running
service checks), using the passed in configurations.
+    # Expected : WARN error as 'Service checks may not run as remaining available capacity
is less than 512M'.
+    # With current configs passed in, also getting selected queue capacity is < than the
minimum required for LLAP app to run.
+    configurations = {
+      "yarn-site": {
+        "properties": {
+          "yarn.nodemanager.resource.memory-mb":"512",
+          "yarn.scheduler.minimum-allocation-mb":"341"
+        }
+      },
+      "hive-site": {
+        "properties": {
+          "hive.tez.container.size" : "341",
+          "tez.am.resource.memory.mb" : "341"
+        }
+      },
+      "tez-site": {
+        "properties": {
+          "tez.am.resource.memory.mb" : "341"
+        }
+      },
+    }
+    res_expected4 = [
+      {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity
(49%) is less than minimum required capacity (200%) for LLAP app to run",
+       'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
+      {'config-type': 'hive-interactive-site', 'message': "Capacity used by 'llap' queue
is '250.88'. Service checks may not run as remaining available capacity "
+                                                          "(261.12) in cluster is less than
512 MB.", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level':
'WARN'}]
+    res4 = self.stackAdvisor.validateHiveInteractiveSiteConfigurations({}, {}, configurations,
services1, hosts)
+
+    self.assertEquals(res4, res_expected4)
     pass
 
 
@@ -1537,7 +1569,7 @@ class TestHDP25StackAdvisor(TestCase):
     cap_sched_expected_dict = convertToDict(self.expected_capacity_scheduler_llap_queue_size_20['properties']['capacity-scheduler'])
     self.assertEqual(cap_sched_output_dict, cap_sched_expected_dict)
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
-                      {'maximum': '100', 'minimum': '20', 'visible': 'true'})
+                      {'maximum': '100', 'minimum': '20', 'visible': 'false'})
 
 
 


Mime
View raw message