ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-12796 Cannot add service after upgrade and enabling ranger (dsen)
Date Mon, 17 Aug 2015 20:30:33 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1.1 5e4782a20 -> 0c3c4a345


AMBARI-12796 Cannot add service after upgrade and enabling ranger (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0c3c4a34
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0c3c4a34
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0c3c4a34

Branch: refs/heads/branch-2.1.1
Commit: 0c3c4a34528bc2c47404bb9fabb29fa0c7b07600
Parents: 5e4782a
Author: Dmytro Sen <dsen@apache.org>
Authored: Mon Aug 17 23:28:33 2015 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Mon Aug 17 23:30:24 2015 +0300

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 21 ++++-
 .../stacks/HDP/2.2/services/stack_advisor.py    | 40 +++++++---
 .../stacks/HDP/2.3/services/stack_advisor.py    |  4 +-
 .../src/main/resources/stacks/stack_advisor.py  |  7 +-
 .../stacks/2.3/common/test_stack_advisor.py     | 81 ++++++++++++++++++++
 5 files changed, 134 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/0c3c4a34/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 5c65535..d3ebce7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -372,8 +372,21 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
               print("SiteName: %s, method: %s\n" % (siteName, method.__name__))
               print("Site properties: %s\n" % str(siteProperties))
               print("Recommendations: %s\n********\n" % str(siteRecommendations))
-              resultItems = method(siteProperties, siteRecommendations, configurations, services,
hosts)
-              items.extend(resultItems)
+              try:
+                resultItems = method(siteProperties, siteRecommendations, configurations,
services, hosts)
+                items.extend(resultItems)
+              except (AttributeError, TypeError, LookupError) as e:
+                msg = "Failed to validate configuration "
+                print msg
+                print e
+                items.extend([{
+                                'message': msg,
+                                'level': 'ERROR',
+                                'config-type': siteName,
+                                'config-name': '',
+                                'type': 'configuration'
+                              }])
+
     clusterWideItems = self.validateClusterConfigurations(configurations, services, hosts)
     items.extend(clusterWideItems)
     self.validateMinMax(items, recommendedDefaults, configurations)
@@ -473,7 +486,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     distributed_item = None
     distributed = properties.get("hbase.cluster.distributed")
-    if hbase_rootdir.startswith("hdfs://") and not distributed.lower() == "true":
+    if hbase_rootdir and hbase_rootdir.startswith("hdfs://") and not distributed.lower()
== "true":
       distributed_item = self.getErrorItem("Distributed property should be set to true if
hbase.rootdir points to HDFS.")
 
     validationItems.extend([{"config-name":'hbase.rootdir', "item": rootdir_item },
@@ -523,7 +536,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     hbase_site = getSiteProperties(configurations, "ams-hbase-site")
     hbase_rootdir = hbase_site.get("hbase.rootdir")
     regionServerMinMemItem = None
-    if hbase_rootdir.startswith("hdfs://"):
+    if hbase_rootdir and hbase_rootdir.startswith("hdfs://"):
       regionServerMinMemItem = self.validateMinMemorySetting(properties, 1024, 'hbase_regionserver_heapsize')
 
     validationItems = [{"config-name": "hbase_regionserver_heapsize", "item": regionServerItem},

http://git-wip-us.apache.org/repos/asf/ambari/blob/0c3c4a34/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 6bb6a9d..ac94185 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -750,19 +750,22 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                         {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
                         {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}]
 
-    if checkXmxValueFormat(properties['mapreduce.map.java.opts']):
+    if 'mapreduce.map.java.opts' in properties and \
+      checkXmxValueFormat(properties['mapreduce.map.java.opts']):
       mapreduceMapJavaOpts = formatXmxSizeToBytes(getXmxSize(properties['mapreduce.map.java.opts']))
/ (1024.0 * 1024)
       mapreduceMapMemoryMb = to_number(properties['mapreduce.map.memory.mb'])
       if mapreduceMapJavaOpts > mapreduceMapMemoryMb:
         validationItems.append({"config-name": 'mapreduce.map.java.opts', "item": self.getWarnItem("mapreduce.map.java.opts
Xmx should be less than mapreduce.map.memory.mb ({0})".format(mapreduceMapMemoryMb))})
 
-    if checkXmxValueFormat(properties['mapreduce.reduce.java.opts']):
+    if 'mapreduce.reduce.java.opts' in properties and \
+      checkXmxValueFormat(properties['mapreduce.reduce.java.opts']):
       mapreduceReduceJavaOpts = formatXmxSizeToBytes(getXmxSize(properties['mapreduce.reduce.java.opts']))
/ (1024.0 * 1024)
       mapreduceReduceMemoryMb = to_number(properties['mapreduce.reduce.memory.mb'])
       if mapreduceReduceJavaOpts > mapreduceReduceMemoryMb:
         validationItems.append({"config-name": 'mapreduce.reduce.java.opts', "item": self.getWarnItem("mapreduce.reduce.java.opts
Xmx should be less than mapreduce.reduce.memory.mb ({0})".format(mapreduceReduceMemoryMb))})
 
-    if checkXmxValueFormat(properties['yarn.app.mapreduce.am.command-opts']):
+    if 'yarn.app.mapreduce.am.command-opts' in properties and \
+      checkXmxValueFormat(properties['yarn.app.mapreduce.am.command-opts']):
       yarnAppMapreduceAmCommandOpts = formatXmxSizeToBytes(getXmxSize(properties['yarn.app.mapreduce.am.command-opts']))
/ (1024.0 * 1024)
       yarnAppMapreduceAmResourceMb = to_number(properties['yarn.app.mapreduce.am.resource.mb'])
       if yarnAppMapreduceAmCommandOpts > yarnAppMapreduceAmResourceMb:
@@ -822,13 +825,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties
else 'No'
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()):
-      if hdfs_site['dfs.permissions.enabled'] != 'true':
+      if 'dfs.permissions.enabled' in hdfs_site and \
+        hdfs_site['dfs.permissions.enabled'] != 'true':
         validationItems.append({"config-name": 'dfs.permissions.enabled',
                                     "item": self.getWarnItem(
                                       "dfs.permissions.enabled needs to be set to true if
Ranger HDFS Plugin is enabled.")})
 
     if (not wire_encryption_enabled and   # If wire encryption is enabled at Hadoop, it disables
all our checks
+          'hadoop.security.authentication' in core_site and
           core_site['hadoop.security.authentication'] == 'kerberos' and
+          'hadoop.security.authorization' in core_site and
           core_site['hadoop.security.authorization'] == 'true'):
       # security is enabled
 
@@ -961,7 +967,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     validationItems = []
     hive_env = properties
     hive_site = getSiteProperties(configurations, "hive-site")
-    if str(hive_env["hive_security_authorization"]).lower() == "none" \
+    if "hive_security_authorization" in hive_env and \
+        str(hive_env["hive_security_authorization"]).lower() == "none" \
       and str(hive_site["hive.security.authorization.enabled"]).lower() == "true":
       authorization_item = self.getErrorItem("hive_security_authorization should not be None
"
                                              "if hive.security.authorization.enabled is set")
@@ -976,7 +983,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     #Adding Ranger Plugin logic here
     ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties")
     hive_env_properties = getSiteProperties(configurations, "hive-env")
-    ranger_plugin_enabled = 'hive_security_authorization' in hive_env_properties and hive_env_properties['hive_security_authorization'].lower()
== 'ranger'
+    ranger_plugin_enabled = hive_env_properties \
+                            and 'hive_security_authorization' in hive_env_properties \
+                            and hive_env_properties['hive_security_authorization'].lower()
== 'ranger'
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     ##Add stack validations only if Ranger is enabled.
     if ("RANGER" in servicesList):
@@ -984,7 +993,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       if ranger_plugin_enabled:
         prop_name = 'hive.security.authorization.enabled'
         prop_val = 'true'
-        if hive_site[prop_name] != prop_val:
+        if hive_site and \
+            prop_name in hive_site and \
+          hive_site[prop_name] != prop_val:
           validationItems.append({"config-name": prop_name,
                                   "item": self.getWarnItem(
                                     "If Ranger Hive Plugin is enabled." \
@@ -992,7 +1003,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
 
         prop_name = 'hive.conf.restricted.list'
         prop_vals = 'hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager'.split(',')
-        current_vals = hive_site[prop_name].split(',')
+        current_vals = []
+        if hive_site and prop_name in hive_site:
+          current_vals = hive_site[prop_name].split(',')
+
         missing_vals = []
 
         for val in prop_vals:
@@ -1006,7 +1020,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                                   " {0} needs to contain {1}".format(prop_name, ','.join(missing_vals)))})
     stripe_size_values = [8388608, 16777216, 33554432, 67108864, 134217728, 268435456]
     stripe_size_property = "hive.exec.orc.default.stripe.size"
-    if int(properties[stripe_size_property]) not in stripe_size_values:
+    if stripe_size_property in properties and \
+        int(properties[stripe_size_property]) not in stripe_size_values:
       validationItems.append({"config-name": stripe_size_property,
                               "item": self.getWarnItem("Correct values are {0}".format(stripe_size_values))
                              }
@@ -1023,15 +1038,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     prop_name2 = 'hfile.block.cache.size'
     props_max_sum = 0.8
 
-    if not is_number(hbase_site[prop_name1]):
+    if prop_name1 in hbase_site and not is_number(hbase_site[prop_name1]):
       validationItems.append({"config-name": prop_name1,
                               "item": self.getWarnItem(
                               "{0} should be float value".format(prop_name1))})
-    elif not is_number(hbase_site[prop_name2]):
+    elif prop_name2 in hbase_site and not is_number(hbase_site[prop_name2]):
       validationItems.append({"config-name": prop_name2,
                               "item": self.getWarnItem(
                               "{0} should be float value".format(prop_name2))})
-    elif float(hbase_site[prop_name1]) + float(hbase_site[prop_name2]) > props_max_sum:
+    elif prop_name1 in hbase_site and prop_name2 in hbase_site and \
+          float(hbase_site[prop_name1]) + float(hbase_site[prop_name2]) > props_max_sum:
       validationItems.append({"config-name": prop_name1,
                               "item": self.getWarnItem(
                               "{0} and {1} sum should not exceed {2}".format(prop_name1,
prop_name2, props_max_sum))})

http://git-wip-us.apache.org/repos/asf/ambari/blob/0c3c4a34/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 2f84be1..9329032 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -242,14 +242,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       if ranger_plugin_enabled:
         prop_name = 'hive.security.authorization.manager'
         prop_val = "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory"
-        if hive_server2[prop_name] != prop_val:
+        if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
           validationItems.append({"config-name": prop_name,
                                   "item": self.getWarnItem(
                                   "If Ranger Hive Plugin is enabled."\
                                   " {0} needs to be set to {1}".format(prop_name,prop_val))})
         prop_name = 'hive.security.authenticator.manager'
         prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
-        if hive_server2[prop_name] != prop_val:
+        if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
           validationItems.append({"config-name": prop_name,
                                   "item": self.getWarnItem(
                                   "If Ranger Hive Plugin is enabled."\

http://git-wip-us.apache.org/repos/asf/ambari/blob/0c3c4a34/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 0ef953d..708aaf7 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -566,7 +566,12 @@ class DefaultStackAdvisor(StackAdvisor):
       for service in servicesList:
         calculation = self.getServiceConfigurationRecommender(service)
         if calculation is not None:
-          calculation(configurations, clusterSummary, services, hosts)
+          try:
+            calculation(configurations, clusterSummary, services, hosts)
+          except (AttributeError, TypeError, LookupError) as e:
+            # NOP
+            print "Failed to recommend configuration "
+            print e
 
     return recommendations
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/0c3c4a34/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 8a37391..7f948bd 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -731,3 +731,84 @@ class TestHDP23StackAdvisor(TestCase):
     res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations,
services, {})
     self.assertEquals(res, res_expected)
 
+  # This test intentionally calls all validate methods with
+  # incorrect parameters (empty configs)
+  def test_noRiskyDictLookups(self):
+    properties = {}
+    recommendedDefaults = {}
+    configurations = {"core-site": {"properties": {}}}
+    services = {
+      "services": [],
+      "Versions": {
+        "stack_name": "HDP",
+        "stack_version": "2.3"
+      },
+      "configurations": configurations
+    }
+
+    hosts = {
+      "items" : [
+        {
+          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "disk_info" : [
+              {
+                "available" : "4564632",
+                "used" : "5230344",
+                "percent" : "54%",
+                "size" : "10319160",
+                "type" : "ext4",
+                "mountpoint" : "/"
+              },
+              {
+                "available" : "1832436",
+                "used" : "0",
+                "percent" : "0%",
+                "size" : "1832436",
+                "type" : "tmpfs",
+                "mountpoint" : "/dev/shm"
+              }
+            ],
+            "host_name" : "c6401.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6401.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        }
+      ]
+    }
+
+    def return_c6401_hostname(services, service_name, component_name):
+      return ["c6401.ambari.apache.org"]
+    self.stackAdvisor.getComponentHostNames = return_c6401_hostname
+
+    validators = self.stackAdvisor.getServiceConfigurationValidators()
+
+    # Setting up empty configs and services info
+    for serviceName, validator in validators.items():
+      services["services"].extend([{"StackServices": {"service_name": serviceName},
+                                    "components": []}])
+      for siteName in validator.keys():
+        configurations[siteName] = {"properties": {}}
+
+    # Emulate enabled RANGER
+    services["services"].extend([{"StackServices": {"service_name": "RANGER"},
+                                "components": []}])
+    configurations["ranger-hbase-plugin-properties"] = {
+      "ranger-hbase-plugin-enabled": "Yes"
+    }
+
+    recommendations = self.stackAdvisor.recommendConfigurations(services, hosts)
+
+    for serviceName, validator in validators.items():
+      recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"]
+      for siteName, method in validator.items():
+        if siteName in recommendedDefaults:
+          siteRecommendations = recommendedDefaults[siteName]["properties"]
+          method(properties, siteRecommendations, configurations, services, hosts)
+
+    pass


Mime
View raw message