ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dmitriu...@apache.org
Subject [1/2] ambari git commit: AMBARI-19296. When adding service, custom core-site properties get reverted back (echekanskiy via dlisnichenko)
Date Wed, 28 Dec 2016 14:59:00 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 cf28b98aa -> ebaa0ea79
  refs/heads/trunk 1bfffc0c8 -> d84aa3bcb


AMBARI-19296. When adding service, custom core-site properties get reverted back (echekanskiy
via dlisnichenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ebaa0ea7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ebaa0ea7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ebaa0ea7

Branch: refs/heads/branch-2.5
Commit: ebaa0ea79c20702e2e03e1cdf331f8816515fecc
Parents: cf28b98
Author: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Authored: Wed Dec 28 16:57:35 2016 +0200
Committer: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Committed: Wed Dec 28 16:57:35 2016 +0200

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 86 ++++++++++++++------
 .../stacks/HDP/2.1/services/stack_advisor.py    | 20 +----
 .../stacks/HDP/2.3/services/stack_advisor.py    |  2 +-
 .../stacks/HDP/2.5/services/stack_advisor.py    | 20 +----
 .../stacks/2.0.6/common/test_stack_advisor.py   | 10 +--
 5 files changed, 67 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ebaa0ea7/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index a98be40..9c0b357 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -200,12 +200,23 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
          ambari_user = ambari_user.split('@')[0]
     return ambari_user
 
-  def get_hosts_for_proxyuser(self, user_name, services):
+  def get_data_for_proxyuser(self, user_name, services, groups=False):
+    """
+    Returns values of proxyuser properties for given user. Properties can be
+    hadoop.proxyuser.username.groups or hadoop.proxyuser.username.hosts
+    :param user_name:
+    :param services:
+    :param groups: if true, will return values for group property, not hosts
+    :return: tuple (wildcard_value, set[values]), where wildcard_value indicates if property
value was *
+    """
     if "core-site" in services["configurations"]:
       coreSite = services["configurations"]["core-site"]['properties']
     else:
       coreSite = {}
-    property_name = "hadoop.proxyuser.{0}.hosts".format(user_name)
+    if groups:
+      property_name = "hadoop.proxyuser.{0}.groups".format(user_name)
+    else:
+      property_name = "hadoop.proxyuser.{0}.hosts".format(user_name)
     if property_name in coreSite:
       property_value = coreSite[property_name]
       if property_value == "*":
@@ -214,15 +225,40 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         return False, set(property_value.split(","))
     return False, set()
 
+  def put_proxyuser_value(self, user_name, value, is_groups=False, services=None, put_function=None):
+    is_wildcard_value, current_value = self.get_data_for_proxyuser(user_name, services, is_groups)
+    result_value = "*"
+    result_values_set = self.merge_proxyusers_values(current_value, value)
+    if len(result_values_set) > 0:
+      result_value = ",".join(sorted([val for val in result_values_set if val]))
+
+    if is_groups:
+      property_name = "hadoop.proxyuser.{0}.groups".format(user_name)
+    else:
+      property_name = "hadoop.proxyuser.{0}.hosts".format(user_name)
+
+    put_function(property_name, result_value)
+
+  def merge_proxyusers_values(self, first, second):
+    result = set()
+    def append(data):
+      if isinstance(data, str) or isinstance(data, unicode):
+        if data != "*":
+          result.update(data.split(","))
+      else:
+        result.update(data)
+    append(first)
+    append(second)
+    return result
+
   def recommendAmbariProxyUsersForHDFS(self, services, servicesList, putCoreSiteProperty,
putCoreSitePropertyAttribute):
       if "HDFS" in servicesList:
           ambari_user = self.getAmbariUser(services)
           ambariHostName = socket.getfqdn()
-          is_wildcard_value, hosts = self.get_hosts_for_proxyuser(ambari_user, services)
-          if not is_wildcard_value:
-            hosts.add(ambariHostName)
-            putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(ambari_user), ",".join(hosts))
-          putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(ambari_user), "*")
+
+          self.put_proxyuser_value(ambari_user,ambariHostName,services=services,put_function=putCoreSiteProperty)
+          self.put_proxyuser_value(ambari_user, "*", is_groups=True, services=services, put_function=putCoreSiteProperty)
+
           old_ambari_user = self.getOldAmbariUser(services)
           if old_ambari_user is not None:
             putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(old_ambari_user),
'delete', 'true')
@@ -246,7 +282,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       oozie_user = None
       if "oozie-env" in services["configurations"] and "oozie_user" in services["configurations"]["oozie-env"]["properties"]:
         oozie_user = services["configurations"]["oozie-env"]["properties"]["oozie_user"]
-        _, oozieServerHostsNameSet = self.get_hosts_for_proxyuser(oozie_user, services)
+        oozieServerHostsNameSet = set()
         oozieServerHosts = self.getHostsWithComponent("OOZIE", "OOZIE_SERVER", services,
hosts)
         if oozieServerHosts is not None:
           for oozieServerHost in oozieServerHosts:
@@ -267,27 +303,27 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         webHcatServerHosts = self.getHostsWithComponent("HIVE", "WEBHCAT_SERVER", services,
hosts)
 
         if hiveServerHosts is not None:
-          _, hiveServerHostsNameList = self.get_hosts_for_proxyuser(hive_user, services)
+          hiveServerHostsNameSet = set()
           for hiveServerHost in hiveServerHosts:
-            hiveServerHostsNameList.add(hiveServerHost["Hosts"]["host_name"])
+            hiveServerHostsNameSet.add(hiveServerHost["Hosts"]["host_name"])
           # Append Hive Server Interactive host as well, as it is Hive2/HiveServer2 component.
           if hiveServerInteractiveHosts:
             for hiveServerInteractiveHost in hiveServerInteractiveHosts:
               hiveServerInteractiveHostName = hiveServerInteractiveHost["Hosts"]["host_name"]
-              if hiveServerInteractiveHostName not in hiveServerHostsNameList:
-                hiveServerHostsNameList.add(hiveServerInteractiveHostName)
-                Logger.info("Appended (if not exiting), Hive Server Interactive Host : '{0}',
to Hive Server Host List : '{1}'".format(hiveServerInteractiveHostName, hiveServerHostsNameList))
+              if hiveServerInteractiveHostName not in hiveServerHostsNameSet:
+                hiveServerHostsNameSet.add(hiveServerInteractiveHostName)
+                Logger.info("Appended (if not exiting), Hive Server Interactive Host : '{0}',
to Hive Server Host List : '{1}'".format(hiveServerInteractiveHostName, hiveServerHostsNameSet))
 
-          hiveServerHostsNames = ",".join(sorted(hiveServerHostsNameList))  # includes Hive
Server interactive host also.
-          Logger.info("Hive Server and Hive Server Interactive (if enabled) Host List : {0}".format(hiveServerHostsNameList))
+          hiveServerHostsNames = ",".join(sorted(hiveServerHostsNameSet))  # includes Hive
Server interactive host also.
+          Logger.info("Hive Server and Hive Server Interactive (if enabled) Host List : {0}".format(hiveServerHostsNameSet))
           if not hive_user in users and hive_user is not None:
             users[hive_user] = {"propertyHosts" : hiveServerHostsNames,"propertyGroups" :
"*", "config" : "hive-env", "propertyName" : "hive_user"}
 
         if webHcatServerHosts is not None:
-          _, webHcatServerHostsNameList = self.get_hosts_for_proxyuser(hive_user, services)
+          webHcatServerHostsNameSet = set()
           for webHcatServerHost in webHcatServerHosts:
-            webHcatServerHostsNameList.add(webHcatServerHost["Hosts"]["host_name"])
-          webHcatServerHostsNames = ",".join(sorted(webHcatServerHostsNameList))
+            webHcatServerHostsNameSet.add(webHcatServerHost["Hosts"]["host_name"])
+          webHcatServerHostsNames = ",".join(sorted(webHcatServerHostsNameSet))
           if not webhcat_user in users and webhcat_user is not None:
             users[webhcat_user] = {"propertyHosts" : webHcatServerHostsNames,"propertyGroups"
: "*", "config" : "hive-env", "propertyName" : "webhcat_user"}
 
@@ -298,10 +334,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         rmHosts = self.getHostsWithComponent("YARN", "RESOURCEMANAGER", services, hosts)
 
         if len(rmHosts) > 1:
-          _, rmHostsNameList = self.get_hosts_for_proxyuser(hive_user, services)
+          rmHostsNameSet = set()
           for rmHost in rmHosts:
-            rmHostsNameList.add(rmHost["Hosts"]["host_name"])
-          rmHostsNames = ",".join(sorted(rmHostsNameList))
+            rmHostsNameSet.add(rmHost["Hosts"]["host_name"])
+          rmHostsNames = ",".join(sorted(rmHostsNameSet))
           if not yarn_user in users and yarn_user is not None:
             users[yarn_user] = {"propertyHosts" : rmHostsNames, "config" : "yarn-env", "propertyName"
: "yarn_user"}
 
@@ -324,16 +360,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
 
     for user_name, user_properties in users.iteritems():
-      is_wildcard_value, _ = self.get_hosts_for_proxyuser(user_name, services)
       if hive_user and hive_user == user_name:
         if "propertyHosts" in user_properties:
           services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(hive_user)})
       # Add properties "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" to core-site
for all users
-      if not is_wildcard_value:
-        putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(user_name) , user_properties["propertyHosts"])
-        Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(hive_user, user_properties["propertyHosts"]))
+      self.put_proxyuser_value(user_name, user_properties["propertyHosts"], services=services,
put_function=putCoreSiteProperty)
+      Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(hive_user, user_properties["propertyHosts"]))
       if "propertyGroups" in user_properties:
-        putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(user_name) , user_properties["propertyGroups"])
+        self.put_proxyuser_value(user_name, user_properties["propertyGroups"], is_groups=True,
services=services, put_function=putCoreSiteProperty)
 
       # Remove old properties if user was renamed
       userOldValue = getOldValue(self, services, user_properties["config"], user_properties["propertyName"])

http://git-wip-us.apache.org/repos/asf/ambari/blob/ebaa0ea7/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 17225d0..f34f5a1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -139,27 +139,11 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
             webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user),
'delete', 'true')
 
     if self.is_secured_cluster(services):
-      appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
-
-      def updateCallback(originalValue, newValue):
-        """
-        :type originalValue str
-        :type newValue list
-        """
-        if originalValue and not originalValue.isspace():
-          hosts = originalValue.split(',')
-
-          if newValue:
-            hosts.extend(newValue)
-
-          result = ','.join(set(hosts))
-          return result
-        else:
-          return ','.join(set(newValue))
+      putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
 
       meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services)
       if "hostnames" in meta:
-        appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback)
+        self.put_proxyuser_value("HTTP", meta["hostnames"], services=services, put_function=putCoreSiteProperty)
 
   def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
     putTezProperty = self.putProperty(configurations, "tez-site")

http://git-wip-us.apache.org/repos/asf/ambari/blob/ebaa0ea7/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index cc53ae5..e09ffde 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -485,7 +485,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS'
in servicesList:
       kmsUser = kmsEnvProperties['kms_user']
       kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user')
-      putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*')
+      self.put_proxyuser_value(kmsUser, '*', is_groups=True, services=services, put_function=putCoreSiteProperty)
       if kmsUserOld is not None and kmsUser != kmsUserOld:
         putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete',
'true')
         services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)})

http://git-wip-us.apache.org/repos/asf/ambari/blob/ebaa0ea7/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index a130566..b52d753 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -618,7 +618,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
   def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData,
services, hosts)
     putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
-    appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
+    putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
 
     if "cluster-env" in services["configurations"] \
          and "security_enabled" in services["configurations"]["cluster-env"]["properties"]
\
@@ -632,24 +632,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
         Logger.debug("Attempting to update hadoop.proxyuser.HTTP.hosts with %s" % str(phoenix_query_server_hosts))
         # The PQS hosts we want to ensure are set
         new_value = ','.join(phoenix_query_server_hosts)
-        # Compute the unique set of hosts for the property
-        def updateCallback(originalValue, newValue):
-          Logger.debug("Original hadoop.proxyuser.HTTP.hosts value %s, appending %s" % (originalValue,
newValue))
-          # Only update the original value if it's not whitespace only
-          if originalValue and not originalValue.isspace():
-            hosts = originalValue.split(',')
-            # Add in the new hosts if we have some
-            if newValue and not newValue.isspace():
-              hosts.extend(newValue.split(','))
-            # Return the combined (uniqued) list of hosts
-            result = ','.join(set(hosts))
-            Logger.debug("Setting final to %s" % result)
-            return result
-          else:
-            Logger.debug("Setting final value to %s" % newValue)
-            return newValue
         # Update the proxyuser setting, deferring to out callback to merge results together
-        appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', new_value, updateCallback)
+        self.put_proxyuser_value("HTTP", new_value, services=services, put_function=putCoreSiteProperty)
       else:
         Logger.debug("No phoenix query server hosts to update")
     else:

http://git-wip-us.apache.org/repos/asf/ambari/blob/ebaa0ea7/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 4fa9e02..c88e929 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1318,7 +1318,7 @@ class TestHDP206StackAdvisor(TestCase):
                       'hadoop.proxyuser.ambari_user.hosts': ambariHostName,
                       'hadoop.proxyuser.oozie.groups': '*',
                       'hadoop.proxyuser.hive.groups': '*',
-                      'hadoop.proxyuser.webhcat.hosts': 'c6401.ambari.apache.org,c6402.ambari.apache.org,c6403.ambari.apache.org',
+                      'hadoop.proxyuser.webhcat.hosts': 'c6401.ambari.apache.org,c6402.ambari.apache.org',
                       'hadoop.proxyuser.falcon.hosts': '*',
                       'hadoop.proxyuser.webhcat.groups': '*',
                       'hadoop.proxyuser.hdfs.groups': '*',
@@ -1352,12 +1352,6 @@ class TestHDP206StackAdvisor(TestCase):
     self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services2,
hosts)
     self.assertEquals(configurations, expected)
 
-
-
-
-
-
-
     configurations["hadoop-env"]["properties"]['hdfs_user'] = "hdfs1"
 
     changedConfigurations = [{"type":"hadoop-env",
@@ -1448,7 +1442,7 @@ class TestHDP206StackAdvisor(TestCase):
                                  'hadoop.proxyuser.hive.groups': '*',
                                  'hadoop.proxyuser.hdfs1.groups': '*',
                                  'hadoop.proxyuser.hdfs1.hosts': '*',
-                                 'hadoop.proxyuser.webhcat.hosts': 'c6401.ambari.apache.org,c6402.ambari.apache.org,c6403.ambari.apache.org',
+                                 'hadoop.proxyuser.webhcat.hosts': 'c6401.ambari.apache.org,c6402.ambari.apache.org',
                                  'hadoop.proxyuser.falcon.hosts': '*',
                                  'hadoop.proxyuser.webhcat.groups': '*',
                                  'hadoop.proxyuser.hdfs.groups': '*',


Mime
View raw message