ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-18671. Ranger KMS should add proxy users for yarn and livy (Mugdha Varadkar via smohanty)
Date Wed, 09 Nov 2016 21:09:42 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk f33dd9e38 -> 2a0c9414b


AMBARI-18671. Ranger KMS should add proxy users for yarn and livy (Mugdha Varadkar via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2a0c9414
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2a0c9414
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2a0c9414

Branch: refs/heads/trunk
Commit: 2a0c9414b4c1a6244ff3af29227072b6c9d6f43e
Parents: f33dd9e
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed Nov 9 13:06:32 2016 -0800
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed Nov 9 13:09:18 2016 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.3/services/stack_advisor.py    | 119 +++++++++++++++----
 .../stacks/HDP/2.5/services/stack_advisor.py    |  15 ++-
 .../stacks/2.0.6/common/test_stack_advisor.py   |  29 +++++
 .../stacks/2.3/common/test_stack_advisor.py     |  28 +++++
 .../stacks/2.5/common/test_stack_advisor.py     |   9 +-
 5 files changed, 164 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index d65248b..4e2a994 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -436,6 +436,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
     putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
     putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services)
+    security_enabled = self.isSecurityEnabled(services)
+    putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
+    putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
 
     if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']):
 
@@ -477,6 +480,95 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         default_fs = services['configurations']['core-site']['properties']['fs.defaultFS']
         putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit'))
 
+    required_services = [{'service' : 'YARN', 'config-type': 'yarn-env', 'property-name':
'yarn_user', 'proxy-category': ['hosts', 'users', 'groups']},
+    {'service' : 'SPARK', 'config-type': 'livy-env', 'property-name': 'livy_user', 'proxy-category':
['hosts', 'users', 'groups']}]
+
+    required_services_for_secure = [{'service' : 'HIVE', 'config-type': 'hive-env', 'property-name':
'hive_user', 'proxy-category': ['hosts', 'users']},
+    {'service' : 'OOZIE', 'config-type': 'oozie-env', 'property-name': 'oozie_user', 'proxy-category':
['hosts', 'users']}]
+
+    if security_enabled:
+      required_services.extend(required_services_for_secure)
+
+    # recommendations for kms proxy related properties
+    self.recommendKMSProxyUsers(configurations, services, hosts, required_services)
+
+    ambari_user = self.getAmbariUser(services)
+    if security_enabled:
+      # adding for ambari user
+      putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), '*')
+      putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), '*')
+      # adding for HTTP
+      putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.users', '*')
+      putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.hosts', '*')
+    else:
+      self.deleteKMSProxyUsers(configurations, services, hosts, required_services_for_secure)
+      # deleting ambari user proxy properties
+      putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user),
'delete', 'true')
+      putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(ambari_user),
'delete', 'true')
+      # deleting HTTP proxy properties
+      putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.hosts', 'delete', 'true')
+      putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.users', 'delete', 'true')
+
+  def recommendKMSProxyUsers(self, configurations, services, hosts, requiredServices):
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
+    putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+
+    if 'forced-configurations' not in services:
+      services["forced-configurations"] = []
+
+    for index in range(len(requiredServices)):
+      service = requiredServices[index]['service']
+      config_type = requiredServices[index]['config-type']
+      property_name = requiredServices[index]['property-name']
+      proxy_category = requiredServices[index]['proxy-category']
+
+      if service in servicesList:
+        if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']:
+          service_user = services['configurations'][config_type]['properties'][property_name]
+          service_old_user = getOldValue(self, services, config_type, property_name)
+
+          if 'groups' in proxy_category:
+            putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(service_user),
'*')
+          if 'hosts' in proxy_category:
+            putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(service_user),
'*')
+          if 'users' in proxy_category:
+            putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(service_user),
'*')
+
+          if service_old_user is not None and service_user != service_old_user:
+            if 'groups' in proxy_category:
+              putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_old_user),
'delete', 'true')
+              services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_old_user)})
+              services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_user)})
+            if 'hosts' in proxy_category:
+              putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_old_user),
'delete', 'true')
+              services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_old_user)})
+              services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_user)})
+            if 'users' in proxy_category:
+              putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_old_user),
'delete', 'true')
+              services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_old_user)})
+              services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_user)})
+
+  def deleteKMSProxyUsers(self, configurations, services, hosts, requiredServices):
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+
+    for index in range(len(requiredServices)):
+      service = requiredServices[index]['service']
+      config_type = requiredServices[index]['config-type']
+      property_name = requiredServices[index]['property-name']
+      proxy_category = requiredServices[index]['proxy-category']
+
+      if service in servicesList:
+        if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']:
+          service_user = services['configurations'][config_type]['properties'][property_name]
+
+          if 'groups' in proxy_category:
+            putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_user),
'delete', 'true')
+          if 'hosts' in proxy_category:
+            putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_user),
'delete', 'true')
+          if 'users' in proxy_category:
+            putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_user),
'delete', 'true')
 
   def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName):
     connection_string = self.getDBConnectionHostPort(db_type, db_host)
@@ -696,16 +788,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     else:
       putYarnSitePropertyAttributes('yarn.authorization-provider', 'delete', 'true')
 
-    if 'yarn-site' in services["configurations"] and 'yarn.resourcemanager.proxy-user-privileges.enabled'
in services["configurations"]["yarn-site"]["properties"]:
-      if self.isSecurityEnabled(services):
-        # enable proxy-user privileges for secure clusters for long-running services (spark
streaming etc)
-        putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'true')
-        if 'RANGER_KMS' in servicesList:
-          # disable proxy-user privileges on secure clusters as it does not work with TDE
-          putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false')
-      else:
-        putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false')
-
 
   def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts):
     putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services)
@@ -753,8 +835,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
                "hive-site": self.validateHiveConfigurations},
       "HBASE": {"hbase-site": self.validateHBASEConfigurations},
-      "KAFKA": {"kafka-broker": self.validateKAFKAConfigurations},
-      "YARN": {"yarn-site": self.validateYARNConfigurations},
+      "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
       "RANGER": {"admin-properties": self.validateRangerAdminConfigurations,
                  "ranger-env": self.validateRangerConfigurationsEnv}
     }
@@ -950,20 +1031,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
 
     return self.toConfigurationValidationProblems(validationItems, "kafka-broker")
 
-  def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services,
hosts):
-    yarn_site = properties
-    validationItems = []
-    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-    if 'RANGER_KMS' in servicesList and 'KERBEROS' in servicesList:
-      yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled']
-      if yarn_resource_proxy_enabled.lower() == 'true':
-        validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled',
-          "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled
" \
-          "property value as false under yarn-site"
-        )})
-
-    return self.toConfigurationValidationProblems(validationItems, "yarn-site")
-
   def isComponentUsingCardinalityForLayout(self, componentName):
     return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 9ceebb9..dfb7b0c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -1699,16 +1699,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
   def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor, self).recommendRangerKMSConfigurations(configurations, clusterData,
services, hosts)
-    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-    putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
 
-    if 'ranger-env' in services['configurations'] and 'ranger_user' in services['configurations']['ranger-env']['properties']:
-      rangerUser = services['configurations']['ranger-env']['properties']['ranger_user']
+    security_enabled = self.isSecurityEnabled(services)
+    required_services = [{'service' : 'RANGER', 'config-type': 'ranger-env', 'property-name':
'ranger_user', 'proxy-category': ['hosts', 'users', 'groups']}]
 
-      if 'kms-site' in services['configurations'] and 'KERBEROS' in servicesList:
-        putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(rangerUser), '*')
-        putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(rangerUser), '*')
-        putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(rangerUser), '*')
+    if security_enabled:
+      # recommendations for kms proxy related properties
+      self.recommendKMSProxyUsers(configurations, services, hosts, required_services)
+    else:
+      self.deleteKMSProxyUsers(configurations, services, hosts, required_services)
 
   def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData,
services, hosts)

http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 0a73c55..125c9ca 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2648,3 +2648,32 @@ class TestHDP206StackAdvisor(TestCase):
     self.assertFalse(recommendations['blueprint']['host_groups'][0]['components'])
     # Assert that DATANODE is placed on host-group-2
     self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][0]['name'],
'DATANODE')
+
+  def test_validateYARNConfigurations(self):
+    configurations = {
+      "cluster-env": {
+        "properties": {
+          "user_group": "hadoop",
+        }
+      },
+      "yarn-site": {
+        "properties": {
+          'yarn.nodemanager.resource.memory-mb' : '12288',
+          'yarn.scheduler.minimum-allocation-mb' : '3072',
+          'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+          'yarn.scheduler.maximum-allocation-mb': '12288'
+        }
+      }
+    }
+    services = {'configurations': {} }
+    recommendedDefaults = {'yarn.nodemanager.resource.memory-mb' : '12288',
+      'yarn.scheduler.minimum-allocation-mb' : '3072',
+      'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+      'yarn.scheduler.maximum-allocation-mb': '12288'}
+    properties = {'yarn.nodemanager.resource.memory-mb' : '12288',
+      'yarn.scheduler.minimum-allocation-mb' : '3072',
+      'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+      'yarn.scheduler.maximum-allocation-mb': '12288'}
+
+    res = self.stackAdvisor.validateYARNConfigurations(properties, recommendedDefaults, configurations,
services, {})
+    self.assertFalse(res)

http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index ad6842f..39cbbce 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1543,6 +1543,9 @@ class TestHDP23StackAdvisor(TestCase):
   def test_recommendRangerKMSConfigurations(self):
     clusterData = {}
     services = {
+      "ambari-server-properties": {
+        "ambari-server.user": "root"
+        },
       "Versions": {
         "stack_version" : "2.3",
         },
@@ -1583,6 +1586,11 @@ class TestHDP23StackAdvisor(TestCase):
             'db_host' : 'c6401.ambari.apache.org:1521:XE',
             'db_name' : "XE"
           }
+        },
+        'cluster-env': {
+          'properties': {
+            'security_enabled': 'false'
+          }
         }
       },
       "forced-configurations": []
@@ -1604,6 +1612,16 @@ class TestHDP23StackAdvisor(TestCase):
       'ranger-kms-audit': {
           'properties': {
           }
+      },
+      'kms-site': {
+        'properties': {
+        },
+        'property_attributes': {
+        'hadoop.kms.proxyuser.HTTP.hosts': {'delete': 'true'},
+        'hadoop.kms.proxyuser.HTTP.users': {'delete': 'true'},
+        'hadoop.kms.proxyuser.root.hosts': {'delete': 'true'},
+        'hadoop.kms.proxyuser.root.users': {'delete': 'true'}
+        }
       }
     }
 
@@ -1618,6 +1636,8 @@ class TestHDP23StackAdvisor(TestCase):
         "service_name": "KERBEROS"
       }
     })
+    services['configurations']['cluster-env']['properties']['security_enabled'] = "true"
+    services['configurations']['cluster-env']['properties']['ambari_principal_name'] = "ambari-cl1@EXAMPLE.COM"
 
     expected = {
       'kms-properties': {
@@ -1637,6 +1657,14 @@ class TestHDP23StackAdvisor(TestCase):
       'ranger-kms-audit': {
           'properties': {
           }
+      },
+      'kms-site': {
+        'properties': {
+        'hadoop.kms.proxyuser.HTTP.hosts': '*',
+        'hadoop.kms.proxyuser.HTTP.users': '*',
+        'hadoop.kms.proxyuser.ambari-cl1.hosts': '*',
+        'hadoop.kms.proxyuser.ambari-cl1.users': '*'
+        }
       }
     }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 6fe357a..7124307 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -446,9 +446,9 @@ class TestHDP25StackAdvisor(TestCase):
   def test_validateYarnConfigurations(self):
     properties = {'enable_hive_interactive': 'true',
                   'hive_server_interactive_host': 'c6401.ambari.apache.org',
-                  'hive.tez.container.size': '2048'}
+                  'hive.tez.container.size': '2048', "yarn.nodemanager.linux-container-executor.group":
"hadoop"}
     recommendedDefaults = {'enable_hive_interactive': 'true',
-                           "hive_server_interactive_host": "c6401.ambari.apache.org"}
+                           "hive_server_interactive_host": "c6401.ambari.apache.org", "yarn.nodemanager.linux-container-executor.group":
"hadoop"}
     configurations = {
       "hive-interactive-env": {
         "properties": {'enable_hive_interactive': 'true', "hive_server_interactive_host":
"c6401.ambari.apache.org"}
@@ -461,6 +461,11 @@ class TestHDP25StackAdvisor(TestCase):
       },
       "yarn-site": {
         "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "false"}
+      },
+      "cluster-env": {
+        "properties": {
+          "user_group": "hadoop",
+        }
       }
     }
     services = self.load_json("services-normal-his-valid.json")


Mime
View raw message