Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 461F7200B58 for ; Wed, 27 Jul 2016 22:21:10 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 44C3A160A93; Wed, 27 Jul 2016 20:21:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 20EE1160A6F for ; Wed, 27 Jul 2016 22:21:08 +0200 (CEST) Received: (qmail 18082 invoked by uid 500); 27 Jul 2016 20:21:08 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 18067 invoked by uid 99); 27 Jul 2016 20:21:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 27 Jul 2016 20:21:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 330C4E02A2; Wed, 27 Jul 2016 20:21:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jluniya@apache.org To: commits@ambari.apache.org Date: Wed, 27 Jul 2016 20:21:08 -0000 Message-Id: <5d2e74c0228949a6a522594e6ca421cb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] ambari git commit: AMBARI-17902: Config changes to support external solr and internal solr for Ranger (Mugdha Varadkar via jluniya) archived-at: Wed, 27 Jul 2016 20:21:10 -0000 Repository: ambari Updated Branches: refs/heads/trunk d6b861716 -> 567037bbf http://git-wip-us.apache.org/repos/asf/ambari/blob/567037bb/ambari-server/src/main/resources/stacks/HDP/2.5/services/KNOX/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/KNOX/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/KNOX/kerberos.json new file mode 100644 index 0000000..2d8aa0d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/KNOX/kerberos.json @@ -0,0 +1,81 @@ +{ + "services": [ + { + "name": "KNOX", + "components": [ + { + "name": "KNOX_GATEWAY", + "identities": [ + { + "name": "knox_principal", + "principal": { + "value": "${knox-env/knox_user}/_HOST@${realm}", + "type" : "service", + "configuration": "knox-env/knox_principal_name", + "local_username": "${knox-env/knox_user}" + + }, + "keytab": { + "file": "${keytab_dir}/knox.service.keytab", + "owner": { + "name": "${knox-env/knox_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "knox-env/knox_keytab_path" + } + }, + { + "name": "/KNOX/KNOX_GATEWAY/knox_principal", + "principal": { + "configuration": "ranger-knox-audit/xasecure.audit.jaas.Client.option.principal" + }, + "keytab": { + "configuration": "ranger-knox-audit/xasecure.audit.jaas.Client.option.keyTab" + } + } + ], + "configurations": [ + { + "gateway-site": { + "gateway.hadoop.kerberos.secured": "true", + "java.security.krb5.conf": "/etc/krb5.conf" + } + }, + { + "core-site": { + "hadoop.proxyuser.${knox-env/knox_user}.groups": "${hadoop-env/proxyuser_group}", + "hadoop.proxyuser.${knox-env/knox_user}.hosts": "${clusterHostInfo/knox_gateway_hosts}" + } + }, + { + "webhcat-site": { + "webhcat.proxyuser.${knox-env/knox_user}.groups": "${hadoop-env/proxyuser_group}", + "webhcat.proxyuser.${knox-env/knox_user}.hosts": "${clusterHostInfo/knox_gateway_hosts}" + } + }, + { + "oozie-site": { + "oozie.service.ProxyUserService.proxyuser.${knox-env/knox_user}.groups": "${hadoop-env/proxyuser_group}", + "oozie.service.ProxyUserService.proxyuser.${knox-env/knox_user}.hosts": "${clusterHostInfo/knox_gateway_hosts}" + } + }, + { + "ranger-knox-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/567037bb/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/ranger-kms-audit.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/ranger-kms-audit.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/ranger-kms-audit.xml index 8c8278a..86b963b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/ranger-kms-audit.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/ranger-kms-audit.xml @@ -82,139 +82,4 @@ - - xasecure.audit.jaas.Client.option.principal - {{rangerkms_principal}} - - - true - - - - - - xasecure.audit.jaas.Client.option.keyTab - {{rangerkms_keytab}} - - - true - - - - - - xasecure.audit.jaas.Client.loginModuleName - - - - - ranger-admin-site - xasecure.audit.jaas.Client.loginModuleName - - - ranger-admin-site - ranger.is.solr.kerberised - - - - true - - - - - - xasecure.audit.jaas.Client.loginModuleControlFlag - - - - - ranger-admin-site - xasecure.audit.jaas.Client.loginModuleControlFlag - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - true - - - - - xasecure.audit.jaas.Client.option.useKeyTab - false - - - boolean - - - - ranger-admin-site - xasecure.audit.jaas.Client.option.useKeyTab - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - - - xasecure.audit.jaas.Client.option.storeKey - false - - - boolean - - - - ranger-admin-site - xasecure.audit.jaas.Client.option.storeKey - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - - - xasecure.audit.jaas.Client.option.serviceName - - - - - ranger-admin-site - xasecure.audit.jaas.Client.option.serviceName - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - true - - - - - xasecure.audit.destination.solr.force.use.inmemory.jaas.config - false - - - boolean - - - - ranger-admin-site - ranger.is.solr.kerberised - - - - http://git-wip-us.apache.org/repos/asf/ambari/blob/567037bb/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json index bfd142a..a54783e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json @@ -22,6 +22,16 @@ "hadoop.kms.authentication.type": "kerberos", "hadoop.kms.authentication.kerberos.principal": "*" } + }, + { + "ranger-kms-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } } ], "components": [ @@ -56,6 +66,15 @@ }, "configuration": "dbks-site/ranger.ks.kerberos.keytab" } + }, + { + "name": "/RANGER_KMS/RANGER_KMS_SERVER/rangerkms", + "principal": { + "configuration": "ranger-kms-audit/xasecure.audit.jaas.Client.option.principal" + }, + "keytab": { + "configuration": "ranger-kms-audit/xasecure.audit.jaas.Client.option.keyTab" + } } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/567037bb/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/ranger-yarn-audit.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/ranger-yarn-audit.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/ranger-yarn-audit.xml index da24576..fff9132 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/ranger-yarn-audit.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/ranger-yarn-audit.xml @@ -55,139 +55,4 @@ - - xasecure.audit.jaas.Client.option.principal - {{rm_principal_name}} - - - true - - - - - - xasecure.audit.jaas.Client.option.keyTab - {{rm_keytab}} - - - true - - - - - - xasecure.audit.jaas.Client.loginModuleName - - - - - ranger-admin-site - xasecure.audit.jaas.Client.loginModuleName - - - ranger-admin-site - ranger.is.solr.kerberised - - - - true - - - - - - xasecure.audit.jaas.Client.loginModuleControlFlag - - - - - ranger-admin-site - xasecure.audit.jaas.Client.loginModuleControlFlag - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - true - - - - - xasecure.audit.jaas.Client.option.useKeyTab - false - - - boolean - - - - ranger-admin-site - xasecure.audit.jaas.Client.option.useKeyTab - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - - - xasecure.audit.jaas.Client.option.storeKey - false - - - boolean - - - - ranger-admin-site - xasecure.audit.jaas.Client.option.storeKey - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - - - xasecure.audit.jaas.Client.option.serviceName - - - - - ranger-admin-site - xasecure.audit.jaas.Client.option.serviceName - - - ranger-admin-site - ranger.is.solr.kerberised - - - - - true - - - - - xasecure.audit.destination.solr.force.use.inmemory.jaas.config - false - - - boolean - - - - ranger-admin-site - ranger.is.solr.kerberised - - - - http://git-wip-us.apache.org/repos/asf/ambari/blob/567037bb/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json index 38896f5..e690204 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json @@ -49,6 +49,16 @@ "yarn.scheduler.capacity.root.default.acl_administer_jobs": "${yarn-env/yarn_user}", "yarn.scheduler.capacity.root.default.acl_submit_applications": "${yarn-env/yarn_user}" } + }, + { + "ranger-yarn-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } } ], "components": [ @@ -161,6 +171,15 @@ "keytab": { "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file" } + }, + { + "name": "/YARN/RESOURCEMANAGER/resource_manager_rm", + "principal": { + "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.principal" + }, + "keytab": { + "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.keyTab" + } } ] }, http://git-wip-us.apache.org/repos/asf/ambari/blob/567037bb/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 2028db0..fc9bd94 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -110,8 +110,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): "HIVE": {"hive-interactive-env": self.validateHiveInteractiveEnvConfigurations, "hive-interactive-site": self.validateHiveInteractiveSiteConfigurations}, "YARN": {"yarn-site": self.validateYarnConfigurations}, - "RANGER": {"ranger-tagsync-site": self.validateRangerTagsyncConfigurations, - "ranger-admin-site": self.validateRangerAdminConfigurations}, + "RANGER": {"ranger-tagsync-site": self.validateRangerTagsyncConfigurations}, "SPARK2": {"spark2-defaults": self.validateSpark2Defaults, "spark2-thrift-sparkconf": self.validateSpark2ThriftSparkConf} } @@ -1725,14 +1724,18 @@ class HDP25StackAdvisor(HDP24StackAdvisor): final_kafka_host = ",".join(kafka_host_port) putTagsyncAppProperty('atlas.kafka.bootstrap.servers', final_kafka_host) - if 'ranger-env' in services['configurations'] and 'is_solrCloud_enabled' in services['configurations']["ranger-env"]["properties"]: - isSolrCloudEnabled = services['configurations']["ranger-env"]["properties"]["is_solrCloud_enabled"] == "true" - else: - isSolrCloudEnabled = False + is_solr_cloud_enabled = False + if 'ranger-env' in services['configurations'] and 'is_solrCloud_enabled' in services['configurations']['ranger-env']['properties']: + is_solr_cloud_enabled = services['configurations']['ranger-env']['properties']['is_solrCloud_enabled'] == 'true' + + is_external_solr_cloud_enabled = False + if 'ranger-env' in services['configurations'] and 'is_external_solrCloud_enabled' in services['configurations']['ranger-env']['properties']: + is_external_solr_cloud_enabled = services['configurations']['ranger-env']['properties']['is_external_solrCloud_enabled'] == 'true' ranger_audit_zk_port = '' - if 'LOGSEARCH' in servicesList and zookeeper_host_port and isSolrCloudEnabled: + #TODO to change check for LOGSEARCH after implemenation of AMBARI-17822 + if 'LOGSEARCH' in servicesList and zookeeper_host_port and is_solr_cloud_enabled and not is_external_solr_cloud_enabled: zookeeper_host_port = zookeeper_host_port.split(',') zookeeper_host_port.sort() zookeeper_host_port = ",".join(zookeeper_host_port) @@ -1743,17 +1746,12 @@ class HDP25StackAdvisor(HDP24StackAdvisor): logsearch_solr_znode = services['configurations']['logsearch-solr-env']['properties']['logsearch_solr_znode'] ranger_audit_zk_port = '{0}{1}'.format(zookeeper_host_port, logsearch_solr_znode) putRangerAdminProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port) - elif zookeeper_host_port and isSolrCloudEnabled: + elif zookeeper_host_port and is_solr_cloud_enabled and is_external_solr_cloud_enabled: ranger_audit_zk_port = '{0}/{1}'.format(zookeeper_host_port, 'ranger_audits') putRangerAdminProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port) else: putRangerAdminProperty('ranger.audit.solr.zookeepers', 'NONE') - if 'ranger-admin-site' in services['configurations'] and 'ranger.is.solr.kerberised' in services['configurations']['ranger-admin-site']['properties']: - is_solr_kerberised = services['configurations']['ranger-admin-site']['properties']['ranger.is.solr.kerberised'] == 'true' - else: - is_solr_kerberised = False - ranger_services = [ {'service_name': 'HDFS', 'audit_file': 'ranger-hdfs-audit'}, {'service_name': 'YARN', 'audit_file': 'ranger-yarn-audit'}, @@ -1784,37 +1782,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor): rangerAuditProperty = services["configurations"][item['filename']]["properties"][item['configname']] putRangerAuditProperty(item['target_configname'], rangerAuditProperty) - if is_solr_kerberised: - ranger_solr_kerberised = [ - {'configname': 'xasecure.audit.jaas.Client.loginModuleName'}, - {'configname': 'xasecure.audit.jaas.Client.loginModuleControlFlag'}, - {'configname': 'xasecure.audit.jaas.Client.option.useKeyTab'}, - {'configname': 'xasecure.audit.jaas.Client.option.storeKey'}, - {'configname': 'xasecure.audit.jaas.Client.option.serviceName'} - ] - - for item in ranger_solr_kerberised: - if 'ranger-admin-site' in services['configurations'] and item['configname'] in services["configurations"]['ranger-admin-site']["properties"]: - if 'ranger-admin-site' in configurations and item['configname'] in configurations['ranger-admin-site']["properties"]: - solrKerberisedProperty = configurations['ranger-admin-site']["properties"][item['configname']] - else: - solrKerberisedProperty = services['configurations']['ranger-admin-site']['properties'][item['configname']] - putRangerAuditProperty(item['configname'], solrKerberisedProperty) - - putRangerAuditProperty('xasecure.audit.destination.solr.force.use.inmemory.jaas.config', 'true') - else: - set_solr_kerberised_default = [ - {'configname': 'xasecure.audit.jaas.Client.loginModuleName', 'default_value': ''}, - {'configname': 'xasecure.audit.jaas.Client.loginModuleControlFlag', 'default_value': ''}, - {'configname': 'xasecure.audit.jaas.Client.option.useKeyTab', 'default_value': 'false'}, - {'configname': 'xasecure.audit.jaas.Client.option.storeKey', 'default_value': 'false'}, - {'configname': 'xasecure.audit.jaas.Client.option.serviceName', 'default_value': ''}, - {'configname': 'xasecure.audit.destination.solr.force.use.inmemory.jaas.config', 'default_value': 'false'} - ] - - for item in set_solr_kerberised_default: - putRangerAuditProperty(item['configname'], item['default_value']) - if "HDFS" in servicesList: hdfs_user = None if "hadoop-env" in services["configurations"] and "hdfs_user" in services["configurations"]["hadoop-env"]["properties"]: @@ -1888,18 +1855,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "ranger-tagsync-site") - def validateRangerAdminConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - ranger_admin_properties = properties - validationItems = [] - security_enabled = self.isSecurityEnabled(services) - - if 'ranger.is.solr.kerberised' in ranger_admin_properties and ranger_admin_properties['ranger.is.solr.kerberised'].lower() == 'true'\ - and not security_enabled: - validationItems.append({"config-name": "ranger.is.solr.kerberised", - "item": self.getWarnItem("Kerberos Solr (ranger.is.solr.kerberised) should not be enabled in non-kerberos environment.")}) - - return self.toConfigurationValidationProblems(validationItems, "ranger-admin-site") - def __isServiceDeployed(self, services, serviceName): servicesList = [service["StackServices"]["service_name"] for service in services["services"]] return serviceName in servicesList http://git-wip-us.apache.org/repos/asf/ambari/blob/567037bb/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py index 2345b8e..e39e1cd 100644 --- a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py +++ b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py @@ -176,8 +176,8 @@ class TestRangerAdmin(RMFTestCase): group = 'hadoop', mode = 0664, ) - self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger_solr_jass.conf', - content = Template('ranger_solr_jass_conf.j2'), + self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger_solr_jaas.conf', + content = Template('ranger_solr_jaas_conf.j2'), owner = 'ranger', ) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /ambari-solr --check-znode --retry 5 --interval 10')