Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9064618E2F for ; Tue, 1 Dec 2015 18:20:25 +0000 (UTC) Received: (qmail 70586 invoked by uid 500); 1 Dec 2015 18:19:51 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 70058 invoked by uid 500); 1 Dec 2015 18:19:51 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 70030 invoked by uid 99); 1 Dec 2015 18:19:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Dec 2015 18:19:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0C379E0A02; Tue, 1 Dec 2015 18:19:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Tue, 01 Dec 2015 18:19:52 -0000 Message-Id: <4b127b233a8c4c8aa3e551dcd6f43f0b@git.apache.org> In-Reply-To: <04c3a98999a345bbb6caa96a2ba0cd9d@git.apache.org> References: <04c3a98999a345bbb6caa96a2ba0cd9d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/50] ambari git commit: AMBARI-14065. Ranger audit to HDFS - Create prerequisite directories in HDFS AMBARI-14065. Ranger audit to HDFS - Create prerequisite directories in HDFS Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c3cf499 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c3cf499 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c3cf499 Branch: refs/heads/branch-dev-patch-upgrade Commit: 6c3cf4993e520c2bfc60707fa54e1aa0783bd557 Parents: db2ca77 Author: Gautam Borad Authored: Thu Nov 26 12:59:46 2015 +0530 Committer: Gautam Borad Committed: Fri Nov 27 11:49:12 2015 +0530 ---------------------------------------------------------------------- .../0.96.0.2.0/package/scripts/params_linux.py | 1 + .../package/scripts/setup_ranger_hbase.py | 27 +++++++++++++++ .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 3 +- .../2.1.0.2.0/package/scripts/params_linux.py | 1 + .../package/scripts/setup_ranger_hdfs.py | 29 +++++++++++++++- .../0.12.0.2.0/package/scripts/params_linux.py | 1 + .../package/scripts/setup_ranger_hive.py | 19 +++++++++++ .../KAFKA/0.8.1.2.2/package/scripts/params.py | 33 +++++++++++++++++- .../package/scripts/setup_ranger_kafka.py | 20 +++++++++++ .../0.5.0.2.2/package/scripts/params_linux.py | 28 +++++++++++++++ .../package/scripts/setup_ranger_knox.py | 20 +++++++++++ .../0.9.1.2.1/package/scripts/params_linux.py | 36 ++++++++++++++++++-- .../package/scripts/setup_ranger_storm.py | 20 +++++++++++ .../2.1.0.2.0/package/scripts/params_linux.py | 1 + .../package/scripts/setup_ranger_yarn.py | 19 +++++++++++ .../stacks/HDP/2.3/role_command_order.json | 6 +++- 16 files changed, 257 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 7dee23b..a05abd4 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -326,6 +326,7 @@ if has_ranger_admin: ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py index 1d1be6c..5c68583 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py @@ -39,6 +39,33 @@ def setup_ranger_hbase(upgrade_type=None): else: Logger.info("HBase: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + if params.xml_configurations_supported and params.enable_ranger_hbase and params.xa_audit_hdfs_is_enabled: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/hbaseMaster", + type="directory", + action="create_on_execute", + owner=params.hbase_user, + group=params.hbase_user, + mode=0700, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/hbaseRegional", + type="directory", + action="create_on_execute", + owner=params.hbase_user, + group=params.hbase_user, + mode=0700, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + setup_ranger_plugin('hbase-client', 'hbase', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index 44119ab..0902637 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -38,7 +38,7 @@ from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from utils import service, safe_zkfc_op, is_previous_fs_image -from setup_ranger_hdfs import setup_ranger_hdfs +from setup_ranger_hdfs import setup_ranger_hdfs, create_ranger_audit_hdfs_directories @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) @@ -177,6 +177,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, e # Always run this on non-HA, or active NameNode during HA. create_hdfs_directories(is_active_namenode_cmd) + create_ranger_audit_hdfs_directories(is_active_namenode_cmd) elif action == "stop": import params http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 587306b..b67a4ae 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -469,6 +469,7 @@ if has_ranger_admin: ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py index bd158ec..622dcba 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py @@ -35,7 +35,7 @@ def setup_ranger_hdfs(upgrade_type=None): hdp_version = params.version if params.retryAble: - Logger.info("HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !") + Logger.info("HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !") else: Logger.info("HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") @@ -56,3 +56,30 @@ def setup_ranger_hdfs(upgrade_type=None): hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble) else: Logger.info('Ranger admin not installed') + +def create_ranger_audit_hdfs_directories(check): + import params + + if params.has_ranger_admin: + if params.xml_configurations_supported and params.enable_ranger_hdfs and params.xa_audit_hdfs_is_enabled: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True, + only_if=check + ) + params.HdfsResource("/ranger/audit/hdfs", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0700, + recursive_chmod=True, + only_if=check + ) + params.HdfsResource(None, action="execute", only_if=check) + else: + Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index f360651..a2131b0 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -555,6 +555,7 @@ if has_ranger_admin: xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py index c17def0..8b2e4e4 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py @@ -39,6 +39,25 @@ def setup_ranger_hive(upgrade_type = None): else: Logger.info("Hive: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + if params.xml_configurations_supported and params.enable_ranger_hive and params.xa_audit_hdfs_is_enabled: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/hiveServer2", + type="directory", + action="create_on_execute", + owner=params.hive_user, + group=params.hive_user, + mode=0700, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + setup_ranger_plugin('hive-server2', 'hive', params.ranger_downloaded_custom_connector, params.ranger_driver_curl_source, params.ranger_driver_curl_target, params.java64_home, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py index bd4fa6c..da76952 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py @@ -24,9 +24,12 @@ from resource_management.libraries.functions.default import default from utils import get_bare_principal from resource_management.libraries.functions.get_hdp_version import get_hdp_version from resource_management.libraries.functions.is_empty import is_empty - import status_params from resource_management.core.logger import Logger +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import get_kinit_path # server configurations @@ -232,6 +235,7 @@ if has_ranger_admin and is_supported_kafka_ranger: ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None + xa_audit_hdfs_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None @@ -244,3 +248,30 @@ if has_ranger_admin and is_supported_kafka_ranger: if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False +namenode_hosts = default("/clusterHostInfo/namenode_host", []) +has_namenode = not len(namenode_hosts) == 0 + +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None +hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None +default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None +hadoop_bin_dir = hdp_select.get_hadoop_dir("bin") if has_namenode else None +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py index c210791..a99dc76 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py @@ -30,6 +30,26 @@ def setup_ranger_kafka(): else: Logger.info("Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + if params.xml_configurations_supported and params.enable_ranger_kafka and params.xa_audit_hdfs_is_enabled: + if params.has_namenode: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/kafka", + type="directory", + action="create_on_execute", + owner=params.kafka_user, + group=params.kafka_user, + mode=0700, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + setup_ranger_plugin('kafka-broker', 'kafka', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index c723de9..ec972f6 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -28,6 +28,9 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script from status_params import * +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions import conf_select # server configurations config = Script.get_config() @@ -318,6 +321,7 @@ if has_ranger_admin: ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None + xa_audit_hdfs_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None @@ -325,3 +329,27 @@ if has_ranger_admin: #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False + +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None +hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None +default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None +hadoop_bin_dir = hdp_select.get_hadoop_dir("bin") if has_namenode else None +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py index 8ea1427..1a08d54 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py @@ -38,6 +38,26 @@ def setup_ranger_knox(upgrade_type=None): else: Logger.info("Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + if params.xml_configurations_supported and params.enable_ranger_knox and params.xa_audit_hdfs_is_enabled: + if params.has_namenode: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/knox", + type="directory", + action="create_on_execute", + owner=params.knox_user, + group=params.knox_user, + mode=0700, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + setup_ranger_plugin('knox-server', 'knox', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java_home, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py index f5d944c..f186a89 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py @@ -29,9 +29,10 @@ from resource_management.libraries.functions.version import format_hdp_stack_ver from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_bare_principal import get_bare_principal from resource_management.libraries.script import Script - - - +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import get_kinit_path # server configurations config = Script.get_config() @@ -260,6 +261,7 @@ if has_ranger_admin: ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-storm-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None + xa_audit_hdfs_is_enabled = config['configurations']['ranger-storm-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None @@ -267,3 +269,31 @@ if has_ranger_admin: #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False + +namenode_hosts = default("/clusterHostInfo/namenode_host", []) +has_namenode = not len(namenode_hosts) == 0 + +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None +hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None +default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None +hadoop_bin_dir = hdp_select.get_hadoop_dir("bin") if has_namenode else None +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py index 037f20a..a76457f 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py @@ -41,6 +41,26 @@ def setup_ranger_storm(upgrade_type=None): else: Logger.info("Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + if params.xml_configurations_supported and params.enable_ranger_storm and params.xa_audit_hdfs_is_enabled: + if params.has_namenode: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/storm", + type="directory", + action="create_on_execute", + owner=params.storm_user, + group=params.storm_user, + mode=0700, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + setup_ranger_plugin('storm-nimbus', 'storm', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index d45375f..cb8f77b 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -390,6 +390,7 @@ if has_ranger_admin: ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-yarn-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None + xa_audit_hdfs_is_enabled = config['configurations']['ranger-yarn-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py index 5db65d0d..21fe8e1 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py @@ -28,6 +28,25 @@ def setup_ranger_yarn(): else: Logger.info("YARN: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + if params.xml_configurations_supported and params.enable_ranger_yarn and params.xa_audit_hdfs_is_enabled: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/yarn", + type="directory", + action="create_on_execute", + owner=params.yarn_user, + group=params.yarn_user, + mode=0700, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + setup_ranger_plugin('hadoop-yarn-resourcemanager', 'yarn', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, http://git-wip-us.apache.org/repos/asf/ambari/blob/6c3cf499/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json index bfe286b..d634ce1 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json @@ -11,6 +11,10 @@ "SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"], "HAWQMASTER-START" : ["NAMENODE-START","DATANODE-START","HAWQSTANDBY-START"], "HAWQSEGMENT-START" : ["HAWQMASTER-START","HAWQSTANDBY-START"], - "HAWQ_SERVICE_CHECK-SERVICE_CHECK" : ["HAWQMASTER-START"] + "HAWQ_SERVICE_CHECK-SERVICE_CHECK" : ["HAWQMASTER-START"], + "KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"], + "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"], + "NIMBUS-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"], + "STORM_UI_SERVER-START" : ["NIMBUS-START", "NAMENODE-START"] } }