ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [15/53] [abbrv] ambari git commit: AMBARI-20779 Create Ranger KMS HDFS audit folder as part of install (mugdha)
Date Wed, 03 May 2017 16:46:58 GMT
AMBARI-20779 Create Ranger KMS HDFS audit folder as part of install (mugdha)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9dd542be
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9dd542be
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9dd542be

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 9dd542be2c7bb3b12ea50c8902f3d8e1413eebf3
Parents: 4857a6e
Author: Mugdha Varadkar <mugdha@apache.org>
Authored: Fri Apr 28 13:35:27 2017 +0530
Committer: Gautam Borad <gautam@apache.org>
Committed: Sun Apr 30 23:23:08 2017 +0530

----------------------------------------------------------------------
 .../RANGER/0.5.0/role_command_order.json        |  2 +-
 .../RANGER_KMS/0.5.0.2.3/metainfo.xml           |  9 ++
 .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 20 ++++
 .../0.5.0.2.3/package/scripts/params.py         | 34 ++++++-
 .../0.5.0.2.3/role_command_order.json           |  2 +-
 .../stacks/HDP/2.3/role_command_order.json      |  4 +-
 .../stacks/2.5/RANGER_KMS/test_kms_server.py    | 98 ++++++++++++++++++++
 .../stacks/2.5/configs/ranger-kms-secured.json  |  2 +-
 8 files changed, 165 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
index df62dfd..073a077 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json
@@ -1,6 +1,6 @@
 {
   "general_deps" : {
     "_comment" : "dependencies for RANGER",
-    "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"]
+    "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"]
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
index b20201c..bc0aa74 100644
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml
@@ -45,6 +45,15 @@
               <primary>true</primary>
             </log>
           </logs>
+          <dependencies>
+            <dependency>
+              <name>HDFS/HDFS_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
         </component>
 
       </components>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
index 423cdec..5a25b92 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
@@ -506,6 +506,26 @@ def enable_kms_plugin():
       mode = 0640
       )
 
+    # create ranger kms audit directory
+    if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
+      params.HdfsResource("/ranger/audit",
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.hdfs_user,
+                        group=params.hdfs_user,
+                        mode=0755,
+                        recursive_chmod=True
+      )
+      params.HdfsResource("/ranger/audit/kms",
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.kms_user,
+                        group=params.kms_group,
+                        mode=0750,
+                        recursive_chmod=True
+      )
+      params.HdfsResource(None, action="execute")
+
     if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
       Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml')
       XmlConfig("hdfs-site.xml",

http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
index 9fe0a61..2445f2e 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py
@@ -29,6 +29,9 @@ from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.get_bare_principal import get_bare_principal
 from resource_management.libraries.functions.is_empty import is_empty
 from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import get_kinit_path
 
 config  = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
@@ -296,4 +299,33 @@ xa_audit_hdfs_is_enabled = default("/configurations/ranger-kms-audit/xasecure.au
 namenode_host = default("/clusterHostInfo/namenode_host", [])
 
 # need this to capture cluster name from where ranger kms plugin is enabled
-cluster_name = config['clusterName']
\ No newline at end of file
+cluster_name = config['clusterName']
+
+has_namenode = len(namenode_host) > 0
+
+hdfs_user = default("/configurations/hadoop-env/hdfs_user", None)
+hdfs_user_keytab = default("/configurations/hadoop-env/hdfs_user_keytab", None)
+hdfs_principal_name = default("/configurations/hadoop-env/hdfs_principal_name", None)
+default_fs = default("/configurations/core-site/fs.defaultFS", None)
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+
+import functools
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource
in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
+  security_enabled = security_enabled,
+  keytab = hdfs_user_keytab,
+  kinit_path_local = kinit_path_local,
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir,
+  principal_name = hdfs_principal_name,
+  hdfs_site = hdfs_site,
+  default_fs = default_fs
+)
+
+local_component_list = default("/localComponents", [])
+has_hdfs_client_on_node = 'HDFS_CLIENT' in local_component_list
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
index 006d177..7ddab41 100644
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json
@@ -1,7 +1,7 @@
 {
   "general_deps" : {
     "_comment" : "dependencies for RANGER-KMS",
-    "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
+    "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"],
     "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"]
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index 4f279c6..5b1b50e 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -4,8 +4,8 @@
   "general_deps" : {
     "_comment" : "dependencies for all cases",
     "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
-    "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"],
-    "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
+    "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"],
+    "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"],
     "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"],
     "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
     "SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"],

http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
index 7082a33..3d03c5f 100644
--- a/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py
@@ -151,6 +151,55 @@ class TestRangerKMS(RMFTestCase):
       mode = 0640
     )
 
+    self.assertResourceCalled('HdfsResource', '/ranger/audit',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'hdfs',
+                        group = 'hdfs',
+                        mode = 0755,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = False,
+                        keytab = None,
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = None,
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', '/ranger/audit/kms',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'kms',
+                        group = 'kms',
+                        mode = 0750,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = False,
+                        keytab = None,
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = None,
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', None,
+                        action = ['execute'],
+                        user = 'hdfs',
+                        security_enabled = False,
+                        keytab = None,
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = None,
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
     self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml',
       action = ['delete'],
     )
@@ -537,6 +586,55 @@ class TestRangerKMS(RMFTestCase):
       mode = 0640
     )
 
+    self.assertResourceCalled('HdfsResource', '/ranger/audit',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'hdfs',
+                        group = 'hdfs',
+                        mode = 0755,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = True,
+                        keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = 'hdfs-cl1@EXAMPLE.COM',
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', '/ranger/audit/kms',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'kms',
+                        group = 'kms',
+                        mode = 0750,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = True,
+                        keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = 'hdfs-cl1@EXAMPLE.COM',
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', None,
+                        action = ['execute'],
+                        user = 'hdfs',
+                        security_enabled = True,
+                        keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = 'hdfs-cl1@EXAMPLE.COM',
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
     self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml',
       action = ['delete'],
     )

http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
index 7054e8f..4e7d857 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json
@@ -751,7 +751,7 @@
             "hdfs_log_dir_prefix": "/var/log/hadoop", 
             "hdfs_user_nofile_limit": "128000", 
             "hdfs_user": "hdfs", 
-            "hdfs_principal_name": "hdfs-test_cluster01@EXAMPLE.COM", 
+            "hdfs_principal_name": "hdfs-cl1@EXAMPLE.COM",
             "keyserver_host": " ", 
             "namenode_opt_maxnewsize": "128m", 
             "hdfs_user_keytab": "/etc/security/keytabs/hdfs.headless.keytab", 


Mime
View raw message