ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jai...@apache.org
Subject ambari git commit: AMBARI-13897. Ambari does not configure hbase.coprocessor.regionserver.classes. (jaimin)
Date Mon, 16 Nov 2015 19:26:33 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 9f715b3aa -> b2910e1cb


AMBARI-13897. Ambari does not configure hbase.coprocessor.regionserver.classes. (jaimin)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b2910e1c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b2910e1c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b2910e1c

Branch: refs/heads/branch-2.1
Commit: b2910e1cbba6fca5dc697adb3ff7dfd09089394a
Parents: 9f715b3
Author: Jaimin Jetly <jaimin@hortonworks.com>
Authored: Mon Nov 16 11:25:37 2015 -0800
Committer: Jaimin Jetly <jaimin@hortonworks.com>
Committed: Mon Nov 16 11:25:37 2015 -0800

----------------------------------------------------------------------
 .../HBASE/0.96.0.2.0/kerberos.json              |  5 +-
 .../0.96.0.2.0/package/scripts/params_linux.py  |  5 +-
 .../stacks/HDP/2.2/services/stack_advisor.py    | 68 ++++++++++++--------
 .../stacks/2.2/common/test_stack_advisor.py     | 11 ++--
 .../stacks/2.3/common/test_stack_advisor.py     |  2 +
 5 files changed, 56 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b2910e1c/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json
index b59f727..dc5ef2e 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/kerberos.json
@@ -37,8 +37,9 @@
             "hbase.security.authentication": "kerberos",
             "hbase.security.authorization": "true",
             "zookeeper.znode.parent": "/hbase-secure",
-              "hbase.coprocessor.master.classes": "{{hbase_coprocessor_master_classes}}",
-              "hbase.coprocessor.region.classes": "{{hbase_coprocessor_region_classes}}",
+            "hbase.coprocessor.master.classes": "{{hbase_coprocessor_master_classes}}",
+            "hbase.coprocessor.region.classes": "{{hbase_coprocessor_region_classes}}",
+            "hbase.coprocessor.regionserver.classes": "{{hbase_coprocessor_regionserver_classes}}",
             "hbase.bulkload.staging.dir": "/apps/hbase/staging"
           }
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/b2910e1c/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index bab2cc5..6fcdd75 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -327,15 +327,18 @@ if has_ranger_admin:
   if xa_audit_db_flavor == 'sqla':
     xa_audit_db_is_enabled = False
 
-# Used to dynamically set the hbase-site props that are referenced during Kerbenization
+# Used to dynamically set the hbase-site props that are referenced during Kerberization
 if security_enabled:
   if not enable_ranger_hbase: # Default props, no ranger plugin
     hbase_coprocessor_master_classes = "org.apache.hadoop.hbase.security.access.AccessController"
+    hbase_coprocessor_regionserver_classes = "org.apache.hadoop.hbase.security.access.AccessController"
     hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.hadoop.hbase.security.access.AccessController"
   elif xml_configurations_supported: # HDP stack 2.3+ ranger plugin enabled
     hbase_coprocessor_master_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor
"
+    hbase_coprocessor_regionserver_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
     hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
   else: # HDP Stack 2.2 and less / ranger plugin enabled
     hbase_coprocessor_master_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor"
+    hbase_coprocessor_regionserver_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor"
     hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor"
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b2910e1c/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 6645083..b51af8b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -631,15 +631,20 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
         putHbaseEnvPropertyAttributes('hbase_max_direct_memory_size', 'delete', 'true')
 
     # Authorization
-    hbase_coprocessor_region_classes = None
-    if 'hbase.coprocessor.region.classes' in configurations["hbase-site"]["properties"]:
-      hbase_coprocessor_region_classes = configurations["hbase-site"]["properties"]["hbase.coprocessor.region.classes"].strip()
-    elif 'hbase-site' in services['configurations'] and 'hbase.coprocessor.region.classes'
in services['configurations']["hbase-site"]["properties"]:
-      hbase_coprocessor_region_classes = services['configurations']["hbase-site"]["properties"]["hbase.coprocessor.region.classes"].strip()
-    if hbase_coprocessor_region_classes:
-      coprocessorRegionClassList = hbase_coprocessor_region_classes.split(',')
-    else:
-      coprocessorRegionClassList = []
+    hbaseCoProcessorConfigs = {
+      'hbase.coprocessor.region.classes': [],
+      'hbase.coprocessor.regionserver.classes': [],
+      'hbase.coprocessor.master.classes': []
+    }
+    for key in hbaseCoProcessorConfigs:
+      hbase_coprocessor_classes = None
+      if key in configurations["hbase-site"]["properties"]:
+        hbase_coprocessor_classes = configurations["hbase-site"]["properties"][key].strip()
+      elif 'hbase-site' in services['configurations'] and key in services['configurations']["hbase-site"]["properties"]:
+        hbase_coprocessor_classes = services['configurations']["hbase-site"]["properties"][key].strip()
+      if hbase_coprocessor_classes:
+        hbaseCoProcessorConfigs[key] = hbase_coprocessor_classes.split(',')
+
     # If configurations has it - it has priority as it is calculated. Then, the service's
configurations will be used.
     hbase_security_authorization = None
     if 'hbase-site' in configurations and 'hbase.security.authorization' in configurations['hbase-site']['properties']:
@@ -648,18 +653,21 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       hbase_security_authorization = services['configurations']['hbase-site']['properties']['hbase.security.authorization']
     if hbase_security_authorization:
       if 'true' == hbase_security_authorization.lower():
-        putHbaseSiteProperty('hbase.coprocessor.master.classes', "org.apache.hadoop.hbase.security.access.AccessController")
-        coprocessorRegionClassList.append("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint")
-        coprocessorRegionClassList.append("org.apache.hadoop.hbase.security.access.AccessController")
-        putHbaseSiteProperty('hbase.coprocessor.regionserver.classes', "org.apache.hadoop.hbase.security.access.AccessController")
+        hbaseCoProcessorConfigs['hbase.coprocessor.master.classes'].append('org.apache.hadoop.hbase.security.access.AccessController')
+        hbaseCoProcessorConfigs['hbase.coprocessor.regionserver.classes'].append('org.apache.hadoop.hbase.security.access.AccessController')
+        # regional classes when hbase authorization is enabled
+        authRegionClasses = ['org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint',
'org.apache.hadoop.hbase.security.access.AccessController']
+        for item in range(len(authRegionClasses)):
+          hbaseCoProcessorConfigs['hbase.coprocessor.region.classes'].append(authRegionClasses[item])
       else:
-        putHbaseSiteProperty('hbase.coprocessor.master.classes', "")
-        coprocessorRegionClassList.append("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint")
+        if 'org.apache.hadoop.hbase.security.access.AccessController' in hbaseCoProcessorConfigs['hbase.coprocessor.region.classes']:
+          hbaseCoProcessorConfigs['hbase.coprocessor.master.classes'].remove('org.apache.hadoop.hbase.security.access.AccessController')
+        hbaseCoProcessorConfigs['hbase.coprocessor.region.classes'].append("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint")
         if ('hbase.coprocessor.regionserver.classes' in configurations["hbase-site"]["properties"])
or \
                 ('hbase-site' in services['configurations'] and 'hbase.coprocessor.regionserver.classes'
in services['configurations']["hbase-site"]["properties"]):
           putHbaseSitePropertyAttributes('hbase.coprocessor.regionserver.classes', 'delete',
'true')
     else:
-      coprocessorRegionClassList.append("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint")
+      hbaseCoProcessorConfigs['hbase.coprocessor.region.classes'].append("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint")
       if ('hbase.coprocessor.regionserver.classes' in configurations["hbase-site"]["properties"])
or \
               ('hbase-site' in services['configurations'] and 'hbase.coprocessor.regionserver.classes'
in services['configurations']["hbase-site"]["properties"]):
         putHbaseSitePropertyAttributes('hbase.coprocessor.regionserver.classes', 'delete',
'true')
@@ -667,20 +675,24 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     # Authentication
     if 'hbase-site' in services['configurations'] and 'hbase.security.authentication' in
services['configurations']['hbase-site']['properties']:
       if 'kerberos' == services['configurations']['hbase-site']['properties']['hbase.security.authentication'].lower():
-        if 'org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint' not in coprocessorRegionClassList:
-          coprocessorRegionClassList.append('org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint')
-        if 'org.apache.hadoop.hbase.security.token.TokenProvider' not in coprocessorRegionClassList:
-          coprocessorRegionClassList.append('org.apache.hadoop.hbase.security.token.TokenProvider')
+        if 'org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint' not in hbaseCoProcessorConfigs['hbase.coprocessor.region.classes']:
+          hbaseCoProcessorConfigs['hbase.coprocessor.region.classes'].append('org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint')
+        if 'org.apache.hadoop.hbase.security.token.TokenProvider' not in hbaseCoProcessorConfigs['hbase.coprocessor.region.classes']:
+          hbaseCoProcessorConfigs['hbase.coprocessor.region.classes'].append('org.apache.hadoop.hbase.security.token.TokenProvider')
       else:
-        if 'org.apache.hadoop.hbase.security.token.TokenProvider' in coprocessorRegionClassList:
-          coprocessorRegionClassList.remove('org.apache.hadoop.hbase.security.token.TokenProvider')
+        if 'org.apache.hadoop.hbase.security.token.TokenProvider' in hbaseCoProcessorConfigs['hbase.coprocessor.region.classes']:
+          hbaseCoProcessorConfigs['hbase.coprocessor.region.classes'].remove('org.apache.hadoop.hbase.security.token.TokenProvider')
 
     #Remove duplicates
-    uniqueCoprocessorRegionClassList = []
-    [uniqueCoprocessorRegionClassList.append(i)
-     for i in coprocessorRegionClassList if
-     not i in uniqueCoprocessorRegionClassList and i != '{{hbase_coprocessor_region_classes}}']
-    putHbaseSiteProperty('hbase.coprocessor.region.classes', ','.join(set(uniqueCoprocessorRegionClassList)))
+    for key in hbaseCoProcessorConfigs:
+      uniqueCoprocessorRegionClassList = []
+      [uniqueCoprocessorRegionClassList.append(i)
+       for i in hbaseCoProcessorConfigs[key] if
+       not i in uniqueCoprocessorRegionClassList
+       and (i.strip() not in ['{{hbase_coprocessor_region_classes}}', '{{hbase_coprocessor_master_classes}}',
'{{hbase_coprocessor_regionserver_classes}}'])]
+      putHbaseSiteProperty(key, ','.join(set(uniqueCoprocessorRegionClassList)))
+
+
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     rangerServiceVersion=''
     if 'RANGER' in servicesList:
@@ -692,7 +704,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       rangerClass = 'org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor'
 
     nonRangerClass = 'org.apache.hadoop.hbase.security.access.AccessController'
-    hbaseClassConfigs = ['hbase.coprocessor.master.classes', 'hbase.coprocessor.region.classes']
+    hbaseClassConfigs =  hbaseCoProcessorConfigs.keys()
 
     for item in range(len(hbaseClassConfigs)):
       if 'hbase-site' in services['configurations']:

http://git-wip-us.apache.org/repos/asf/ambari/blob/b2910e1c/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 2a30119..0acaaff 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2327,7 +2327,9 @@ class TestHDP22StackAdvisor(TestCase):
           "hbase.regionserver.wal.codec": "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec",
           "phoenix.functions.allowUserDefinedFunctions": "true",
           "hbase.regionserver.global.memstore.size": "0.4",
-          "hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint"
+          "hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint",
+          "hbase.coprocessor.regionserver.classes": "",
+          "hbase.coprocessor.master.classes": "",
         },
         'property_attributes': {
           "hbase.bucketcache.size": {
@@ -2433,7 +2435,7 @@ class TestHDP22StackAdvisor(TestCase):
     expected['hbase-site']['properties']['hbase.security.authorization'] = "true"
     expected['hbase-site']['properties']['hbase.coprocessor.region.classes'] = 'org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor'
     expected['hbase-site']['properties']['hbase.coprocessor.master.classes'] = 'com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor'
-    expected['hbase-site']['properties']['hbase.coprocessor.regionserver.classes'] = 'org.apache.hadoop.hbase.security.access.AccessController'
+    expected['hbase-site']['properties']['hbase.coprocessor.regionserver.classes'] = 'com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor'
     self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services,
None)
     self.assertEquals(configurations, expected, "Test when Ranger plugin HBase is enabled
in non-kerberos environment")
 
@@ -2446,11 +2448,11 @@ class TestHDP22StackAdvisor(TestCase):
     services['configurations']['hbase-site']['properties']['hbase.security.authorization']
= 'false'
     services['configurations']['hbase-site']['properties']['hbase.security.authentication']
= 'kerberos'
     services['configurations']['hbase-site']['properties']['hbase.coprocessor.master.classes']
= ''
-    services['configurations']['hbase-site']['properties']['hbase.coprocessor.region.classes']
= 'a.b.c.d'
+    services['configurations']['hbase-site']['properties']['hbase.coprocessor.region.classes']
= 'a.b.c.d, {{hbase_coprocessor_region_classes}}'
     expected['hbase-site']['properties']['hbase.coprocessor.region.classes'] = 'a.b.c.d,org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint'
     expected['hbase-site']['properties']['hbase.coprocessor.master.classes'] = ''
+    expected['hbase-site']['properties']['hbase.coprocessor.regionserver.classes'] = ''
     del expected['hbase-site']['properties']['hbase.security.authorization']
-    del expected['hbase-site']['properties']['hbase.coprocessor.regionserver.classes']
     self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services,
None)
     self.assertEquals(configurations, expected, "Test with Kerberos enabled and hbase.coprocessor.region.classes
predefined")
 
@@ -2474,6 +2476,7 @@ class TestHDP22StackAdvisor(TestCase):
     services['configurations']['ranger-hbase-plugin-properties']['properties']['ranger-hbase-plugin-enabled']
= 'Yes'
     expected['hbase-site']['properties']['hbase.security.authorization']  = 'true'
     expected['hbase-site']['properties']['hbase.coprocessor.master.classes'] = 'com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor'
+    expected['hbase-site']['properties']['hbase.coprocessor.regionserver.classes'] = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor"
     expected['hbase-site']['properties']['hbase.coprocessor.region.classes'] = 'org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor'
     self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services,
None)
     self.assertEquals(configurations, expected, "Test with Kerberos enabled and HBase ranger
plugin enabled")

http://git-wip-us.apache.org/repos/asf/ambari/blob/b2910e1c/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 4db688c..02753d4 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -310,6 +310,8 @@ class TestHDP23StackAdvisor(TestCase):
           "hbase.regionserver.global.memstore.size": "0.4",
           "hfile.block.cache.size": "0.4",
           "hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint",
+          "hbase.coprocessor.master.classes": "",
+          "hbase.coprocessor.regionserver.classes": "",
           "hbase.rpc.controllerfactory.class": "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory",
           "hbase.region.server.rpc.scheduler.factory.class": "org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory",
           'hbase.regionserver.wal.codec': 'org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec',


Mime
View raw message