ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ab...@apache.org
Subject ranger git commit: RANGER-2024: tagsync updates to handle name-services in federated hdfs
Date Sun, 18 Mar 2018 20:50:56 GMT
Repository: ranger
Updated Branches:
  refs/heads/master a1929a824 -> 54fab186a


RANGER-2024: tagsync updates to handle name-services in federated hdfs


Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/54fab186
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/54fab186
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/54fab186

Branch: refs/heads/master
Commit: 54fab186a40a04ba88527a811e378125fbedd98d
Parents: a1929a8
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Sun Mar 18 13:50:48 2018 -0700
Committer: Abhay Kulkarni <akulkarni@hortonworks.com>
Committed: Sun Mar 18 13:50:48 2018 -0700

----------------------------------------------------------------------
 .../source/atlas/AtlasHdfsResourceMapper.java   | 32 +++++++++++-
 .../source/atlas/AtlasResourceMapper.java       | 10 ++--
 .../src/main/resources/ranger-tagsync-site.xml  | 16 ++++++
 .../tagsync/process/TestHdfsResourceMapper.java | 54 +++++++++++++++++++-
 4 files changed, 104 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ranger/blob/54fab186/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java
index 378542c..8355c91 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java
@@ -32,9 +32,13 @@ import org.apache.ranger.tagsync.source.atlasrest.RangerAtlasEntity;
 public class AtlasHdfsResourceMapper extends AtlasResourceMapper {
 	public static final String ENTITY_TYPE_HDFS_PATH = "hdfs_path";
 	public static final String RANGER_TYPE_HDFS_PATH = "path";
+	public static final String TAGSYNC_ATLAS_NAME_SERVICE_IDENTIFIER = ".nameservice.";
+	public static final String ENTITY_TYPE_HDFS_CLUSTER_AND_NAME_SERVICE_SEPARATOR = "_";
+
 
 	public static final String ENTITY_ATTRIBUTE_PATH           = "path";
 	public static final String ENTITY_ATTRIBUTE_CLUSTER_NAME   = "clusterName";
+	public static final String ENTITY_ATTRIBUTE_NAME_SERVICE_ID   = "nameServiceId";
 
 	public static final String[] SUPPORTED_ENTITY_TYPES = { ENTITY_TYPE_HDFS_PATH };
 
@@ -43,6 +47,10 @@ public class AtlasHdfsResourceMapper extends AtlasResourceMapper {
 		super("hdfs", SUPPORTED_ENTITY_TYPES);
 	}
 
+	/*
+		If cl2_hdfs is the service-name for HDFS in 'cl2' cluster, with no specific nameService
(in a non-Federated) setup
+		# ranger.ragsync.atlas.hdfs.instance.cl2.ranger.service=cl2_hdfs
+	 */
 	@Override
 	public String getRangerServiceName(String clusterName) {
 		String ret = getCustomRangerServiceName(clusterName);
@@ -54,11 +62,33 @@ public class AtlasHdfsResourceMapper extends AtlasResourceMapper {
 		return ret;
 	}
 
+	/*
+		If c11_hdfs_ns1 is the service-name for HDFS in 'cl1' cluster, that services nameService
'ns1' in a Federated setup
+		# ranger.tagsync.atlas.hdfs.instance.cl1.nameservice.ns1.ranger.service=cl1_hdfs_ns1
+	*/
+
+	private String getCustomRangerServiceNameForClusterAndNameService(String clusterName, String
nameServiceId) {
+		String ret = null;
+		if(properties != null) {
+			String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName
+					+ TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + clusterName
+					+ TAGSYNC_ATLAS_NAME_SERVICE_IDENTIFIER + nameServiceId
+					+ TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX;
+
+			ret = properties.getProperty(propName);
+		}
+		if (ret == null) {
+			ret = getRangerServiceName(clusterName) + ENTITY_TYPE_HDFS_CLUSTER_AND_NAME_SERVICE_SEPARATOR
+ nameServiceId;
+		}
+		return ret;
+	}
+
 	@Override
 	public RangerServiceResource buildResource(final RangerAtlasEntity entity) throws Exception
{
 		String path          = (String)entity.getAttributes().get(ENTITY_ATTRIBUTE_PATH);
 		String clusterName   = (String)entity.getAttributes().get(ENTITY_ATTRIBUTE_CLUSTER_NAME);
 		String qualifiedName = (String)entity.getAttributes().get(AtlasResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME);
+		String nameServiceId = (String)entity.getAttributes().get(ENTITY_ATTRIBUTE_NAME_SERVICE_ID);
 
 		if(StringUtils.isEmpty(path)) {
 			path = getResourceNameFromQualifiedName(qualifiedName);
@@ -81,7 +111,7 @@ public class AtlasHdfsResourceMapper extends AtlasResourceMapper {
 		}
 
 		String  entityGuid  = entity.getGuid();
-		String  serviceName = getRangerServiceName(clusterName);
+		String  serviceName = StringUtils.isNotBlank(nameServiceId) ? getCustomRangerServiceNameForClusterAndNameService(clusterName,
nameServiceId) : getRangerServiceName(clusterName);
 		Boolean isExcludes  = Boolean.FALSE;
 		Boolean isRecursive = Boolean.TRUE;
 

http://git-wip-us.apache.org/repos/asf/ranger/blob/54fab186/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
index 5d067a5..955c443 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
@@ -35,11 +35,11 @@ public abstract class AtlasResourceMapper {
 	public static final String QUALIFIED_NAME_DELIMITER        = "\\.";
 	public static final Character QUALIFIED_NAME_DELIMITER_CHAR    = '.';
 
-	protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX                  =
"ranger.tagsync.atlas.";
-	protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX                  =
".ranger.service";
-	protected static final String TAGSYNC_ATLAS_CLUSTER_IDENTIFIER                        =
".instance.";
-	protected static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR =
"_";
-	protected static final String CLUSTER_DELIMITER                                       =
"@";
+	public static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX                  = "ranger.tagsync.atlas.";
+	public static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX                  = ".ranger.service";
+	public static final String TAGSYNC_ATLAS_CLUSTER_IDENTIFIER                        = ".instance.";
+	public static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_";
+	public static final String CLUSTER_DELIMITER                                       = "@";
 
 	protected final String   componentName;
 	protected final String[] supportedEntityTypes;

http://git-wip-us.apache.org/repos/asf/ranger/blob/54fab186/tagsync/src/main/resources/ranger-tagsync-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml
index ea36ed6..9a14c1c 100644
--- a/tagsync/src/main/resources/ranger-tagsync-site.xml
+++ b/tagsync/src/main/resources/ranger-tagsync-site.xml
@@ -97,4 +97,20 @@
 		<name>ranger.tagsync.cookie.enabled</name>
 		<value>true</value>
 	</property>
+
+    <!-- Ranger-tagsync uses the following two properties to derive name of Ranger Service
in a Federated or non-Federated HDFS setup -->
+
+    <!-- service-name for HDFS in 'cl1' cluster, that services nameService 'ns1' in a
Federated setup
+    <property>
+        <name>ranger.tagsync.atlas.hdfs.instance.cl1.nameservice.ns1.ranger.service</name>
+        <value>cl1_hdfs_ns1</value>
+    </property>
+    -->
+    <!-- service-name for HDFS in 'cl2' cluster, with no specific nameService (generally,
in a non-Federated) setup
+    <property>
+        <name>ranger.ragsync.atlas.hdfs.instance.cl2.ranger.service</name>
+        <value>cl2_hdfs</value>
+    </property>
+    -->
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/ranger/blob/54fab186/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java
b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java
index 6276904..e12f640 100644
--- a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java
+++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java
@@ -39,11 +39,14 @@ public class TestHdfsResourceMapper {
 	private static final String PATH           = "hdfs://localhost:8020/user/testuser/finance";
 	private static final String QUALIFIED_NAME = "hdfs://localhost:8020/user/testuser/finance@cl1";
 
-	private static final String SERVICE_NAME = "cl1_hadoop";
-	private static final String RANGER_PATH  = "/user/testuser/finance";
+	private static final String SERVICE_NAME   = "cl1_hadoop";
+	private static final String RANGER_PATH    = "/user/testuser/finance";
+	private static final String NAMESERVICE_ID = "name-service-1";
+
 
 	AtlasHdfsResourceMapper resourceMapper = new AtlasHdfsResourceMapper();
 	AtlasHdfsResourceMapper resourceMapperWithDefaultClusterName = new AtlasHdfsResourceMapper();
+	AtlasHdfsResourceMapper resourceMapperWithFederatedService = new AtlasHdfsResourceMapper();
 
 	{
 		Properties properties = new Properties();
@@ -51,6 +54,15 @@ public class TestHdfsResourceMapper {
 		properties.setProperty(AtlasResourceMapper.TAGSYNC_DEFAULT_CLUSTER_NAME, CLUSTER_NAME);
 
 		resourceMapperWithDefaultClusterName.initialize(properties);
+
+		String propName = AtlasHdfsResourceMapper.TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + "hdfs"
+				+ AtlasHdfsResourceMapper.TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + CLUSTER_NAME
+				+ AtlasHdfsResourceMapper.TAGSYNC_ATLAS_NAME_SERVICE_IDENTIFIER + NAMESERVICE_ID
+				+ AtlasHdfsResourceMapper.TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX;
+
+		properties.setProperty(propName, SERVICE_NAME);
+
+		resourceMapperWithFederatedService.initialize(properties);
 	}
 
 	@Test
@@ -145,6 +157,32 @@ public class TestHdfsResourceMapper {
 		}
 	}
 
+	@Test
+	public void testHdfsResourceFromQualifiedNameAndNameServiceId() throws Exception {
+		Map<String, Object> entAttribs = new HashMap<String, Object>();
+
+		entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, QUALIFIED_NAME);
+		entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_NAME_SERVICE_ID, NAMESERVICE_ID);
+
+		RangerAtlasEntity entity   = getHdfsPathEntity(entAttribs);
+		RangerServiceResource  resource = resourceMapper.buildResource(entity);
+
+		assertFederatedServiceResource(resource);
+	}
+
+	@Test
+	public void testHdfsResourceFromQualifiedNameAndNameServiceIdFromProperty() throws Exception
{
+		Map<String, Object> entAttribs = new HashMap<String, Object>();
+
+		entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, QUALIFIED_NAME);
+		entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_NAME_SERVICE_ID, NAMESERVICE_ID);
+
+		RangerAtlasEntity entity   = getHdfsPathEntity(entAttribs);
+		RangerServiceResource  resource = resourceMapperWithFederatedService.buildResource(entity);
+
+		assertServiceResource(resource);
+	}
+
 	private RangerAtlasEntity getHdfsPathEntity(Map<String, Object> entAttribs) throws
Exception {
 		RangerAtlasEntity entity = Mockito.mock(RangerAtlasEntity.class);
 
@@ -166,4 +204,16 @@ public class TestHdfsResourceMapper {
 		Assert.assertEquals(1, resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues().size());
 		Assert.assertEquals(RANGER_PATH, resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues().get(0));
 	}
+
+	private void assertFederatedServiceResource(RangerServiceResource resource) {
+		String serviceName = SERVICE_NAME + AtlasHdfsResourceMapper.ENTITY_TYPE_HDFS_CLUSTER_AND_NAME_SERVICE_SEPARATOR
+ NAMESERVICE_ID;
+		Assert.assertNotNull(resource);
+		Assert.assertEquals(serviceName, resource.getServiceName());
+		Assert.assertNotNull(resource.getResourceElements());
+		Assert.assertEquals(1, resource.getResourceElements().size());
+		Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH));
+		Assert.assertNotNull(resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues());
+		Assert.assertEquals(1, resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues().size());
+		Assert.assertEquals(RANGER_PATH, resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues().get(0));
+	}
 }


Mime
View raw message