atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suma...@apache.org
Subject incubator-atlas git commit: ATLAS-1139 Parameter name of a HDFS DataSet entity should contain filesystem path (svimal2106 via sumasai)
Date Mon, 24 Oct 2016 22:26:19 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master fdf97ae4d -> aa15cd0ae


ATLAS-1139 Parameter name of a HDFS DataSet entity should contain filesystem path (svimal2106
via sumasai)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/aa15cd0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/aa15cd0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/aa15cd0a

Branch: refs/heads/master
Commit: aa15cd0aec15000a225f2a32477c4158d224b24e
Parents: fdf97ae
Author: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Authored: Mon Oct 24 15:26:10 2016 -0700
Committer: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Committed: Mon Oct 24 15:26:10 2016 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/atlas/falcon/bridge/FalconBridge.java  | 4 +++-
 .../java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java  | 2 +-
 .../src/test/java/org/apache/atlas/hive/HiveITBase.java         | 5 +++--
 .../src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java  | 4 ++--
 .../main/java/org/apache/atlas/storm/hook/StormAtlasHook.java   | 2 +-
 release-log.txt                                                 | 1 +
 6 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
index 919bd39..05d072f 100644
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
+++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
@@ -46,6 +46,7 @@ import org.apache.falcon.entity.v0.process.Input;
 import org.apache.falcon.entity.v0.process.Output;
 import org.apache.falcon.entity.v0.process.Workflow;
 import org.apache.falcon.workflow.WorkflowExecutionArgs;
+import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -323,7 +324,8 @@ public class FalconBridge {
         //        Path path = new Path(pathUri);
         //        ref.set("name", path.getName());
         //TODO - Fix after ATLAS-542 to shorter Name
-        ref.set(AtlasClient.NAME, pathUri);
+        Path path = new Path(pathUri);
+        ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
         ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
         ref.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
         entities.add(ref);

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 31ce140..2d2039b 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -563,7 +563,7 @@ public class HiveMetaStoreBridge {
         Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString());
         ref.set("path", pathUri);
         Path path = new Path(pathUri);
-        ref.set(AtlasClient.NAME, path.getName());
+        ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
         ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
         return ref;
     }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
index bc3312d..bdf0d2d 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
@@ -218,13 +218,14 @@ public class HiveITBase {
         List<Id> hdfsPathRefs = (List<Id>) processReference.get(attributeName);
 
         for (int i = 0; i < testPaths.length; i++) {
-            final String testPathNormed = lower(new Path(testPaths[i]).toString());
+            final Path path = new Path(testPaths[i]);
+            final String testPathNormed = lower(path.toString());
             String hdfsPathId = assertHDFSPathIsRegistered(testPathNormed);
             Assert.assertEquals(hdfsPathRefs.get(0)._getId(), hdfsPathId);
 
             Referenceable hdfsPathRef = atlasClient.getEntity(hdfsPathId);
             Assert.assertEquals(hdfsPathRef.get("path"), testPathNormed);
-            Assert.assertEquals(hdfsPathRef.get(NAME), new Path(testPathNormed).getName());
+            Assert.assertEquals(hdfsPathRef.get(NAME), Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
             Assert.assertEquals(hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME),
testPathNormed);
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
index 772f1b2..d4fb3af 100644
--- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
+++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
@@ -119,8 +119,8 @@ public class SqoopHookIT {
     private String assertHiveTableIsRegistered(String dbName, String tableName) throws Exception
{
         LOG.debug("Searching for table {}.{}", dbName, tableName);
         String query = String.format(
-                "%s as t where " + AtlasClient.NAME + " = '%s',  db where " + AtlasClient.NAME
+ " = '%s' and clusterName = '%s'" + " select t",
-                HiveDataTypes.HIVE_TABLE.getName(), tableName.toLowerCase(), dbName.toLowerCase(),
CLUSTER_NAME);
+                "%s as t where " + AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME + " = '%s', 
db where " + AtlasClient.NAME + " = '%s' and clusterName = '%s'" + " select t",
+                HiveDataTypes.HIVE_TABLE.getName(), HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
dbName, tableName), dbName.toLowerCase(), CLUSTER_NAME);
         return assertEntityIsRegistered(query);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
index 89e1097..bb6a476 100644
--- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
+++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
@@ -223,7 +223,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook
{
                 dataSetReferenceable.set("path", hdfsPathStr);
                 dataSetReferenceable.set(AtlasClient.OWNER, stormConf.get("hdfs.kerberos.principal"));
                 final Path hdfsPath = new Path(hdfsPathStr);
-                dataSetReferenceable.set(AtlasClient.NAME, hdfsPath.getName());
+                dataSetReferenceable.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(hdfsPath).toString().toLowerCase());
                 break;
 
             case "HiveBolt":

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 9d29bb9..76c6a77 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements
for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1139 Parameter name of a HDFS DataSet entity should contain filesystem path (svimal2106
via sumasai)
 ATLAS-1200 Error Catalog enhancement (apoorvnaik via sumasai)
 ATLAS-1207 Dataset exists query in lineage APIs takes longer (shwethags)
 ATLAS-1232 added preCreate(), preDelete() in typedef persistence, to enable edge creation
for references in a later stage (mneethiraj)


Mime
View raw message