atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suma...@apache.org
Subject incubator-atlas git commit: ATLAS-758 hdfs location of hive table is pointing to old location even after rename.(sumasai)
Date Fri, 13 May 2016 03:38:46 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 54dc670af -> 07b8b4d3c


ATLAS-758 hdfs location of hive table is pointing to old location even after rename.(sumasai)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/07b8b4d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/07b8b4d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/07b8b4d3

Branch: refs/heads/master
Commit: 07b8b4d3c9673f231f1c9dab175ed3fbb2a98267
Parents: 54dc670
Author: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Authored: Thu May 12 20:38:36 2016 -0700
Committer: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Committed: Thu May 12 20:38:36 2016 -0700

----------------------------------------------------------------------
 .../atlas/hive/bridge/HiveMetaStoreBridge.java  |  4 +-
 .../org/apache/atlas/hive/hook/HiveHook.java    |  3 --
 .../hive/model/HiveDataModelGenerator.java      |  5 ++-
 .../org/apache/atlas/hive/hook/HiveHookIT.java  | 40 ++++++++++++--------
 release-log.txt                                 |  1 +
 5 files changed, 30 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/07b8b4d3/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 675f211..254e150 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -164,7 +164,7 @@ public class HiveMetaStoreBridge {
         dbRef.set(HiveDataModelGenerator.NAME, dbName);
         dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
         dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription());
-        dbRef.set("locationUri", hiveDB.getLocationUri());
+        dbRef.set(HiveDataModelGenerator.LOCATION, hiveDB.getLocationUri());
         dbRef.set(HiveDataModelGenerator.PARAMETERS, hiveDB.getParameters());
         dbRef.set(HiveDataModelGenerator.OWNER, hiveDB.getOwnerName());
         if (hiveDB.getOwnerType() != null) {
@@ -466,7 +466,7 @@ public class HiveMetaStoreBridge {
             sdReferenceable.set("sortCols", sortColsStruct);
         }
 
-        sdReferenceable.set("location", storageDesc.getLocation());
+        sdReferenceable.set(HiveDataModelGenerator.LOCATION, storageDesc.getLocation());
         sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
         sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
         sdReferenceable.set("compressed", storageDesc.isCompressed());

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/07b8b4d3/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index ffd5d22..5a1a36e 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -440,14 +440,11 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext
{
     private Referenceable replaceTableQFName(HiveMetaStoreBridge dgiBridge, HiveEventContext
event, Table oldTable, Table newTable, final Referenceable tableEntity, final String oldTableQFName,
final String newTableQFName) throws HiveException {
         tableEntity.set(HiveDataModelGenerator.NAME, oldTableQFName);
         tableEntity.set(HiveDataModelGenerator.TABLE_NAME, oldTable.getTableName().toLowerCase());
-        final Referenceable newDbInstance = (Referenceable) tableEntity.get(HiveDataModelGenerator.DB);
-        tableEntity.set(HiveDataModelGenerator.DB, dgiBridge.createDBInstance(dgiBridge.hiveClient.getDatabase(oldTable.getDbName())));
 
         //Replace table entity with new name
         final Referenceable newEntity = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
         newEntity.set(HiveDataModelGenerator.NAME, newTableQFName);
         newEntity.set(HiveDataModelGenerator.TABLE_NAME, newTable.getTableName().toLowerCase());
-        newEntity.set(HiveDataModelGenerator.DB, newDbInstance);
 
         messages.add(new HookNotification.EntityPartialUpdateRequest(event.getUser(),
             HiveDataTypes.HIVE_TABLE.getName(), HiveDataModelGenerator.NAME,

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/07b8b4d3/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
index 6418770..7cbb1df 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
@@ -78,6 +78,7 @@ public class HiveDataModelGenerator {
     public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
     public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
     public static final String OWNER = "owner";
+    public static final String LOCATION = "location";
 
     public static final String TABLE_TYPE_ATTR = "tableType";
 
@@ -171,7 +172,7 @@ public class HiveDataModelGenerator {
                 //Optional to keep it backward-compatible
                 new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL,
false,
                         STORAGE_DESC),
-                new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false,
+                new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false,
                         null),
                 new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false,
                         null),
@@ -209,7 +210,7 @@ public class HiveDataModelGenerator {
                         null),
                 new AttributeDefinition("description", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false,
                         null),
-                new AttributeDefinition("locationUri", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false,
+                new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false,
                         null),
                 new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
                 new AttributeDefinition(OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
false,

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/07b8b4d3/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index da21195..43bba0e 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -623,7 +623,7 @@ public class HiveHookIT {
         //Add trait to part col keys
         String partColTraitDetails = createTrait(partColumnGuid);
 
-        String newTableName = tableName();
+        final String newTableName = tableName();
         String query = String.format("alter table %s rename to %s", DEFAULT_DB + "." + tableName,
newDBName + "." + newTableName);
         runCommand(query);
 
@@ -640,7 +640,15 @@ public class HiveHookIT {
         assertTrait(partColumnGuid, partColTraitDetails);
 
         assertTableIsNotRegistered(DEFAULT_DB, tableName);
-        assertTableIsRegistered(newDBName, newTableName);
+
+        assertTableIsRegistered(newDBName, newTableName, new AssertPredicate() {
+            @Override
+            public void assertOnEntity(final Referenceable entity) throws Exception {
+                Referenceable sd = ((Referenceable) entity.get(HiveDataModelGenerator.STORAGE_DESC));
+                String location = (String) sd.get(HiveDataModelGenerator.LOCATION);
+                Assert.assertTrue(location.contains(newTableName));
+            }
+        });
     }
 
     private List<Referenceable> getColumns(String dbName, String tableName) throws
Exception {
@@ -673,8 +681,8 @@ public class HiveHookIT {
         runCommand(query);
 
         assertColumnIsRegistered(HiveMetaStoreBridge
-                .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName),
-                    column));
+            .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName),
+                column));
 
         //Verify the number of columns present in the table
         final List<Referenceable> columns = getColumns(DEFAULT_DB, tableName);
@@ -904,7 +912,7 @@ public class HiveHookIT {
             @Override
             public void assertOnEntity(Referenceable tableRef) throws Exception {
                 Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
-                Assert.assertEquals(new Path((String)sdRef.get("location")).toString(), new
Path(testPath).toString());
+                Assert.assertEquals(new Path((String)sdRef.get(HiveDataModelGenerator.LOCATION)).toString(),
new Path(testPath).toString());
             }
         });
 
@@ -1009,7 +1017,7 @@ public class HiveHookIT {
                                                ImmutableList<String>  sortcolNames)
throws Exception {
         Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
         Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveDataModelGenerator.STORAGE_NUM_BUCKETS)).intValue(),
-                numBuckets);
+            numBuckets);
         Assert.assertEquals(sdRef.get("bucketCols"), bucketColNames);
 
         List<Struct> hiveOrderStructList = (List<Struct>) sdRef.get("sortCols");
@@ -1050,11 +1058,11 @@ public class HiveHookIT {
         final String query = String.format("drop table %s ", tableName);
         runCommand(query);
         assertColumnIsNotRegistered(HiveMetaStoreBridge
-                .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName),
-                    "id"));
+            .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName),
+                "id"));
         assertColumnIsNotRegistered(HiveMetaStoreBridge
-                .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName),
-                    HiveDataModelGenerator.NAME));
+            .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName),
+                HiveDataModelGenerator.NAME));
         assertTableIsNotRegistered(DEFAULT_DB, tableName);
     }
 
@@ -1077,8 +1085,8 @@ public class HiveHookIT {
         assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(
                 HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, dbName, tableNames[0]),
"id"));
         assertColumnIsNotRegistered(HiveMetaStoreBridge
-                .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
dbName, tableNames[0]),
-                    HiveDataModelGenerator.NAME));
+            .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
dbName, tableNames[0]),
+                HiveDataModelGenerator.NAME));
 
         for(int i = 0; i < numTables; i++) {
             assertTableIsNotRegistered(dbName, tableNames[i]);
@@ -1152,8 +1160,8 @@ public class HiveHookIT {
                 .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName),
                     "id"));
         assertColumnIsNotRegistered(HiveMetaStoreBridge
-                .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName),
-                    HiveDataModelGenerator.NAME));
+            .getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName),
+                HiveDataModelGenerator.NAME));
         assertTableIsNotRegistered(DEFAULT_DB, viewName);
     }
 
@@ -1348,7 +1356,7 @@ public class HiveHookIT {
         LOG.debug("Searching for table {}.{}", dbName, tableName);
         String tableQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
dbName, tableName, isTemporary);
         return assertEntityIsRegistered(HiveDataTypes.HIVE_TABLE.getName(), AtlasClient.NAME,
tableQualifiedName,
-                assertPredicate);
+            assertPredicate);
     }
 
     private String assertTableIsRegistered(String dbName, String tableName, AssertPredicate
assertPredicate) throws Exception {
@@ -1373,7 +1381,7 @@ public class HiveHookIT {
             public void evaluate() throws Exception {
                 Referenceable entity = atlasClient.getEntity(typeName, property, value);
                 assertNotNull(entity);
-                if(assertPredicate != null) {
+                if (assertPredicate != null) {
                     assertPredicate.assertOnEntity(entity);
                 }
             }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/07b8b4d3/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f4d9054..f68e86c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -20,6 +20,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file
(dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via
shwethags)
 
 ALL CHANGES:
+ATLAS-758 hdfs location of hive table is pointing to old location even after rename ( sumasai
)
 ATLAS-667 Entity delete should check for required reverse references ( dkantor via sumasai
)
 ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)
 ATLAS-692 Create abstraction layer for graph databases (jnhagelb via yhemanth)


Mime
View raw message