atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suma...@apache.org
Subject incubator-atlas git commit: ATLAS-528 Support drop table, view (sumasai)
Date Sat, 09 Apr 2016 16:54:05 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master b8f4ffb68 -> 755e59c08


ATLAS-528 Support drop table,view (sumasai)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/755e59c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/755e59c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/755e59c0

Branch: refs/heads/master
Commit: 755e59c08dbbc3b03fc2f6fc4908396433b97e88
Parents: b8f4ffb
Author: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Authored: Sat Apr 9 09:53:58 2016 -0700
Committer: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Committed: Sat Apr 9 09:53:58 2016 -0700

----------------------------------------------------------------------
 .../atlas/hive/bridge/HiveMetaStoreBridge.java  | 12 ++-
 .../org/apache/atlas/hive/hook/HiveHook.java    | 22 +++++
 .../org/apache/atlas/hive/hook/HiveHookIT.java  | 88 +++++++++++++++++++-
 release-log.txt                                 |  1 +
 4 files changed, 115 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index eb5f1e6..6b348e2 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -335,8 +335,8 @@ public class HiveMetaStoreBridge {
 
         tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(),
tableQualifiedName));
 
-        // add reference to the StorageDescriptorx
-        Referenceable sdReferenceable = fillStorageDescStruct(hiveTable.getSd(), tableQualifiedName,
tableQualifiedName);
+        // add reference to the StorageDescriptor
+        Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName,
getStorageDescQFName(tableQualifiedName));
         tableReference.set("sd", sdReferenceable);
 
         // add reference to the Partition Keys
@@ -359,6 +359,10 @@ public class HiveMetaStoreBridge {
         return tableReference;
     }
 
+    private String getStorageDescQFName(String entityQualifiedName) {
+        return entityQualifiedName + "_storage";
+    }
+
     private Referenceable registerTable(Referenceable dbReference, Table table) throws Exception
{
         String dbName = table.getDbName();
         String tableName = table.getTableName();
@@ -410,7 +414,7 @@ public class HiveMetaStoreBridge {
         return new Referenceable(sd.getId().id, sd.getTypeName(), null);
     }
 
-    public Referenceable fillStorageDescStruct(StorageDescriptor storageDesc, String tableQualifiedName,
+    public Referenceable fillStorageDesc(StorageDescriptor storageDesc, String tableQualifiedName,
         String sdQualifiedName) throws Exception {
         LOG.debug("Filling storage descriptor information for " + storageDesc);
 
@@ -466,7 +470,7 @@ public class HiveMetaStoreBridge {
         ref.set("path", pathUri);
 //        Path path = new Path(pathUri);
 //        ref.set("name", path.getName());
-//        TODO - Fix after ATLAS-542 to shorter Name
+        //TODO - Fix after ATLAS-542 to shorter Name
         ref.set("name", pathUri);
         ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
         return ref;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 4102263..505e5e7 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -308,6 +308,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext
{
         case EXPORT:
         case IMPORT:
         case QUERY:
+        case TRUNCATETABLE:
             registerProcess(dgiBridge, event);
             break;
 
@@ -326,6 +327,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext
{
         case ALTERTABLE_ADDCOLS:
         case ALTERTABLE_REPLACECOLS:
         case ALTERTABLE_RENAMECOL:
+        case ALTERTABLE_PARTCOLTYPE:
             handleEventOutputs(dgiBridge, event, Type.TABLE);
             break;
         case ALTERTABLE_LOCATION:
@@ -334,17 +336,37 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext
{
                 //Track altered lineage in case of external tables
                 handleExternalTables(dgiBridge, event, tablesUpdated.get(0).getLeft(), tablesUpdated.get(0).getRight());
             }
+            break;
         case ALTERDATABASE:
         case ALTERDATABASE_OWNER:
             handleEventOutputs(dgiBridge, event, Type.DATABASE);
             break;
 
+        case DROPTABLE:
+        case DROPVIEW:
+            deleteTable(dgiBridge, event);
+            break;
+
         default:
         }
 
         notifyEntities(messages);
     }
 
+    private void deleteTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) {
+        for (Entity output : event.outputs) {
+            if (Type.TABLE.equals(output.getType())) {
+                final String tblQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(),
output.getTable().getDbName(), output.getTable().getTableName());
+                LOG.info("Deleting table {} ", tblQualifiedName);
+                messages.add(
+                    new HookNotification.EntityDeleteRequest(event.getUser(),
+                        HiveDataTypes.HIVE_TABLE.getName(),
+                        HiveDataModelGenerator.NAME,
+                        tblQualifiedName));
+            }
+        }
+    }
+
     private void renameTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) throws
Exception {
         //crappy, no easy of getting new name
         assert event.getInputs() != null && event.getInputs().size() == 1;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 8ef8479..7a09b47 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -497,7 +497,6 @@ public class HiveHookIT {
         runCommand(query);
         Referenceable processReference = validateProcess(query, 1, 1);
         validateHDFSPaths(processReference, filename, OUTPUTS);
-
         validateInputTables(processReference, tableId);
 
         //Import
@@ -510,7 +509,6 @@ public class HiveHookIT {
         validateHDFSPaths(processReference, filename, INPUTS);
 
         validateOutputTables(processReference, tableId);
-
     }
 
     @Test
@@ -541,8 +539,6 @@ public class HiveHookIT {
         validateHDFSPaths(processReference, filename, INPUTS);
 
         validateOutputTables(processReference, tableId);
-
-
     }
 
     @Test
@@ -684,6 +680,41 @@ public class HiveHookIT {
         Assert.assertEquals(columns.get(0).get(HiveDataModelGenerator.NAME), "id");
     }
 
+    @Test()
+    public void testTruncateTable() throws Exception {
+        String tableName = createTable(false);
+        String query = String.format("truncate table %s", tableName);
+        runCommand(query);
+
+        String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
+        validateProcess(query, 0, 1);
+
+        //Check lineage
+        String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB,
tableName);
+        JSONObject response = dgiCLient.getInputGraph(datasetName);
+        JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
+        //Below should be assertTrue - Fix https://issues.apache.org/jira/browse/ATLAS-653
+        Assert.assertFalse(vertices.has(tableId));
+    }
+
+    @Test
+    public void testAlterTablePartitionColumnType() throws Exception {
+        String tableName = createTable(true, true, false);
+        final String newType = "int";
+        String query = String.format("ALTER TABLE %s PARTITION COLUMN (dt %s)", tableName,
newType);
+        runCommand(query);
+
+        final String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
+        final String dtColId = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName), "dt"));
+        Referenceable table = dgiCLient.getEntity(tableId);
+        Referenceable column = dgiCLient.getEntity(dtColId);
+        Assert.assertEquals(column.get("type"), newType);
+
+        final List<Referenceable> partitionKeys = (List<Referenceable>) table.get("partitionKeys");
+        Assert.assertEquals(partitionKeys.size(), 1);
+        Assert.assertEquals(partitionKeys.get(0).getId()._getId(), dtColId);
+    }
+
     @Test
     public void testAlterViewRename() throws Exception {
         String tableName = createTable();
@@ -834,7 +865,56 @@ public class HiveHookIT {
 
         //Add another property
         runSerdePropsQuery(tableName, expectedProps);
+    }
+
+    @Test
+    public void testDropTable() throws Exception {
+        //Test Deletion of tables and its corrresponding columns
+        String tableName = createTable(true, true, false);
 
+        assertTableIsRegistered(DEFAULT_DB, tableName);
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName), "id"));
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName), "name"));
+
+        final String query = String.format("drop table %s ", tableName);
+
+        runCommand(query);
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName), "id"));
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, tableName), "name"));
+        assertTableIsNotRegistered(DEFAULT_DB, tableName);
+    }
+
+    @Test
+    public void testDropNonExistingTable() throws Exception {
+        //Test Deletion of a non existing table
+        final String tableName = "nonexistingtable";
+        assertTableIsNotRegistered(DEFAULT_DB, tableName);
+        final String query = String.format("drop table if exists %s", tableName);
+        runCommand(query);
+
+        //Should have no effect
+        assertTableIsNotRegistered(DEFAULT_DB, tableName);
+        assertProcessIsNotRegistered(query);
+    }
+
+    @Test
+    public void testDropView() throws Exception {
+        //Test Deletion of tables and its corrresponding columns
+        String tableName = createTable(true, true, false);
+        String viewName = tableName();
+        String query = "create view " + viewName + " as select * from " + tableName;
+        runCommand(query);
+
+        assertTableIsRegistered(DEFAULT_DB, viewName);
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName), "id"));
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName), "name"));
+
+        query = String.format("drop view %s ", viewName);
+
+        runCommand(query);
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName), "id"));
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName), "name"));
+        assertTableIsNotRegistered(DEFAULT_DB, viewName);
     }
 
     private void runSerdePropsQuery(String tableName, Map<String, String> expectedProps)
throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a2fd4ea..2125f87 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file
(dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via
shwethags)
 
 ALL CHANGES:
+ATLAS-528 Support drop table,view (sumasai)
 ATLAS-603 Document High Availability of Atlas (yhemanth via sumasai)
 ATLAS-498 Support Embedded HBase (tbeerbower via sumasai)
 ATLAS-527 Support lineage for load table, import, export (sumasai via shwethags)


Mime
View raw message