atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suma...@apache.org
Subject incubator-atlas git commit: ATLAS-523 Support alter view ( sumasai via shwethags)
Date Mon, 14 Mar 2016 17:53:11 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 44dbfe576 -> ad7604fcf


ATLAS-523 Support alter view ( sumasai via shwethags)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ad7604fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ad7604fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ad7604fc

Branch: refs/heads/master
Commit: ad7604fcf4bf4f84a66b339b8ea1f4473cd9f84a
Parents: 44dbfe5
Author: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Authored: Mon Mar 14 10:52:28 2016 -0700
Committer: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Committed: Mon Mar 14 10:53:00 2016 -0700

----------------------------------------------------------------------
 .../org/apache/atlas/hive/hook/HiveHook.java    |   7 +-
 .../org/apache/atlas/hive/hook/HiveHookIT.java  | 123 +++++++++++++++----
 release-log.txt                                 |   1 +
 3 files changed, 99 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ad7604fc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index b5a4ce9..fc2a57a 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -86,7 +86,6 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext
{
     private static final long keepAliveTimeDefault = 10;
     private static final int queueSizeDefault = 10000;
 
-    private static boolean typesRegistered = false;
     private static Configuration atlasProperties;
 
     class HiveEvent {
@@ -212,6 +211,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext
{
 
         case CREATETABLE_AS_SELECT:
         case CREATEVIEW:
+        case ALTERVIEW_AS:
         case LOAD:
         case EXPORT:
         case IMPORT:
@@ -229,15 +229,12 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext
{
         case ALTERTABLE_CLUSTER_SORT:
         case ALTERTABLE_BUCKETNUM:
         case ALTERTABLE_PROPERTIES:
+        case ALTERVIEW_PROPERTIES:
         case ALTERTABLE_SERDEPROPERTIES:
         case ALTERTABLE_SERIALIZER:
             alterTable(dgiBridge, event);
             break;
 
-        case ALTERVIEW_AS:
-            //update inputs/outputs?
-            break;
-
         case ALTERTABLE_ADDCOLS:
         case ALTERTABLE_REPLACECOLS:
         case ALTERTABLE_RENAMECOL:

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ad7604fc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index cb215db..f2084b0 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -18,8 +18,8 @@
 
 package org.apache.atlas.hive.hook;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
-import groovy.transform.Immutable;
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasServiceException;
@@ -38,13 +38,6 @@ import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
@@ -52,14 +45,7 @@ import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-import java.io.DataInput;
-import java.io.DataOutput;
 import java.io.File;
-import java.io.IOException;
-import java.net.URLClassLoader;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -221,6 +207,55 @@ public class HiveHookIT {
     }
 
     @Test
+    public void testAlterViewAsSelect() throws Exception {
+
+        //Create the view from table1
+        String table1Name = createTable();
+        String viewName = tableName();
+        String query = "create view " + viewName + " as select * from " + table1Name;
+        runCommand(query);
+
+        String table1Id = assertTableIsRegistered(DEFAULT_DB, table1Name);
+        assertProcessIsRegistered(query);
+        String viewId = assertTableIsRegistered(DEFAULT_DB, viewName);
+
+        //Check lineage which includes table1
+        String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB,
viewName);
+        JSONObject response = dgiCLient.getInputGraph(datasetName);
+        JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
+        Assert.assertTrue(vertices.has(viewId));
+        Assert.assertTrue(vertices.has(table1Id));
+
+        //Alter the view from table2
+        String table2Name = createTable();
+        query = "alter view " + viewName + " as select * from " + table2Name;
+        runCommand(query);
+
+        //Check if alter view process is reqistered
+        assertProcessIsRegistered(query);
+        String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name);
+        Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName), viewId);
+
+        //Check lineage which includes both table1 and table2
+        datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB,
viewName);
+        response = dgiCLient.getInputGraph(datasetName);
+        vertices = response.getJSONObject("values").getJSONObject("vertices");
+        Assert.assertTrue(vertices.has(viewId));
+
+        //This is through the alter view process
+        Assert.assertTrue(vertices.has(table2Id));
+
+        //This is through the Create view process
+        Assert.assertTrue(vertices.has(table1Id));
+
+        //Outputs dont exist
+        response = dgiCLient.getOutputGraph(datasetName);
+        vertices = response.getJSONObject("values").getJSONObject("vertices");
+        Assert.assertEquals(vertices.length(), 0);
+    }
+
+
+    @Test
     public void testLoadData() throws Exception {
         String tableName = createTable(false);
 
@@ -520,7 +555,7 @@ public class HiveHookIT {
     }
 
     private String getSerializedProps(Map<String, String> expectedProps) {
-        StringBuffer sb = new StringBuffer();
+        StringBuilder sb = new StringBuilder();
         for(String expectedPropKey : expectedProps.keySet()) {
             if(sb.length() > 0) {
                 sb.append(",");
@@ -535,34 +570,68 @@ public class HiveHookIT {
     @Test
     public void testAlterTableProperties() throws Exception {
         String tableName = createTable();
+        final String fmtQuery = "alter table %s %s TBLPROPERTIES (%s)";
+        testAlterProperties(tableName, fmtQuery);
+    }
+
+    private void testAlterProperties(String tableName, String fmtQuery) throws Exception
{
+
+        final String SET_OP = "set";
+        final String UNSET_OP = "unset";
+
         final Map<String, String> expectedProps = new HashMap<String, String>()
{{
             put("testPropKey1", "testPropValue1");
             put("comment", "test comment");
         }};
 
-        final String fmtQuery = "alter table %s set TBLPROPERTIES (%s)";
-        String query = String.format(fmtQuery, tableName, getSerializedProps(expectedProps));
+        String query = String.format(fmtQuery, tableName, SET_OP, getSerializedProps(expectedProps));
         runCommand(query);
 
-        verifyTableProperties(tableName, expectedProps);
-
+        verifyTableProperties(tableName, expectedProps, false);
 
         expectedProps.put("testPropKey2", "testPropValue2");
         //Add another property
-        query = String.format(fmtQuery, tableName, getSerializedProps(expectedProps));
+        query = String.format(fmtQuery, tableName, SET_OP, getSerializedProps(expectedProps));
+        runCommand(query);
+
+        verifyTableProperties(tableName, expectedProps, false);
+
+        //Unset all the props
+        StringBuilder sb = new StringBuilder("'");
+        query = String.format(fmtQuery, tableName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb,
expectedProps.keySet()).append('\''));
+        runCommand(query);
+
+        verifyTableProperties(tableName, expectedProps, true);
+    }
+
+    @Test
+    public void testAlterViewProperties() throws Exception {
+        String tableName = createTable();
+        String viewName = tableName();
+        String query = "create view " + viewName + " as select * from " + tableName;
         runCommand(query);
 
-        verifyTableProperties(tableName, expectedProps);
+        final String fmtQuery = "alter view %s %s TBLPROPERTIES (%s)";
+        testAlterProperties(viewName, fmtQuery);
     }
 
-    private void verifyTableProperties(String tableName, Map<String, String> expectedProps)
throws Exception {
+    private void verifyTableProperties(String tableName, Map<String, String> expectedProps,
boolean checkIfNotExists) throws Exception {
         String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
         Referenceable tableRef = dgiCLient.getEntity(tableId);
         Map<String, String> parameters = (Map<String, String>) tableRef.get(HiveDataModelGenerator.PARAMETERS);
-        Assert.assertNotNull(parameters);
-        //Comment should exist since SET TBLPOPERTIES only adds properties. Doe not remove
existing ones
-        for (String propKey : expectedProps.keySet()) {
-            Assert.assertEquals(parameters.get(propKey), expectedProps.get(propKey));
+        if (checkIfNotExists == false) {
+            //Check if properties exist
+            Assert.assertNotNull(parameters);
+            for (String propKey : expectedProps.keySet()) {
+                Assert.assertEquals(parameters.get(propKey), expectedProps.get(propKey));
+            }
+        } else {
+            //Check if properties dont exist
+            if (expectedProps != null && parameters != null) {
+                for (String propKey : expectedProps.keySet()) {
+                    Assert.assertFalse(parameters.containsKey(propKey));
+                }
+            }
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ad7604fc/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a4cd781..9c73da2 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file
(dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via
shwethags)
 
 ALL CHANGES:
+ATLAS-523 Support alter view (sumasai via shwethags)
 ATLAS-555 Tag creation from UI fails due to missing description attribute (guptaneeru via
shwethags)
 ATLAS-522 Support Alter table commands (sumasai via shwethags)
 ATLAS-512 Decouple currently integrating components from availability of Atlas service for
raising metadata events ( yhemanth via sumasai)


Mime
View raw message