atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject incubator-atlas git commit: ATLAS-588 import-hive.sh fails while importing partitions for a non-partitioned table (sumasai via shwethags)
Date Thu, 24 Mar 2016 06:04:26 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master dd66e1580 -> 985465fc7


ATLAS-588 import-hive.sh fails while importing partitions for a non-partitioned table (sumasai
via shwethags)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/985465fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/985465fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/985465fc

Branch: refs/heads/master
Commit: 985465fc73be355986e326ee3d491d3b6a077e05
Parents: dd66e15
Author: Shwetha GS <sshivalingamurthy@hortonworks.com>
Authored: Thu Mar 24 11:34:16 2016 +0530
Committer: Shwetha GS <sshivalingamurthy@hortonworks.com>
Committed: Thu Mar 24 11:34:16 2016 +0530

----------------------------------------------------------------------
 .../atlas/hive/bridge/HiveMetaStoreBridge.java  |  7 ++++-
 .../hive/bridge/HiveMetaStoreBridgeTest.java    | 31 ++++++++++++++++++++
 release-log.txt                                 |  1 +
 3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index ea38750..0680e65 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -18,6 +18,7 @@
 
 package org.apache.atlas.hive.bridge;
 
+import com.google.common.base.Joiner;
 import com.sun.jersey.api.client.ClientResponse;
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
@@ -435,7 +436,11 @@ public class HiveMetaStoreBridge {
         List<Partition> tableParts = hiveClient.getPartitions(table);
 
         for (Partition hivePart : tableParts) {
-            registerPartition(tableReferenceable, sdReferenceable, hivePart);
+            if (hivePart.getValues() != null && hivePart.getValues().size() >
0) {
+                registerPartition(tableReferenceable, sdReferenceable, hivePart);
+            } else {
+                LOG.info("Skipping partition for table {} since partition values are {}",
getTableQualifiedName(clusterName, table.getDbName(), table.getTableName()), StringUtils.join(hivePart.getValues(),
","));
+            }
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
index f8cfb71..c717c0f 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
@@ -35,6 +35,7 @@ import org.codehaus.jettison.json.JSONObject;
 import org.mockito.ArgumentMatcher;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
+import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 import scala.actors.threadpool.Arrays;
@@ -163,6 +164,34 @@ public class HiveMetaStoreBridgeTest {
                         new Integer(lastAccessTime))));
     }
 
+    @Test
+    public void testImportWhenPartitionKeysAreNull() throws Exception {
+        setupDB(hiveClient, TEST_DB_NAME);
+        Table hiveTable = setupTable(hiveClient, TEST_DB_NAME, TEST_TABLE_NAME);
+
+        returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME);
+
+        when(atlasClient.searchByDSL(HiveMetaStoreBridge.getTableDSLQuery(CLUSTER_NAME, TEST_DB_NAME,
+            TEST_TABLE_NAME,
+            HiveDataTypes.HIVE_TABLE.getName()))).thenReturn(
+            getEntityReference("82e06b34-9151-4023-aa9d-b82103a50e77"));
+        when(atlasClient.getEntity("82e06b34-9151-4023-aa9d-b82103a50e77")).thenReturn(createTableReference());
+
+        Partition partition = mock(Partition.class);
+        when(partition.getTable()).thenReturn(hiveTable);
+        List partitionValues = Arrays.asList(new String[]{});
+        when(partition.getValues()).thenReturn(partitionValues);
+
+        when(hiveClient.getPartitions(hiveTable)).thenReturn(Arrays.asList(new Partition[]{partition}));
+
+        HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient);
+        try {
+            bridge.importHiveMetadata();
+        } catch (Exception e) {
+            Assert.fail("Partition with null key caused import to fail with exception ",
e);
+        }
+    }
+
     private JSONArray getPartitionReference(String id) throws JSONException {
         JSONObject resultEntry = new JSONObject();
         resultEntry.put(HiveMetaStoreBridge.SEARCH_ENTRY_GUID_ATTR, id);
@@ -203,4 +232,6 @@ public class HiveMetaStoreBridgeTest {
             return attrValue.equals(((Referenceable) o).get(attrName));
         }
     }
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 02f2528..4569e55 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file
(dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via
shwethags)
 
 ALL CHANGES:
+ATLAS-588 import-hive.sh fails while importing partitions for a non-partitioned table (sumasai
via shwethags)
 ATLAS-575 jetty-maven-plugin fails with ShutdownMonitorThread already started (shwethags)
 ATLAS-408 UI : Add a close link (x) on the top right when Tag is added (darshankumar89 via
shwethags)
 ATLAS-524 Support alter database (sumasai via shwethags)


Mime
View raw message