hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject hive git commit: HIVE-13115: MetaStore Direct SQL getPartitions call fail when the columns schemas for a partition are null (Ratandeep Ratti reviewed by Carl Steinbach)
Date Sun, 27 Mar 2016 23:09:19 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1.2 2f8175bd4 -> 510ef503b


HIVE-13115: MetaStore Direct SQL getPartitions call fail when the columns schemas for a partition
are null (Ratandeep Ratti reviewed by Carl Steinbach)

(cherry picked from commit 69cfd357eb482c426161aa3c4a00d574ee29416e)
(cherry picked from commit 3a39aba340e1cac60fe27d920e40d816f48ef38f)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/510ef503
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/510ef503
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/510ef503

Branch: refs/heads/branch-1.2
Commit: 510ef503ba7467c0784c1e0c03129d7ed2986fe5
Parents: 2f8175b
Author: Carl Steinbach <cws@apache.org>
Authored: Sun Mar 27 15:41:38 2016 -0700
Committer: Carl Steinbach <cws@apache.org>
Committed: Sun Mar 27 16:05:50 2016 -0700

----------------------------------------------------------------------
 .../hive/metastore/TestHiveMetaStore.java       | 20 +++++++++++-
 .../hive/metastore/MetaStoreDirectSql.java      | 32 +++++++++++---------
 2 files changed, 37 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/510ef503/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 6efb334..6f9e9c7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -273,6 +273,24 @@ public abstract class TestHiveMetaStore extends TestCase {
       }
       assertTrue("Partitions are not same", part.equals(part_get));
 
+      // check null cols schemas for a partition
+      List<String> vals6 = makeVals("2016-02-22 00:00:00", "16");
+      Partition part6 = makePartitionObject(dbName, tblName, vals6, tbl, "/part5");
+      part6.getSd().setCols(null);
+      LOG.info("Creating partition will null field schema");
+      client.add_partition(part6);
+      LOG.info("Listing all partitions for table " + dbName + "." + tblName);
+      final List<Partition> partitions = client.listPartitions(dbName, tblName, (short)
-1);
+      boolean foundPart = false;
+      for (Partition p : partitions) {
+        if (p.getValues().equals(vals6)) {
+          assertNull(p.getSd().getCols());
+          LOG.info("Found partition " + p + " having null field schema");
+          foundPart = true;
+        }
+      }
+      assertTrue(foundPart);
+
       String partName = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=14";
       String part2Name = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=15";
       String part3Name = "ds=" + FileUtils.escapePathName("2008-07-02 14:13:12") + "/hr=15";
@@ -306,7 +324,7 @@ public abstract class TestHiveMetaStore extends TestCase {
       partialVals.clear();
       partialVals.add("");
       partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1);
-      assertTrue("Should have returned 4 partition names", partialNames.size() == 4);
+      assertTrue("Should have returned 5 partition names", partialNames.size() == 5);
       assertTrue("Not all part names returned", partialNames.containsAll(partNames));
 
       // Test partition listing with a partial spec - hr is specified but ds is not

http://git-wip-us.apache.org/repos/asf/hive/blob/510ef503/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 071a4c4..eb0fec2 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -531,14 +531,14 @@ class MetaStoreDirectSql {
       Long sdId = extractSqlLong(fields[1]);
       Long colId = extractSqlLong(fields[2]);
       Long serdeId = extractSqlLong(fields[3]);
-      // A partition must have either everything set, or nothing set if it's a view.
-      if (sdId == null || colId == null || serdeId == null) {
+      // A partition must have at least sdId and serdeId set, or nothing set if it's a view.
+      if (sdId == null || serdeId == null) {
         if (isView == null) {
           isView = isViewTable(dbName, tblName);
         }
         if ((sdId != null || colId != null || serdeId != null) || !isView) {
-          throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + ", column
"
-              + colId + ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view");
+          throw new MetaException("Unexpected null for one of the IDs, SD " + sdId +
+                  ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view");
         }
       }
 
@@ -554,7 +554,7 @@ class MetaStoreDirectSql {
       partitions.put(partitionId, part);
 
       if (sdId == null) continue; // Probably a view.
-      assert colId != null && serdeId != null;
+      assert serdeId != null;
 
       // We assume each partition has an unique SD.
       StorageDescriptor sd = new StorageDescriptor();
@@ -579,14 +579,16 @@ class MetaStoreDirectSql {
       sdSb.append(sdId).append(",");
       part.setSd(sd);
 
-      List<FieldSchema> cols = colss.get(colId);
-      // We expect that colId will be the same for all (or many) SDs.
-      if (cols == null) {
-        cols = new ArrayList<FieldSchema>();
-        colss.put(colId, cols);
-        colsSb.append(colId).append(",");
+      if (colId != null) {
+        List<FieldSchema> cols = colss.get(colId);
+        // We expect that colId will be the same for all (or many) SDs.
+        if (cols == null) {
+          cols = new ArrayList<FieldSchema>();
+          colss.put(colId, cols);
+          colsSb.append(colId).append(",");
+        }
+        sd.setCols(cols);
       }
-      sd.setCols(cols);
 
       // We assume each SD has an unique serde.
       SerDeInfo serde = new SerDeInfo();
@@ -628,8 +630,10 @@ class MetaStoreDirectSql {
       assert serdeSb.length() == 0 && colsSb.length() == 0;
       return orderedResult; // No SDs, probably a view.
     }
-    String sdIds = trimCommaList(sdSb), serdeIds = trimCommaList(serdeSb),
-        colIds = trimCommaList(colsSb);
+
+    String sdIds = trimCommaList(sdSb);
+    String serdeIds = trimCommaList(serdeSb);
+    String colIds = trimCommaList(colsSb);
 
     // Get all the stuff for SD. Don't do empty-list check - we expect partitions do have
SDs.
     queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"SD_PARAMS\""


Mime
View raw message