hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1635579 - in /hive/branches/branch-0.14/hcatalog: core/src/main/java/org/apache/hive/hcatalog/data/ hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/ hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/
Date Thu, 30 Oct 2014 18:29:36 GMT
Author: daijy
Date: Thu Oct 30 18:29:35 2014
New Revision: 1635579

URL: http://svn.apache.org/r1635579
Log:
HIVE-7282: HCatLoader fail to load Orc map with null key (Daniel Dai)

Modified:
    hive/branches/branch-0.14/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java
    hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
    hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java

Modified: hive/branches/branch-0.14/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java?rev=1635579&r1=1635578&r2=1635579&view=diff
==============================================================================
--- hive/branches/branch-0.14/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java
(original)
+++ hive/branches/branch-0.14/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java
Thu Oct 30 18:29:35 2014
@@ -23,7 +23,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.TreeMap;
+import java.util.HashMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.serde.serdeConstants;
@@ -212,7 +212,7 @@ public class HCatRecordSerDe implements 
   private static Map<?, ?> serializeMap(Object f, MapObjectInspector moi) throws SerDeException
{
     ObjectInspector koi = moi.getMapKeyObjectInspector();
     ObjectInspector voi = moi.getMapValueObjectInspector();
-    Map<Object, Object> m = new TreeMap<Object, Object>();
+    Map<Object, Object> m = new HashMap<Object, Object>();
 
     Map<?, ?> readMap = moi.getMap(f);
     if (readMap == null) {

Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1635579&r1=1635578&r2=1635579&view=diff
==============================================================================
--- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
(original)
+++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
Thu Oct 30 18:29:35 2014
@@ -480,7 +480,9 @@ class PigHCatUtil {
     Map<String, Object> result = new HashMap<String, Object>();
     for (Entry<?, ?> entry : map.entrySet()) {
       // since map key for Pig has to be Strings
-      result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0)));
+      if (entry.getKey()!=null) {
+        result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0)));
+      }
     }
     return result;
   }

Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1635579&r1=1635578&r2=1635579&view=diff
==============================================================================
--- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
(original)
+++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
Thu Oct 30 18:29:35 2014
@@ -18,8 +18,6 @@
  */
 package org.apache.hive.hcatalog.pig;
 
-import com.google.common.collect.ImmutableSet;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -52,7 +50,6 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
 
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -79,11 +76,13 @@ public class TestHCatLoaderComplexSchema
           add("testSyntheticComplexSchema");
           add("testTupleInBagInTupleInBag");
           add("testMapWithComplexData");
+          add("testMapNullKey");
         }});
         put(IOConstants.PARQUETFILE, new HashSet<String>() {{
           add("testSyntheticComplexSchema");
           add("testTupleInBagInTupleInBag");
           add("testMapWithComplexData");
+          add("testMapNullKey");
         }});
       }};
 
@@ -223,6 +222,10 @@ public class TestHCatLoaderComplexSchema
 
   private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple>
data, boolean provideSchemaToStorer)
     throws IOException, CommandNeedRetryException, ExecException, FrontendException {
+    verifyWriteRead(tablename, pigSchema, tableSchema, data, data, provideSchemaToStorer);
+  }
+  private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple>
data, List<Tuple> result, boolean provideSchemaToStorer)
+    throws IOException, CommandNeedRetryException, ExecException, FrontendException {
     MockLoader.setData(tablename + "Input", data);
     try {
       createTable(tablename, tableSchema);
@@ -244,7 +247,7 @@ public class TestHCatLoaderComplexSchema
       Iterator<Tuple> it = server.openIterator("X");
       int i = 0;
       while (it.hasNext()) {
-        Tuple input = data.get(i++);
+        Tuple input = result.get(i++);
         Tuple output = it.next();
         compareTuples(input, output);
         LOG.info("tuple : {} ", output);
@@ -354,4 +357,40 @@ public class TestHCatLoaderComplexSchema
     verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true);
     verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false);
   }
+
+  /**
+   * artificially complex nested schema to test nested schema conversion
+   * @throws Exception
+   */
+  @Test
+  public void testMapNullKey() throws Exception {
+    assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
+    String pigSchema = "m:map[]";
+
+    String tableSchema = "m map<string, string>";
+
+    List<Tuple> data = new ArrayList<Tuple>();
+    Tuple t = t(
+      new HashMap<String, String>() {
+      {
+        put("ac test1", "test 1");
+        put("ac test2", "test 2");
+        put(null, "test 3");
+      };
+    });
+    data.add(t);
+
+    List<Tuple> result = new ArrayList<Tuple>();
+    t = t(
+      new HashMap<String, String>() {
+      {
+        put("ac test1", "test 1");
+        put("ac test2", "test 2");
+      };
+    });
+    result.add(t);
+
+    verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, true);
+    verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, false);
+  }
 }



Mime
View raw message