hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject svn commit: r1631487 - in /hive/trunk/serde/src: java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Date Mon, 13 Oct 2014 17:54:39 GMT
Author: jdere
Date: Mon Oct 13 17:54:38 2014
New Revision: 1631487

URL: http://svn.apache.org/r1631487
Log:
HIVE-8115: Hive select query hang when fields contain map (Xiaobing Zhou via Jason Dere)

Modified:
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=1631487&r1=1631486&r2=1631487&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Mon Oct 13 17:54:38
2014
@@ -23,6 +23,8 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.io.Text;
@@ -36,6 +38,7 @@ import org.apache.hadoop.io.Text;
  */
 public class LazyMap extends LazyNonPrimitive<LazyMapObjectInspector> {
 
+  public static final Log LOG = LogFactory.getLog(LazyMap.class);
   /**
    * Whether the data is already parsed or not.
    */
@@ -170,15 +173,19 @@ public class LazyMap extends LazyNonPrim
         valueLength[mapSize] = elementByteEnd - (keyEnd[mapSize] + 1);
         LazyPrimitive<?, ?> lazyKey = uncheckedGetKey(mapSize);
         if (lazyKey == null) {
-          continue;
-        }
-        Object key = lazyKey.getObject();
-        if(!keySet.contains(key)) {
-          mapSize++;
-          keySet.add(key);
-        } else {
+          LOG.warn("skipped empty entry or entry with empty key in the representation of
column with MAP type.");
+          //reset keyInited[mapSize] flag, since it may be set to true in the case of previous
empty entry
           keyInited[mapSize] = false;
+        } else {
+          Object key = lazyKey.getObject();
+          if(!keySet.contains(key)) {
+            mapSize++;
+            keySet.add(key);
+          } else {
+            keyInited[mapSize] = false;
+          }
         }
+
         // reset keyValueSeparatorPosition
         keyValueSeparatorPosition = -1;
         elementByteBegin = elementByteEnd + 1;

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1631487&r1=1631486&r2=1631487&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Mon Oct 13 17:54:38 2014
@@ -194,6 +194,113 @@ public class TestLazyArrayMapStruct exte
     }
   }
 
+  /*
+   * test LazyMap with bad entries, e.g., empty key or empty entries
+   * where '[' and  ']' don't exist, only for notation purpose,
+   * STX with value of 2 as entry separator, ETX with 3 as key/value separator
+   * */
+  public void testLazyMapWithBadEntries() throws Throwable {
+    try {
+      {
+        // Map of String to String
+        Text nullSequence = new Text("");
+        ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+            TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+            0), new byte[] {'\2', '\3'}, 0, nullSequence,
+            false, (byte) 0);
+        LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+       //read friendly string: ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+       byte[] data = new byte[] {
+            'a', 'k', '\3', 'a', 'v',
+            '\02', 'b', 'k', '\3', 'b', 'v',
+            '\02', 'c', 'k', '\3', 'c', 'v',
+            '\02', 'd', 'k', '\3', 'd', 'v'};
+        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+        assertEquals(new Text("av"), ((LazyString) b
+            .getMapValueElement(new Text("ak"))).getWritableObject());
+        assertNull(b.getMapValueElement(new Text("-1")));
+        assertEquals(new Text("bv"), ((LazyString) b
+            .getMapValueElement(new Text("bk"))).getWritableObject());
+        assertEquals(new Text("cv"), ((LazyString) b
+            .getMapValueElement(new Text("ck"))).getWritableObject());
+        assertNull(b.getMapValueElement(new Text("-")));
+        assertEquals(new Text("dv"), ((LazyString) b
+            .getMapValueElement(new Text("dk"))).getWritableObject());
+        assertEquals(4, b.getMapSize());
+      }
+
+      {
+        // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null}
+        // or {"", ""}, but not null style key, e.g., {null:""}
+        Text nullSequence = new Text("");
+        ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+            TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+            0), new byte[] {'\2', '\3'}, 0, nullSequence,
+            false, (byte) 0);
+        LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+       //read friendly string: [STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+        byte[] data = new byte[] {
+            '\02', 'a', 'k', '\3', 'a', 'v',
+            '\02', 'b', 'k', '\3', 'b', 'v',
+            '\02', 'c', 'k', '\3', 'c', 'v',
+            '\02', 'd', 'k', '\3', 'd', 'v'};
+        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+        assertNull(b.getMapValueElement(new Text(""))); //{"" : null}
+        assertEquals(new Text("av"), ((LazyString) b
+            .getMapValueElement(new Text("ak"))).getWritableObject());
+        assertNull(b.getMapValueElement(new Text("-1")));
+        assertEquals(new Text("bv"), ((LazyString) b
+            .getMapValueElement(new Text("bk"))).getWritableObject());
+        assertEquals(new Text("cv"), ((LazyString) b
+            .getMapValueElement(new Text("ck"))).getWritableObject());
+        assertNull(b.getMapValueElement(new Text("-")));
+        assertEquals(new Text("dv"), ((LazyString) b
+            .getMapValueElement(new Text("dk"))).getWritableObject());
+        assertEquals(4, b.getMapSize());
+      }
+
+      {
+        // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null}
+        // or {"", ""}, but not null style key, e.g., {null:""}
+        Text nullSequence = new Text("");
+        ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+            TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+            0), new byte[] {'\2', '\3'}, 0, nullSequence,
+            false, (byte) 0);
+        LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+       //read friendly string: [ETX][STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+        byte[] data = new byte[] {
+            '\03',
+            '\02', 'a', 'k', '\3', 'a', 'v',
+            '\02', 'b', 'k', '\3', 'b', 'v',
+            '\02', 'c', 'k', '\3', 'c', 'v',
+            '\02', 'd', 'k', '\3', 'd', 'v'};
+        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+        assertNull(b.getMapValueElement(new Text(""))); //{"" : null}
+        assertEquals(new Text("av"), ((LazyString) b
+            .getMapValueElement(new Text("ak"))).getWritableObject());
+        assertNull(b.getMapValueElement(new Text("-1")));
+        assertEquals(new Text("bv"), ((LazyString) b
+            .getMapValueElement(new Text("bk"))).getWritableObject());
+        assertEquals(new Text("cv"), ((LazyString) b
+            .getMapValueElement(new Text("ck"))).getWritableObject());
+        assertNull(b.getMapValueElement(new Text("-")));
+        assertEquals(new Text("dv"), ((LazyString) b
+            .getMapValueElement(new Text("dk"))).getWritableObject());
+        assertEquals(4, b.getMapSize());
+      }
+    } catch(Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
   /**
    * Test the LazyMap class.
    */



Mime
View raw message