hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vik...@apache.org
Subject svn commit: r1580671 - in /hive/trunk/serde/src: java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Date Mon, 24 Mar 2014 00:18:55 GMT
Author: vikram
Date: Mon Mar 24 00:18:55 2014
New Revision: 1580671

URL: http://svn.apache.org/r1580671
Log:
HIVE-6707 : Lazy maps are broken (LazyMap and LazyBinaryMap) (Prasanth J via Vikram Dixit,
reviewed by Gunther)

Modified:
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=1580671&r1=1580670&r2=1580671&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java Mon Mar
24 00:18:55 2014
@@ -67,6 +67,23 @@ public abstract class LazyPrimitive<OI e
     return isNull ? 0 : data.hashCode();
   }
 
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof LazyPrimitive<?, ?>)) {
+      return false;
+    }
+
+    if (data == obj) {
+      return true;
+    }
+
+    if (data == null || obj == null) {
+      return false;
+    }
+
+    return data.equals(((LazyPrimitive<?, ?>) obj).getWritableObject());
+  }
+
   public void logExceptionMessage(ByteArrayRef bytes, int start, int length, String dataType)
{
     try {
       if(LOG.isDebugEnabled()) {

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java?rev=1580671&r1=1580670&r2=1580671&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
Mon Mar 24 00:18:55 2014
@@ -65,4 +65,21 @@ public abstract class LazyBinaryPrimitiv
   public int hashCode() {
     return data == null ? 0 : data.hashCode();
   }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof LazyBinaryPrimitive<?, ?>)) {
+      return false;
+    }
+
+    if (data == obj) {
+      return true;
+    }
+
+    if (data == null || obj == null) {
+      return false;
+    }
+
+    return data.equals(((LazyBinaryPrimitive<?, ?>) obj).getWritableObject());
+  }
 }

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1580671&r1=1580670&r2=1580671&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Mon Mar 24 00:18:55 2014
@@ -31,6 +31,9 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap;
+import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -179,7 +182,7 @@ public class TestLazyArrayMapStruct exte
             .getMapValueElement(new Text("8"))).getWritableObject());
         assertNull(b.getMapValueElement(new Text("-")));
 
-        assertEquals("{'2':'d\\tf','2':'d','-1':null,'0':'0','8':'abc'}"
+        assertEquals("{'2':'d\\tf','-1':null,'0':'0','8':'abc'}"
             .replace('\'', '\"'), SerDeUtils.getJSONString(b, oi));
       }
 
@@ -190,6 +193,67 @@ public class TestLazyArrayMapStruct exte
   }
 
   /**
+   * Test the LazyMap class.
+   */
+  public void testLazyMapWithDuplicateKeys() throws Throwable {
+    try {
+      {
+        // Map of Integer to String
+        Text nullSequence = new Text("\\N");
+        ObjectInspector oi = LazyFactory
+            .createLazyObjectInspector(TypeInfoUtils
+            .getTypeInfosFromTypeString("map<int,string>").get(0),
+            new byte[] {(byte) 1, (byte) 2}, 0, nullSequence, false,
+            (byte) 0);
+        LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+        byte[] data = new byte[] {'2', 2, 'd', 'e', 'f', 1, '-', '1', 2, '\\',
+            'N', 1, '0', 2, '0', 1, '2', 2, 'a', 'b', 'c'};
+        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+        assertEquals(new Text("def"), ((LazyString) b
+            .getMapValueElement(new IntWritable(2))).getWritableObject());
+        assertNull(b.getMapValueElement(new IntWritable(-1)));
+        assertEquals(new Text("0"), ((LazyString) b
+            .getMapValueElement(new IntWritable(0))).getWritableObject());
+        assertNull(b.getMapValueElement(new IntWritable(12345)));
+
+        assertEquals("{2:'def',-1:null,0:'0'}".replace('\'', '\"'),
+            SerDeUtils.getJSONString(b, oi));
+
+        assertEquals(3, b.getMap().size());
+      }
+
+      {
+        // Map of String to String
+        Text nullSequence = new Text("\\N");
+        ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+            TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+            0), new byte[] {(byte) '#', (byte) '\t'}, 0, nullSequence,
+            false, (byte) 0);
+        LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+        byte[] data = new byte[] {'2', '\t', 'd', '\t', 'f', '#', '2', '\t',
+            'd', '#', '-', '1', '#', '0', '\t', '0', '#', '2', '\t', 'a', 'b', 'c'};
+        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+        assertEquals(new Text("d\tf"), ((LazyString) b
+            .getMapValueElement(new Text("2"))).getWritableObject());
+        assertNull(b.getMapValueElement(new Text("-1")));
+        assertEquals(new Text("0"), ((LazyString) b
+            .getMapValueElement(new Text("0"))).getWritableObject());
+
+        assertEquals("{'2':'d\\tf','-1':null,'0':'0'}"
+            .replace('\'', '\"'), SerDeUtils.getJSONString(b, oi));
+
+        assertEquals(3, b.getMap().size());
+      }
+
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  /**
    * Test the LazyStruct class.
    */
   public void testLazyStruct() throws Throwable {
@@ -243,7 +307,7 @@ public class TestLazyArrayMapStruct exte
         data = new Text(": : : :");
         TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
             .getLength());
-        assertEquals("{'a':null,'b':['',''],'c':{'':null,'':null},'d':':'}"
+        assertEquals("{'a':null,'b':['',''],'c':{'':null},'d':':'}"
             .replace("'", "\""), SerDeUtils.getJSONString(o, oi));
 
         data = new Text("= = = =");



Mime
View raw message