hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [13/54] [abbrv] hive git commit: HIVE-16207: Add support for Complex Types in Fast SerDe (Teddy Choi, reviewed by Matt McCline)
Date Wed, 24 May 2017 23:51:47 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
index b369462..8356a5c 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hive.serde2.binarysortable;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.io.EOFException;
 import java.util.Arrays;
@@ -30,13 +31,16 @@ import org.apache.hadoop.hive.serde2.SerdeRandomRowSource;
 import org.apache.hadoop.hive.serde2.VerifyFast;
 import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
 import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite;
+import org.apache.hadoop.hive.serde2.lazy.VerifyLazy;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Writable;
 
 import junit.framework.TestCase;
+import org.junit.Assert;
 
 public class TestBinarySortableFast extends TestCase {
 
@@ -48,11 +52,11 @@ public class TestBinarySortableFast extends TestCase {
           boolean[] columnSortOrderIsDesc, byte[] columnNullMarker, byte[] columnNotNullMarker,
           AbstractSerDe serde, StructObjectInspector rowOI,
           AbstractSerDe serde_fewer, StructObjectInspector writeRowOI,
-          boolean ascending, PrimitiveTypeInfo[] primitiveTypeInfos,
+          boolean ascending, TypeInfo[] typeInfos,
           boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable
{
 
     int rowCount = rows.length;
-    int columnCount = primitiveTypeInfos.length;
+    int columnCount = typeInfos.length;
 
     boolean[] columnsToInclude = null;
     if (useIncludeColumns) {
@@ -83,10 +87,7 @@ public class TestBinarySortableFast extends TestCase {
 
       int[] perFieldWriteLengths = new int[columnCount];
       for (int index = 0; index < writeColumnCount; index++) {
-
-        Writable writable = (Writable) row[index];
-
-        VerifyFast.serializeWrite(binarySortableSerializeWrite, primitiveTypeInfos[index],
writable);
+        VerifyFast.serializeWrite(binarySortableSerializeWrite, typeInfos[index], row[index]);
         perFieldWriteLengths[index] = output.getLength();
       }
       perFieldWriteLengthsArray[i] = perFieldWriteLengths;
@@ -95,7 +96,8 @@ public class TestBinarySortableFast extends TestCase {
       bytesWritable.set(output.getData(), 0, output.getLength());
       serializeWriteBytes[i] = bytesWritable;
       if (i > 0) {
-        int compareResult = serializeWriteBytes[i - 1].compareTo(serializeWriteBytes[i]);
+        BytesWritable previousBytesWritable = serializeWriteBytes[i - 1];
+        int compareResult = previousBytesWritable.compareTo(bytesWritable);
         if ((compareResult < 0 && !ascending)
             || (compareResult > 0 && ascending)) {
           System.out.println("Test failed in "
@@ -117,7 +119,7 @@ public class TestBinarySortableFast extends TestCase {
       Object[] row = rows[i];
       BinarySortableDeserializeRead binarySortableDeserializeRead =
               new BinarySortableDeserializeRead(
-                  primitiveTypeInfos,
+                  typeInfos,
                   /* useExternalBuffer */ false,
                   columnSortOrderIsDesc,
                   columnNullMarker,
@@ -132,10 +134,9 @@ public class TestBinarySortableFast extends TestCase {
           binarySortableDeserializeRead.skipNextField();
         } else if (index >= writeColumnCount) {
           // Should come back a null.
-          VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index],
null);
+          VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, typeInfos[index],
null);
         } else {
-          Writable writable = (Writable) row[index];
-          VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index],
writable);
+          verifyRead(binarySortableDeserializeRead, typeInfos[index], row[index]);
         }
       }
       if (writeColumnCount == columnCount) {
@@ -147,7 +148,7 @@ public class TestBinarySortableFast extends TestCase {
        */
       BinarySortableDeserializeRead binarySortableDeserializeRead2 =
           new BinarySortableDeserializeRead(
-              primitiveTypeInfos,
+              typeInfos,
               /* useExternalBuffer */ false,
               columnSortOrderIsDesc,
               columnNullMarker,
@@ -157,22 +158,24 @@ public class TestBinarySortableFast extends TestCase {
           bytesWritable.getBytes(), 0, bytesWritable.getLength() - 1);  // One fewer byte.
 
       for (int index = 0; index < writeColumnCount; index++) {
-        Writable writable = (Writable) row[index];
         if (index == writeColumnCount - 1) {
           boolean threw = false;
           try {
-            VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead2, primitiveTypeInfos[index],
writable);
+            verifyRead(binarySortableDeserializeRead2, typeInfos[index], row[index]);
           } catch (EOFException e) {
 //          debugDetailedReadPositionString = binarySortableDeserializeRead2.getDetailedReadPositionString();
 //          debugStackTrace = e.getStackTrace();
             threw = true;
           }
-          TestCase.assertTrue(threw);
+
+          if (!threw && row[index] != null) {
+            Assert.fail();
+          }
         } else {
           if (useIncludeColumns && !columnsToInclude[index]) {
             binarySortableDeserializeRead2.skipNextField();
           } else {
-            VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead2, primitiveTypeInfos[index],
writable);
+            verifyRead(binarySortableDeserializeRead2, typeInfos[index], row[index]);
           }
         }
       }
@@ -270,7 +273,7 @@ public class TestBinarySortableFast extends TestCase {
               "\nSerDe: " +
                   serDeFields.toString() +
               "\nperFieldWriteLengths " + Arrays.toString(perFieldWriteLengthsArray[i]) +
-              "\nprimitiveTypeInfos " + Arrays.toString(primitiveTypeInfos) +
+              "\nprimitiveTypeInfos " + Arrays.toString(typeInfos) +
               "\nrow " + Arrays.toString(row));
         }
       }
@@ -282,7 +285,7 @@ public class TestBinarySortableFast extends TestCase {
       Object[] row = rows[i];
       BinarySortableDeserializeRead binarySortableDeserializeRead =
               new BinarySortableDeserializeRead(
-                  primitiveTypeInfos,
+                  typeInfos,
                   /* useExternalBuffer */ false,
                   columnSortOrderIsDesc,
                   columnNullMarker,
@@ -297,10 +300,9 @@ public class TestBinarySortableFast extends TestCase {
           binarySortableDeserializeRead.skipNextField();
         } else if (index >= writeColumnCount) {
           // Should come back a null.
-          VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index],
null);
+          verifyRead(binarySortableDeserializeRead, typeInfos[index], null);
         } else {
-          Writable writable = (Writable) row[index];
-          VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index],
writable);
+          verifyRead(binarySortableDeserializeRead, typeInfos[index], row[index]);
         }
       }
       if (writeColumnCount == columnCount) {
@@ -309,11 +311,44 @@ public class TestBinarySortableFast extends TestCase {
     }
   }
 
-  private void testBinarySortableFastCase(int caseNum, boolean doNonRandomFill, Random r)
+  private void verifyRead(BinarySortableDeserializeRead binarySortableDeserializeRead,
+      TypeInfo typeInfo, Object expectedObject) throws IOException {
+    if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) {
+      VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, typeInfo, expectedObject);
+    } else {
+      Object complexFieldObj = VerifyFast.deserializeReadComplexType(binarySortableDeserializeRead,
typeInfo);
+      if (expectedObject == null) {
+        if (complexFieldObj != null) {
+          TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName()
+
+              ", " + complexFieldObj.toString() + ")");
+        }
+      } else {
+        if (complexFieldObj == null) {
+          // It's hard to distinguish a union with null from a null union.
+          if (expectedObject instanceof UnionObject) {
+            UnionObject expectedUnion = (UnionObject) expectedObject;
+            if (expectedUnion.getObject() == null) {
+              return;
+            }
+          }
+          TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName()
+
+              ", " + expectedObject.toString() + ")");
+        }
+      }
+      if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) {
+        TestCase.fail("Comparision failed typeInfo " + typeInfo.toString());
+      }
+    }
+  }
+
+  private void testBinarySortableFastCase(
+      int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes
supportedTypes, int depth)
       throws Throwable {
 
     SerdeRandomRowSource source = new SerdeRandomRowSource();
-    source.init(r);
+
+    // UNDONE: Until Fast BinarySortable supports complex types -- disable.
+    source.init(r, supportedTypes, depth);
 
     int rowCount = 1000;
     Object[][] rows = source.randomRows(rowCount);
@@ -327,8 +362,8 @@ public class TestBinarySortableFast extends TestCase {
 
     StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector();
 
-    PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos();
-    int columnCount = primitiveTypeInfos.length;
+    TypeInfo[] typeInfos = source.typeInfos();
+    int columnCount = typeInfos.length;
 
     int writeColumnCount = columnCount;
     StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector;
@@ -385,14 +420,14 @@ public class TestBinarySortableFast extends TestCase {
         columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
         serde_ascending, rowStructObjectInspector,
         serde_ascending_fewer, writeRowStructObjectInspector,
-        /* ascending */ true, primitiveTypeInfos,
+        /* ascending */ true, typeInfos,
         /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
 
     testBinarySortableFast(source, rows,
         columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
         serde_ascending, rowStructObjectInspector,
         serde_ascending_fewer, writeRowStructObjectInspector,
-        /* ascending */ true, primitiveTypeInfos,
+        /* ascending */ true, typeInfos,
         /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
 
     if (doWriteFewerColumns) {
@@ -400,14 +435,14 @@ public class TestBinarySortableFast extends TestCase {
           columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
           serde_ascending, rowStructObjectInspector,
           serde_ascending_fewer, writeRowStructObjectInspector,
-          /* ascending */ true, primitiveTypeInfos,
+          /* ascending */ true, typeInfos,
           /* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r);
 
       testBinarySortableFast(source, rows,
           columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
           serde_ascending, rowStructObjectInspector,
           serde_ascending_fewer, writeRowStructObjectInspector,
-          /* ascending */ true, primitiveTypeInfos,
+          /* ascending */ true, typeInfos,
           /* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r);
     }
 
@@ -420,14 +455,14 @@ public class TestBinarySortableFast extends TestCase {
         columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
         serde_descending, rowStructObjectInspector,
         serde_ascending_fewer, writeRowStructObjectInspector,
-        /* ascending */ false, primitiveTypeInfos,
+        /* ascending */ false, typeInfos,
         /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
 
     testBinarySortableFast(source, rows,
         columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
         serde_descending, rowStructObjectInspector,
         serde_ascending_fewer, writeRowStructObjectInspector,
-        /* ascending */ false, primitiveTypeInfos,
+        /* ascending */ false, typeInfos,
         /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
 
     if (doWriteFewerColumns) {
@@ -435,27 +470,27 @@ public class TestBinarySortableFast extends TestCase {
           columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
           serde_descending, rowStructObjectInspector,
           serde_descending_fewer, writeRowStructObjectInspector,
-          /* ascending */ false, primitiveTypeInfos,
+          /* ascending */ false, typeInfos,
           /* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r);
 
       testBinarySortableFast(source, rows,
           columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
           serde_descending, rowStructObjectInspector,
           serde_descending_fewer, writeRowStructObjectInspector,
-          /* ascending */ false, primitiveTypeInfos,
+          /* ascending */ false, typeInfos,
           /* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r);
     }
 
   }
 
-  public void testBinarySortableFast() throws Throwable {
+  public void testBinarySortableFast(SerdeRandomRowSource.SupportedTypes supportedTypes,
int depth) throws Throwable {
 
     try {
       Random r = new Random(35790);
 
       int caseNum = 0;
       for (int i = 0; i < 10; i++) {
-        testBinarySortableFastCase(caseNum, (i % 2 == 0), r);
+        testBinarySortableFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth);
         caseNum++;
       }
 
@@ -465,6 +500,18 @@ public class TestBinarySortableFast extends TestCase {
     }
   }
 
+  public void testBinarySortableFastPrimitive() throws Throwable {
+    testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0);
+  }
+
+  public void testBinarySortableFastComplexDepthOne() throws Throwable {
+    testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.ALL_EXCEPT_MAP, 1);
+  }
+
+  public void testBinarySortableFastComplexDepthFour() throws Throwable {
+    testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.ALL_EXCEPT_MAP, 4);
+  }
+
   private static String displayBytes(byte[] bytes, int start, int length) {
     StringBuilder sb = new StringBuilder();
     for (int i = start; i < start + length; i++) {

http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
index c857b42..6e6a4c2 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.serde2.lazy;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Properties;
 import java.util.Random;
@@ -33,10 +35,11 @@ import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead;
 import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 
 import junit.framework.TestCase;
 
@@ -46,12 +49,12 @@ public class TestLazySimpleFast extends TestCase {
     SerdeRandomRowSource source, Object[][] rows,
     LazySimpleSerDe serde, StructObjectInspector rowOI,
     LazySimpleSerDe serde_fewer, StructObjectInspector writeRowOI,
-    byte separator, LazySerDeParameters serdeParams, LazySerDeParameters serdeParams_fewer,
-    PrimitiveTypeInfo[] primitiveTypeInfos,
+    LazySerDeParameters serdeParams, LazySerDeParameters serdeParams_fewer,
+    TypeInfo[] typeInfos,
     boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable {
 
     int rowCount = rows.length;
-    int columnCount = primitiveTypeInfos.length;
+    int columnCount = typeInfos.length;
 
     boolean[] columnsToInclude = null;
     if (useIncludeColumns) {
@@ -62,10 +65,10 @@ public class TestLazySimpleFast extends TestCase {
     }
 
     int writeColumnCount = columnCount;
-    PrimitiveTypeInfo[] writePrimitiveTypeInfos = primitiveTypeInfos;
+    TypeInfo[] writeTypeInfos = typeInfos;
     if (doWriteFewerColumns) {
       writeColumnCount = writeRowOI.getAllStructFieldRefs().size();
-      writePrimitiveTypeInfos = Arrays.copyOf(primitiveTypeInfos, writeColumnCount);
+      writeTypeInfos = Arrays.copyOf(typeInfos, writeColumnCount);
     }
 
     // Try to serialize
@@ -75,16 +78,12 @@ public class TestLazySimpleFast extends TestCase {
       Output output = new Output();
 
       LazySimpleSerializeWrite lazySimpleSerializeWrite =
-          new LazySimpleSerializeWrite(columnCount,
-              separator, serdeParams);
+          new LazySimpleSerializeWrite(columnCount, serdeParams);
 
       lazySimpleSerializeWrite.set(output);
 
       for (int index = 0; index < columnCount; index++) {
-
-        Writable writable = (Writable) row[index];
-
-        VerifyFast.serializeWrite(lazySimpleSerializeWrite, primitiveTypeInfos[index], writable);
+        VerifyFast.serializeWrite(lazySimpleSerializeWrite, typeInfos[index], row[index]);
       }
 
       BytesWritable bytesWritable = new BytesWritable();
@@ -97,29 +96,24 @@ public class TestLazySimpleFast extends TestCase {
       Object[] row = rows[i];
       LazySimpleDeserializeRead lazySimpleDeserializeRead =
               new LazySimpleDeserializeRead(
-                  writePrimitiveTypeInfos,
+                  writeTypeInfos,
                   /* useExternalBuffer */ false,
-                  separator, serdeParams);
+                  serdeParams);
 
       BytesWritable bytesWritable = serializeWriteBytes[i];
       byte[] bytes = bytesWritable.getBytes();
       int length = bytesWritable.getLength();
       lazySimpleDeserializeRead.set(bytes, 0, length);
 
-      char[] chars = new char[length];
-      for (int c = 0; c < chars.length; c++) {
-        chars[c] = (char) (bytes[c] & 0xFF);
-      }
-
       for (int index = 0; index < columnCount; index++) {
         if (useIncludeColumns && !columnsToInclude[index]) {
           lazySimpleDeserializeRead.skipNextField();
         } else if (index >= writeColumnCount) {
           // Should come back a null.
-          VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index],
null);
+          verifyReadNull(lazySimpleDeserializeRead, typeInfos[index]);
         } else {
-          Writable writable = (Writable) row[index];
-          VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index],
writable);
+          Object expectedObject = row[index];
+          verifyRead(lazySimpleDeserializeRead, typeInfos[index], expectedObject);
         }
       }
       if (writeColumnCount == columnCount) {
@@ -128,28 +122,22 @@ public class TestLazySimpleFast extends TestCase {
     }
 
     // Try to deserialize using SerDe class our Writable row objects created by SerializeWrite.
-    for (int i = 0; i < rowCount; i++) {
-      BytesWritable bytesWritable = serializeWriteBytes[i];
+    for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+      BytesWritable bytesWritable = serializeWriteBytes[rowIndex];
       LazyStruct lazySimpleStruct = (LazyStruct) serde.deserialize(bytesWritable);
 
-      Object[] row = rows[i];
+      Object[] row = rows[rowIndex];
 
       for (int index = 0; index < columnCount; index++) {
-        PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
-        Writable writable = (Writable) row[index];
-        LazyPrimitive lazyPrimitive = (LazyPrimitive) lazySimpleStruct.getField(index);
-        Object object;
-        if (lazyPrimitive != null) {
-          object = lazyPrimitive.getWritableObject();
-        } else {
-          object = null;
-        }
-        if (writable == null || object == null) {
-          if (writable != null || object != null) {
+        TypeInfo typeInfo = typeInfos[index];
+        Object expectedObject = row[index];
+        Object object = lazySimpleStruct.getField(index);
+        if (expectedObject == null || object == null) {
+          if (expectedObject != null || object != null) {
             fail("SerDe deserialized NULL column mismatch");
           }
         } else {
-          if (!object.equals(writable)) {
+          if (!VerifyLazy.lazyCompare(typeInfo, object, expectedObject)) {
             fail("SerDe deserialized value does not match");
           }
         }
@@ -185,9 +173,9 @@ public class TestLazySimpleFast extends TestCase {
 
       LazySimpleDeserializeRead lazySimpleDeserializeRead =
               new LazySimpleDeserializeRead(
-                  writePrimitiveTypeInfos,
+                  writeTypeInfos,
                   /* useExternalBuffer */ false,
-                  separator, serdeParams);
+                  serdeParams);
 
       byte[] bytes = serdeBytes[i];
       lazySimpleDeserializeRead.set(bytes, 0, bytes.length);
@@ -197,10 +185,10 @@ public class TestLazySimpleFast extends TestCase {
           lazySimpleDeserializeRead.skipNextField();
         } else if (index >= writeColumnCount) {
           // Should come back a null.
-          VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index],
null);
+          verifyReadNull(lazySimpleDeserializeRead, typeInfos[index]);
         } else {
-          Writable writable = (Writable) row[index];
-          VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index],
writable);
+          Object expectedObject = row[index];
+          verifyRead(lazySimpleDeserializeRead, typeInfos[index], expectedObject);
         }
       }
       if (writeColumnCount == columnCount) {
@@ -209,6 +197,46 @@ public class TestLazySimpleFast extends TestCase {
     }
   }
 
+  private void verifyReadNull(LazySimpleDeserializeRead lazySimpleDeserializeRead,
+      TypeInfo typeInfo) throws IOException {
+    if (typeInfo.getCategory() == Category.PRIMITIVE) {
+      VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, null);
+    } else {
+      Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead,
typeInfo);
+      if (complexFieldObj != null) {
+        TestCase.fail("Field report not null but object is null");
+      }
+    }
+  }
+
+  private void verifyRead(LazySimpleDeserializeRead lazySimpleDeserializeRead,
+      TypeInfo typeInfo, Object expectedObject) throws IOException {
+    if (typeInfo.getCategory() == Category.PRIMITIVE) {
+      VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, expectedObject);
+    } else {
+      Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead,
typeInfo);
+      if (expectedObject == null) {
+        if (complexFieldObj != null) {
+          TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName()
+ ", " + complexFieldObj.toString() + ")");
+        }
+      } else {
+        if (complexFieldObj == null) {
+          // It's hard to distinguish a union with null from a null union.
+          if (expectedObject instanceof UnionObject) {
+            UnionObject expectedUnion = (UnionObject) expectedObject;
+            if (expectedUnion.getObject() == null) {
+              return;
+            }
+          }
+          TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName()
+ ", " + expectedObject.toString() + ")");
+        }
+      }
+      if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) {
+        TestCase.fail("Comparision failed typeInfo " + typeInfo.toString());
+      }
+    }
+  }
+
   private byte[] copyBytes(Text serialized) {
     byte[] result = new byte[serialized.getLength()];
     System.arraycopy(serialized.getBytes(), 0, result, 0, serialized.getLength());
@@ -238,19 +266,25 @@ public class TestLazySimpleFast extends TestCase {
     return serDe;
   }
 
-  private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes) throws
SerDeException {
+  private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes,
+      byte[] separators) throws SerDeException {
     Configuration conf = new Configuration();
     Properties tbl = createProperties(fieldNames, fieldTypes);
-    return new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
+    LazySerDeParameters lazySerDeParams = new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
+    for (int i = 0; i < separators.length; i++) {
+      lazySerDeParams.setSeparator(i, separators[i]);
+    }
+    return lazySerDeParams;
   }
 
-  public void testLazySimpleFastCase(int caseNum, boolean doNonRandomFill, Random r)
+  public void testLazySimpleFastCase(
+      int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes
supportedTypes, int depth)
       throws Throwable {
 
     SerdeRandomRowSource source = new SerdeRandomRowSource();
-    source.init(r);
+    source.init(r, supportedTypes, depth);
 
-    int rowCount = 1000;
+    int rowCount = 100;
     Object[][] rows = source.randomRows(rowCount);
 
     if (doNonRandomFill) {
@@ -259,8 +293,8 @@ public class TestLazySimpleFast extends TestCase {
 
     StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector();
 
-    PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos();
-    int columnCount = primitiveTypeInfos.length;
+    TypeInfo[] typeInfos = source.typeInfos();
+    int columnCount = typeInfos.length;
 
     int writeColumnCount = columnCount;
     StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector;
@@ -277,8 +311,11 @@ public class TestLazySimpleFast extends TestCase {
     String fieldNames = ObjectInspectorUtils.getFieldNames(rowStructObjectInspector);
     String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowStructObjectInspector);
 
+    // Use different separator values.
+    byte[] separators = new byte[] {(byte) 9, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte)
6, (byte) 7, (byte) 8};
+
     LazySimpleSerDe serde = getSerDe(fieldNames, fieldTypes);
-    LazySerDeParameters serdeParams = getSerDeParams(fieldNames, fieldTypes);
+    LazySerDeParameters serdeParams = getSerDeParams(fieldNames, fieldTypes, separators);
 
     LazySimpleSerDe serde_fewer = null;
     LazySerDeParameters serdeParams_fewer = null;
@@ -287,22 +324,22 @@ public class TestLazySimpleFast extends TestCase {
       String partialFieldTypes = ObjectInspectorUtils.getFieldTypes(writeRowStructObjectInspector);
 
         serde_fewer = getSerDe(fieldNames, fieldTypes);
-        serdeParams_fewer = getSerDeParams(partialFieldNames, partialFieldTypes);
+        serdeParams_fewer = getSerDeParams(partialFieldNames, partialFieldTypes, separators);
     }
 
-    byte separator = (byte) '\t';
+
     testLazySimpleFast(
         source, rows,
         serde, rowStructObjectInspector,
         serde_fewer, writeRowStructObjectInspector,
-        separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+        serdeParams, serdeParams_fewer, typeInfos,
         /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
 
     testLazySimpleFast(
         source, rows,
         serde, rowStructObjectInspector,
         serde_fewer, writeRowStructObjectInspector,
-        separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+        serdeParams, serdeParams_fewer, typeInfos,
         /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
 
     if (doWriteFewerColumns) {
@@ -310,26 +347,26 @@ public class TestLazySimpleFast extends TestCase {
           source, rows,
           serde, rowStructObjectInspector,
           serde_fewer, writeRowStructObjectInspector,
-          separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+          serdeParams, serdeParams_fewer, typeInfos,
           /* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r);
 
       testLazySimpleFast(
           source, rows,
           serde, rowStructObjectInspector,
           serde_fewer, writeRowStructObjectInspector,
-          separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+          serdeParams, serdeParams_fewer, typeInfos,
           /* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r);
     }
   }
 
-  public void testLazySimpleFast() throws Throwable {
+  public void testLazySimpleFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int
depth) throws Throwable {
 
     try {
-      Random r = new Random(35790);
+      Random r = new Random(8322);
 
       int caseNum = 0;
-      for (int i = 0; i < 10; i++) {
-        testLazySimpleFastCase(caseNum, (i % 2 == 0), r);
+      for (int i = 0; i < 20; i++) {
+        testLazySimpleFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth);
         caseNum++;
       }
 
@@ -338,4 +375,16 @@ public class TestLazySimpleFast extends TestCase {
       throw e;
     }
   }
+
+  public void testLazyBinarySimplePrimitive() throws Throwable {
+    testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0);
+  }
+
+  public void testLazyBinarySimpleComplexDepthOne() throws Throwable {
+    testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.ALL, 1);
+  }
+
+  public void testLazyBinarySimpleComplexDepthFour() throws Throwable {
+    testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.ALL, 4);
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
index e62a80a..13c73be 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.serde2.lazybinary;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Random;
 
@@ -27,11 +29,14 @@ import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.hive.serde2.SerdeRandomRowSource;
 import org.apache.hadoop.hive.serde2.VerifyFast;
 import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
+import org.apache.hadoop.hive.serde2.lazy.VerifyLazy;
 import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead;
 import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Writable;
 
@@ -41,11 +46,11 @@ public class TestLazyBinaryFast extends TestCase {
       SerdeRandomRowSource source, Object[][] rows,
       AbstractSerDe serde, StructObjectInspector rowOI,
       AbstractSerDe serde_fewer, StructObjectInspector writeRowOI,
-      PrimitiveTypeInfo[] primitiveTypeInfos,
+      TypeInfo[] typeInfos,
       boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable
{
 
     int rowCount = rows.length;
-    int columnCount = primitiveTypeInfos.length;
+    int columnCount = typeInfos.length;
 
     boolean[] columnsToInclude = null;
     if (useIncludeColumns) {
@@ -56,10 +61,10 @@ public class TestLazyBinaryFast extends TestCase {
     }
 
     int writeColumnCount = columnCount;
-    PrimitiveTypeInfo[] writePrimitiveTypeInfos = primitiveTypeInfos;
+    TypeInfo[] writeTypeInfos = typeInfos;
     if (doWriteFewerColumns) {
       writeColumnCount = writeRowOI.getAllStructFieldRefs().size();
-      writePrimitiveTypeInfos = Arrays.copyOf(primitiveTypeInfos, writeColumnCount);
+      writeTypeInfos = Arrays.copyOf(typeInfos, writeColumnCount);
     }
 
     LazyBinarySerializeWrite lazyBinarySerializeWrite =
@@ -73,10 +78,7 @@ public class TestLazyBinaryFast extends TestCase {
       lazyBinarySerializeWrite.set(output);
 
       for (int index = 0; index < writeColumnCount; index++) {
-
-        Writable writable = (Writable) row[index];
-
-        VerifyFast.serializeWrite(lazyBinarySerializeWrite, primitiveTypeInfos[index], writable);
+        VerifyFast.serializeWrite(lazyBinarySerializeWrite, typeInfos[index], row[index]);
       }
 
       BytesWritable bytesWritable = new BytesWritable();
@@ -92,7 +94,7 @@ public class TestLazyBinaryFast extends TestCase {
       // column.
       LazyBinaryDeserializeRead lazyBinaryDeserializeRead =
               new LazyBinaryDeserializeRead(
-                  writePrimitiveTypeInfos,
+                  writeTypeInfos,
                   /* useExternalBuffer */ false);
 
       BytesWritable bytesWritable = serializeWriteBytes[i];
@@ -103,10 +105,9 @@ public class TestLazyBinaryFast extends TestCase {
           lazyBinaryDeserializeRead.skipNextField();
         } else if (index >= writeColumnCount) {
           // Should come back a null.
-          VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index],
null);
+          VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfos[index], null);
         } else {
-          Writable writable = (Writable) row[index];
-          VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index],
writable);
+          verifyRead(lazyBinaryDeserializeRead, typeInfos[index], row[index]);
         }
       }
       if (writeColumnCount == columnCount) {
@@ -127,15 +128,14 @@ public class TestLazyBinaryFast extends TestCase {
       Object[] row = rows[i];
 
       for (int index = 0; index < writeColumnCount; index++) {
-        PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
-        Writable writable = (Writable) row[index];
+        TypeInfo typeInfo = typeInfos[index];
         Object object = lazyBinaryStruct.getField(index);
-        if (writable == null || object == null) {
-          if (writable != null || object != null) {
+        if (row[index] == null || object == null) {
+          if (row[index] != null || object != null) {
             fail("SerDe deserialized NULL column mismatch");
           }
         } else {
-          if (!object.equals(writable)) {
+          if (!VerifyLazy.lazyCompare(typeInfo, object, row[index])) {
             fail("SerDe deserialized value does not match");
           }
         }
@@ -172,10 +172,10 @@ public class TestLazyBinaryFast extends TestCase {
       if (bytes1.length != bytes2.length) {
         fail("SerializeWrite length " + bytes2.length + " and " +
               "SerDe serialization length " + bytes1.length +
-              " do not match (" + Arrays.toString(primitiveTypeInfos) + ")");
+              " do not match (" + Arrays.toString(typeInfos) + ")");
       }
       if (!Arrays.equals(bytes1, bytes2)) {
-        fail("SerializeWrite and SerDe serialization does not match (" + Arrays.toString(primitiveTypeInfos)
+ ")");
+        fail("SerializeWrite and SerDe serialization does not match (" + Arrays.toString(typeInfos)
+ ")");
       }
       serdeBytes[i] = bytesWritable;
     }
@@ -187,7 +187,7 @@ public class TestLazyBinaryFast extends TestCase {
       // When doWriteFewerColumns, try to read more fields than exist in buffer.
       LazyBinaryDeserializeRead lazyBinaryDeserializeRead =
               new LazyBinaryDeserializeRead(
-                  primitiveTypeInfos,
+                  typeInfos,
                   /* useExternalBuffer */ false);
 
       BytesWritable bytesWritable = serdeBytes[i];
@@ -198,10 +198,9 @@ public class TestLazyBinaryFast extends TestCase {
           lazyBinaryDeserializeRead.skipNextField();
         } else if (index >= writeColumnCount) {
           // Should come back a null.
-          VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index],
null);
+          VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfos[index], null);
         } else {
-          Writable writable = (Writable) row[index];
-          VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index],
writable);
+          verifyRead(lazyBinaryDeserializeRead, typeInfos[index], row[index]);
         }
       }
       if (writeColumnCount == columnCount) {
@@ -210,12 +209,45 @@ public class TestLazyBinaryFast extends TestCase {
     }
   }
 
-  public void testLazyBinaryFastCase(int caseNum, boolean doNonRandomFill, Random r) throws
Throwable {
+  private void verifyRead(LazyBinaryDeserializeRead lazyBinaryDeserializeRead,
+      TypeInfo typeInfo, Object expectedObject) throws IOException {
+    if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) {
+      VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfo, expectedObject);
+    } else {
+      Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazyBinaryDeserializeRead,
typeInfo);
+      if (expectedObject == null) {
+        if (complexFieldObj != null) {
+          TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName()
+
+              ", " + complexFieldObj.toString() + ")");
+        }
+      } else {
+        if (complexFieldObj == null) {
+          // It's hard to distinguish a union with null from a null union.
+          if (expectedObject instanceof UnionObject) {
+            UnionObject expectedUnion = (UnionObject) expectedObject;
+            if (expectedUnion.getObject() == null) {
+              return;
+            }
+          }
+          TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName()
+
+              ", " + expectedObject.toString() + ")");
+        }
+      }
+      if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) {
+        TestCase.fail("Comparision failed typeInfo " + typeInfo.toString());
+      }
+    }
+  }
+
+  public void testLazyBinaryFastCase(
+      int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes
supportedTypes, int depth)
+      throws Throwable {
 
     SerdeRandomRowSource source = new SerdeRandomRowSource();
-    source.init(r);
 
-    int rowCount = 1000;
+    source.init(r, supportedTypes, depth);
+
+    int rowCount = 100;
     Object[][] rows = source.randomRows(rowCount);
 
     if (doNonRandomFill) {
@@ -224,8 +256,8 @@ public class TestLazyBinaryFast extends TestCase {
 
     StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector();
 
-    PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos();
-    int columnCount = primitiveTypeInfos.length;
+    TypeInfo[] typeInfos = source.typeInfos();
+    int columnCount = typeInfos.length;
 
     int writeColumnCount = columnCount;
     StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector;
@@ -256,14 +288,14 @@ public class TestLazyBinaryFast extends TestCase {
         source, rows,
         serde, rowStructObjectInspector,
         serde_fewer, writeRowStructObjectInspector,
-        primitiveTypeInfos,
+        typeInfos,
         /* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
 
     testLazyBinaryFast(
         source, rows,
         serde, rowStructObjectInspector,
         serde_fewer, writeRowStructObjectInspector,
-        primitiveTypeInfos,
+        typeInfos,
         /* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
 
     /*
@@ -286,14 +318,13 @@ public class TestLazyBinaryFast extends TestCase {
     // }
   }
 
-  public void testLazyBinaryFast() throws Throwable {
-
+  private void testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int
depth) throws Throwable {
     try {
-      Random r = new Random(35790);
+      Random r = new Random(9983);
 
       int caseNum = 0;
       for (int i = 0; i < 10; i++) {
-        testLazyBinaryFastCase(caseNum, (i % 2 == 0), r);
+        testLazyBinaryFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth);
         caseNum++;
       }
 
@@ -302,4 +333,16 @@ public class TestLazyBinaryFast extends TestCase {
       throw e;
     }
   }
+
+  public void testLazyBinaryFastPrimitive() throws Throwable {
+    testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0);
+  }
+
+  public void testLazyBinaryFastComplexDepthOne() throws Throwable {
+    testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.ALL, 1);
+  }
+
+  public void testLazyBinaryFastComplexDepthFour() throws Throwable {
+    testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.ALL, 4);
+  }
 }
\ No newline at end of file


Mime
View raw message