hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject [50/82] [abbrv] hive git commit: HIVE-9605:Remove parquet nested objects from wrapper writable objects (Sergio Pena, reviewed by Ferdinand Xu)
Date Fri, 29 May 2015 00:50:57 GMT
HIVE-9605:Remove parquet nested objects from wrapper writable objects (Sergio Pena, reviewed
by Ferdinand Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1f75e348
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1f75e348
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1f75e348

Branch: refs/heads/llap
Commit: 1f75e348a90d44bbd6a836ecf7228a092c5ff709
Parents: 93b5588
Author: Ferdinand Xu <cheng.a.xu@intel.com>
Authored: Mon May 25 15:18:50 2015 -0400
Committer: Ferdinand Xu <cheng.a.xu@intel.com>
Committed: Mon May 25 15:18:50 2015 -0400

----------------------------------------------------------------------
 .../benchmark/storage/ColumnarStorageBench.java |  4 +-
 .../convert/HiveCollectionConverter.java        |  5 +--
 .../io/parquet/convert/HiveGroupConverter.java  | 10 -----
 .../hive/ql/io/parquet/convert/Repeated.java    |  8 ++--
 .../serde/AbstractParquetMapInspector.java      | 13 +++---
 .../serde/DeepParquetHiveMapInspector.java      |  6 +--
 .../serde/ParquetHiveArrayInspector.java        | 43 +++++--------------
 .../serde/StandardParquetHiveMapInspector.java  |  6 +--
 .../ql/io/parquet/TestArrayCompatibility.java   | 44 ++++++++++----------
 .../ql/io/parquet/TestDataWritableWriter.java   | 44 ++++++++------------
 .../hive/ql/io/parquet/TestMapStructures.java   | 36 ++++++++--------
 .../hive/ql/io/parquet/TestParquetSerDe.java    |  8 +---
 .../serde/TestAbstractParquetMapInspector.java  |  4 +-
 .../serde/TestDeepParquetHiveMapInspector.java  |  4 +-
 .../serde/TestParquetHiveArrayInspector.java    |  3 +-
 .../TestStandardParquetHiveMapInspector.java    |  4 +-
 16 files changed, 92 insertions(+), 150 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
----------------------------------------------------------------------
diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
index 61c2eb4..4f6985c 100644
--- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
+++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java
@@ -209,13 +209,13 @@ public class ColumnarStorageBench {
         case LIST: {
           List<TypeInfo> elementType = new ArrayList<TypeInfo>();
           elementType.add(((ListTypeInfo) type).getListElementTypeInfo());
-          fields[pos++] = record(createRecord(elementType));
+          fields[pos++] = createRecord(elementType);
         } break;
         case MAP: {
           List<TypeInfo> keyValueType = new ArrayList<TypeInfo>();
           keyValueType.add(((MapTypeInfo) type).getMapKeyTypeInfo());
           keyValueType.add(((MapTypeInfo) type).getMapValueTypeInfo());
-          fields[pos++] = record(record(createRecord(keyValueType)));
+          fields[pos++] = record(createRecord(keyValueType));
         } break;
         case STRUCT: {
           List<TypeInfo> elementType = ((StructTypeInfo) type).getAllStructFieldTypeInfos();

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
index 0fd538e..80405bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.io.parquet.convert;
 import com.google.common.base.Preconditions;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.Writable;
@@ -83,8 +82,8 @@ public class HiveCollectionConverter extends HiveGroupConverter {
 
   @Override
   public void end() {
-    parent.set(index, wrapList(new ArrayWritable(
-        Writable.class, list.toArray(new Writable[list.size()]))));
+    parent.set(index, new ArrayWritable(
+        Writable.class, list.toArray(new Writable[0])));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
index 4809f9b..c6d03a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java
@@ -13,7 +13,6 @@
  */
 package org.apache.hadoop.hive.ql.io.parquet.convert;
 
-import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.Writable;
 import parquet.io.api.Converter;
 import parquet.io.api.GroupConverter;
@@ -72,15 +71,6 @@ public abstract class HiveGroupConverter extends GroupConverter implements
Conve
     return getConverterFromDescription(type.asGroupType(), index, parent);
   }
 
-  /**
-   * The original list and map conversion didn't remove the synthetic layer and
-   * the ObjectInspector had to remove it. This is a temporary fix that adds an
-   * extra layer for the ObjectInspector to remove.
-   */
-  static ArrayWritable wrapList(ArrayWritable list) {
-    return new ArrayWritable(Writable.class, new Writable[] {list});
-  }
-
   public abstract void set(int index, Writable value);
 
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
index 0130aef..4becd20 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
@@ -125,8 +125,8 @@ public interface Repeated extends ConverterParent {
 
     @Override
     public void parentEnd() {
-      parent.set(index, HiveGroupConverter.wrapList(new ArrayWritable(
-          Writable.class, list.toArray(new Writable[list.size()]))));
+      parent.set(index, new ArrayWritable(
+          Writable.class, list.toArray(new Writable[list.size()])));
     }
 
     @Override
@@ -185,8 +185,8 @@ public interface Repeated extends ConverterParent {
 
     @Override
     public void parentEnd() {
-      parent.set(index, wrapList(new ArrayWritable(
-          Writable.class, list.toArray(new Writable[list.size()]))));
+      parent.set(index, new ArrayWritable(
+          Writable.class, list.toArray(new Writable[list.size()])));
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
index 62c61fc..49bf1c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java
@@ -59,15 +59,12 @@ public abstract class AbstractParquetMapInspector implements SettableMapObjectIn
     }
 
     if (data instanceof ArrayWritable) {
-      final Writable[] mapContainer = ((ArrayWritable) data).get();
-
-      if (mapContainer == null || mapContainer.length == 0) {
+      final Writable[] mapArray = ((ArrayWritable) data).get();
+      if (mapArray == null || mapArray.length == 0) {
         return null;
       }
 
-      final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get();
       final Map<Writable, Writable> map = new LinkedHashMap<Writable, Writable>();
-
       for (final Writable obj : mapArray) {
         final ArrayWritable mapObj = (ArrayWritable) obj;
         final Writable[] arr = mapObj.get();
@@ -91,12 +88,12 @@ public abstract class AbstractParquetMapInspector implements SettableMapObjectIn
     }
 
     if (data instanceof ArrayWritable) {
-      final Writable[] mapContainer = ((ArrayWritable) data).get();
+      final Writable[] mapArray = ((ArrayWritable) data).get();
 
-      if (mapContainer == null || mapContainer.length == 0) {
+      if (mapArray == null || mapArray.length == 0) {
         return -1;
       } else {
-        return ((ArrayWritable) mapContainer[0]).get().length;
+        return mapArray.length;
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
index d38c641..143d72e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java
@@ -40,14 +40,12 @@ public class DeepParquetHiveMapInspector extends AbstractParquetMapInspector
{
     }
 
     if (data instanceof ArrayWritable) {
-      final Writable[] mapContainer = ((ArrayWritable) data).get();
+      final Writable[] mapArray = ((ArrayWritable) data).get();
 
-      if (mapContainer == null || mapContainer.length == 0) {
+      if (mapArray == null || mapArray.length == 0) {
         return null;
       }
 
-      final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get();
-
       for (final Writable obj : mapArray) {
         final ArrayWritable mapObj = (ArrayWritable) obj;
         final Writable[] arr = mapObj.get();

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
index 53ca31d..bde0dcb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java
@@ -56,20 +56,13 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector
{
     }
 
     if (data instanceof ArrayWritable) {
-      final Writable[] listContainer = ((ArrayWritable) data).get();
-
-      if (listContainer == null || listContainer.length == 0) {
-        return null;
-      }
-
-      final Writable subObj = listContainer[0];
-
-      if (subObj == null) {
+      final Writable[] array = ((ArrayWritable) data).get();
+      if (array == null || array.length == 0) {
         return null;
       }
 
-      if (index >= 0 && index < ((ArrayWritable) subObj).get().length) {
-        return ((ArrayWritable) subObj).get()[index];
+      if (index >= 0 && index < array.length) {
+        return array[index];
       } else {
         return null;
       }
@@ -85,19 +78,12 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector
{
     }
 
     if (data instanceof ArrayWritable) {
-      final Writable[] listContainer = ((ArrayWritable) data).get();
-
-      if (listContainer == null || listContainer.length == 0) {
+      final Writable[] array = ((ArrayWritable) data).get();
+      if (array == null || array.length == 0) {
         return -1;
       }
 
-      final Writable subObj = listContainer[0];
-
-      if (subObj == null) {
-        return 0;
-      }
-
-      return ((ArrayWritable) subObj).get().length;
+      return array.length;
     }
 
     throw new UnsupportedOperationException("Cannot inspect " + data.getClass().getCanonicalName());
@@ -110,21 +96,12 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector
{
     }
 
     if (data instanceof ArrayWritable) {
-      final Writable[] listContainer = ((ArrayWritable) data).get();
-
-      if (listContainer == null || listContainer.length == 0) {
+      final Writable[] array = ((ArrayWritable) data).get();
+      if (array == null || array.length == 0) {
         return null;
       }
 
-      final Writable subObj = listContainer[0];
-
-      if (subObj == null) {
-        return null;
-      }
-
-      final Writable[] array = ((ArrayWritable) subObj).get();
-      final List<Writable> list = new ArrayList<Writable>();
-
+      final List<Writable> list = new ArrayList<Writable>(array.length);
       for (final Writable obj : array) {
         list.add(obj);
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
index 5aa1448..22250b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java
@@ -37,12 +37,12 @@ public class StandardParquetHiveMapInspector extends AbstractParquetMapInspector
       return null;
     }
     if (data instanceof ArrayWritable) {
-      final Writable[] mapContainer = ((ArrayWritable) data).get();
+      final Writable[] mapArray = ((ArrayWritable) data).get();
 
-      if (mapContainer == null || mapContainer.length == 0) {
+      if (mapArray == null || mapArray.length == 0) {
         return null;
       }
-      final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get();
+
       for (final Writable obj : mapArray) {
         final ArrayWritable mapObj = (ArrayWritable) obj;
         final Writable[] arr = mapObj.get();

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
index f349aa0..e0ea262 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
@@ -62,8 +62,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(
-        new IntWritable(34), new IntWritable(35), new IntWritable(36)));
+    ArrayWritable expected = list(
+        new IntWritable(34), new IntWritable(35), new IntWritable(36));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -109,9 +109,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new FloatWritable(1.0f), new FloatWritable(1.0f)),
-        record(new FloatWritable(2.0f), new FloatWritable(2.0f))));
+        record(new FloatWritable(2.0f), new FloatWritable(2.0f)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -148,8 +148,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
-        new IntWritable(34), new IntWritable(35), new IntWritable(36)));
+    ArrayWritable expected = list(
+        new IntWritable(34), new IntWritable(35), new IntWritable(36));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -198,9 +198,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new LongWritable(1234L)),
-        record(new LongWritable(2345L))));
+        record(new LongWritable(2345L)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -237,8 +237,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
-        new IntWritable(34), new IntWritable(35), new IntWritable(36)));
+    ArrayWritable expected = list(
+        new IntWritable(34), new IntWritable(35), new IntWritable(36));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -287,9 +287,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new LongWritable(1234L)),
-        record(new LongWritable(2345L))));
+        record(new LongWritable(2345L)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -339,9 +339,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         new LongWritable(1234L),
-        new LongWritable(2345L)));
+        new LongWritable(2345L));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -397,9 +397,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new DoubleWritable(0.0), new DoubleWritable(0.0)),
-        record(new DoubleWritable(0.0), new DoubleWritable(180.0))));
+        record(new DoubleWritable(0.0), new DoubleWritable(180.0)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -473,10 +473,10 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new DoubleWritable(0.0), new DoubleWritable(0.0)),
         null,
-        record(new DoubleWritable(0.0), new DoubleWritable(180.0))));
+        record(new DoubleWritable(0.0), new DoubleWritable(180.0)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -546,9 +546,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new DoubleWritable(0.0), new DoubleWritable(180.0)),
-        record(new DoubleWritable(0.0), new DoubleWritable(0.0))));
+        record(new DoubleWritable(0.0), new DoubleWritable(0.0)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -619,9 +619,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect
{
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new DoubleWritable(0.0), new DoubleWritable(180.0)),
-        record(new DoubleWritable(0.0), new DoubleWritable(0.0))));
+        record(new DoubleWritable(0.0), new DoubleWritable(0.0)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
index 8f03c5b..73425e3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java
@@ -311,13 +311,11 @@ public class TestDataWritableWriter {
         + "}\n";
 
     ArrayWritable hiveRecord = createGroup(
-        createGroup(
-            createArray(
-                createInt(1),
-                createNull(),
-                createInt(2)
-            )
-        )
+      createArray(
+          createInt(1),
+          createNull(),
+          createInt(2)
+      )
     );
 
     // Write record to Parquet format
@@ -363,18 +361,16 @@ public class TestDataWritableWriter {
     ArrayWritable hiveRecord = createGroup(
         createGroup(
             createArray(
-                createArray(
-                    createString("key1"),
-                    createInt(1)
-                ),
-                createArray(
-                    createString("key2"),
-                    createInt(2)
-                ),
-                createArray(
-                    createString("key3"),
-                    createNull()
-                )
+                createString("key1"),
+                createInt(1)
+            ),
+            createArray(
+                createString("key2"),
+                createInt(2)
+            ),
+            createArray(
+                createString("key3"),
+                createNull()
             )
         )
     );
@@ -432,14 +428,10 @@ public class TestDataWritableWriter {
         + "}\n";
 
     ArrayWritable hiveRecord = createGroup(
-        createGroup(
+        createArray(
             createArray(
-                createGroup(
-                    createArray(
-                        createInt(1),
-                        createInt(2)
-                    )
-                )
+                createInt(1),
+                createInt(2)
             )
         )
     );

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
index 6fc83e0..32264a4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
@@ -82,9 +82,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new Text("lettuce"), new IntWritable(34)),
-        record(new Text("cabbage"), new IntWritable(18))));
+        record(new Text("cabbage"), new IntWritable(18)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -149,10 +149,10 @@ public class TestMapStructures extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(
+    ArrayWritable expected = list(
         record(new Text("lettuce"), new IntWritable(34)),
         record(new Text("kale"), null),
-        record(new Text("cabbage"), new IntWritable(18))));
+        record(new Text("cabbage"), new IntWritable(18)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -230,9 +230,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(
-        record(new Text("green"), list(new Text("lettuce"), new Text("kale"), null)),
-        record(new Text("brown"), null)));
+    ArrayWritable expected = list(
+        record(new Text("green"), record(new Text("lettuce"), new Text("kale"), null)),
+        record(new Text("brown"), null));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -325,9 +325,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(
-        record(new Text("low"), list(new IntWritable(34), new IntWritable(35), null)),
-        record(new Text("high"), list(new IntWritable(340), new IntWritable(360)))));
+    ArrayWritable expected = list(
+        record(new Text("low"), record(new IntWritable(34), new IntWritable(35), null)),
+        record(new Text("high"), record(new IntWritable(340), new IntWritable(360))));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -386,9 +386,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(record(
+    ArrayWritable expected = list(record(
         record(new IntWritable(7), new IntWritable(22)),
-        new DoubleWritable(3.14))));
+        new DoubleWritable(3.14)));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -447,9 +447,9 @@ public class TestMapStructures extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(record(
+    ArrayWritable expected = list(record(
         new DoubleWritable(3.14),
-        record(new IntWritable(7), new IntWritable(22)))));
+        record(new IntWritable(7), new IntWritable(22))));
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());
@@ -542,13 +542,13 @@ public class TestMapStructures extends AbstractTestParquetDirect {
           }
         });
 
-    ArrayWritable expected = record(list(
-        record(new Text("a"), list(
+    ArrayWritable expected = list(
+        record(new Text("a"), record(
             record(new Text("b"), new IntWritable(1)))),
-        record(new Text("b"), list(
+        record(new Text("b"), record(
             record(new Text("a"), new IntWritable(-1)),
             record(new Text("b"), new IntWritable(-2))))
-    ));
+    );
 
     List<ArrayWritable> records = read(test);
     Assert.assertEquals("Should have only one record", 1, records.size());

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
index 21f889a..dbb2795 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java
@@ -55,7 +55,6 @@ public class TestParquetSerDe extends TestCase {
       arr[4] = new DoubleWritable((double) 5.3);
       arr[5] = new BytesWritable("hive and hadoop and parquet. Big family.".getBytes("UTF-8"));
       arr[6] = new BytesWritable("parquetSerde binary".getBytes("UTF-8"));
-      final Writable[] mapContainer = new Writable[1];
       final Writable[] map = new Writable[3];
       for (int i = 0; i < 3; ++i) {
         final Writable[] pair = new Writable[2];
@@ -63,16 +62,13 @@ public class TestParquetSerDe extends TestCase {
         pair[1] = new IntWritable(i);
         map[i] = new ArrayWritable(Writable.class, pair);
       }
-      mapContainer[0] = new ArrayWritable(Writable.class, map);
-      arr[7] = new ArrayWritable(Writable.class, mapContainer);
+      arr[7] = new ArrayWritable(Writable.class, map);
 
-      final Writable[] arrayContainer = new Writable[1];
       final Writable[] array = new Writable[5];
       for (int i = 0; i < 5; ++i) {
         array[i] = new BytesWritable(("elem_" + i).getBytes("UTF-8"));
       }
-      arrayContainer[0] = new ArrayWritable(Writable.class, array);
-      arr[8] = new ArrayWritable(Writable.class, arrayContainer);
+      arr[8] = new ArrayWritable(Writable.class, array);
 
       final ArrayWritable arrWritable = new ArrayWritable(Writable.class, arr);
       // Test

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
index ef05150..f5d9cb4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java
@@ -71,11 +71,9 @@ public class TestAbstractParquetMapInspector extends TestCase {
     final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
     final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};
 
-    final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
+    final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{
       new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});
 
-    final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});
-
     final Map<Writable, Writable> expected = new HashMap<Writable, Writable>();
     expected.put(new IntWritable(0), new IntWritable(1));
     expected.put(new IntWritable(2), new IntWritable(3));

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
index 8646ff4..1ca6861 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java
@@ -58,11 +58,9 @@ public class TestDeepParquetHiveMapInspector extends TestCase {
     final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
     final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};
 
-    final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
+    final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{
       new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});
 
-    final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});
-
     assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map,
new IntWritable(0)));
     assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map,
new IntWritable(2)));
     assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map,
new ShortWritable((short) 0)));

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
index f3a24af..0ce654d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java
@@ -58,9 +58,8 @@ public class TestParquetHiveArrayInspector extends TestCase {
 
   @Test
   public void testRegularList() {
-    final ArrayWritable internalList = new ArrayWritable(Writable.class,
+    final ArrayWritable list = new ArrayWritable(Writable.class,
             new Writable[]{new IntWritable(3), new IntWritable(5), new IntWritable(1)});
-    final ArrayWritable list = new ArrayWritable(ArrayWritable.class, new ArrayWritable[]{internalList});
 
     final List<Writable> expected = new ArrayList<Writable>();
     expected.add(new IntWritable(3));

http://git-wip-us.apache.org/repos/asf/hive/blob/1f75e348/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
index 278419f..950956a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java
@@ -57,11 +57,9 @@ public class TestStandardParquetHiveMapInspector extends TestCase {
     final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
     final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};
 
-    final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
+    final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{
       new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});
 
-    final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});
-
     assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map,
new IntWritable(0)));
     assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map,
new IntWritable(2)));
     assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short)
0)));


Mime
View raw message