Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8AAE417913 for ; Fri, 29 May 2015 04:46:50 +0000 (UTC) Received: (qmail 98826 invoked by uid 500); 29 May 2015 04:46:44 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 98688 invoked by uid 500); 29 May 2015 04:46:44 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 94160 invoked by uid 99); 29 May 2015 04:46:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 May 2015 04:46:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 562D4E1118; Fri, 29 May 2015 04:46:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xuefu@apache.org To: commits@hive.apache.org Date: Fri, 29 May 2015 04:47:35 -0000 Message-Id: <4d800408f3a9456eb2e84690be46d593@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [56/84] [abbrv] hive git commit: Revert "HIVE-9605:Remove parquet nested objects from wrapper writable objects (Sergio Pena, reviewed by Ferdinand Xu)" Revert "HIVE-9605:Remove parquet nested objects from wrapper writable objects (Sergio Pena, reviewed by Ferdinand Xu)" This reverts commit 1f75e348a90d44bbd6a836ecf7228a092c5ff709. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db8067f9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db8067f9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db8067f9 Branch: refs/heads/spark Commit: db8067f96dc57ad78f24b7128b39892682881423 Parents: c19efd6 Author: Ferdinand Xu Authored: Tue May 26 02:39:05 2015 -0400 Committer: Ferdinand Xu Committed: Tue May 26 02:39:05 2015 -0400 ---------------------------------------------------------------------- .../benchmark/storage/ColumnarStorageBench.java | 4 +- .../convert/HiveCollectionConverter.java | 5 ++- .../io/parquet/convert/HiveGroupConverter.java | 10 +++++ .../hive/ql/io/parquet/convert/Repeated.java | 8 ++-- .../serde/AbstractParquetMapInspector.java | 13 +++--- .../serde/DeepParquetHiveMapInspector.java | 6 ++- .../serde/ParquetHiveArrayInspector.java | 43 ++++++++++++++----- .../serde/StandardParquetHiveMapInspector.java | 6 +-- .../ql/io/parquet/TestArrayCompatibility.java | 44 ++++++++++---------- .../ql/io/parquet/TestDataWritableWriter.java | 44 ++++++++++++-------- .../hive/ql/io/parquet/TestMapStructures.java | 36 ++++++++-------- .../hive/ql/io/parquet/TestParquetSerDe.java | 8 +++- .../serde/TestAbstractParquetMapInspector.java | 4 +- .../serde/TestDeepParquetHiveMapInspector.java | 4 +- .../serde/TestParquetHiveArrayInspector.java | 3 +- .../TestStandardParquetHiveMapInspector.java | 4 +- 16 files changed, 150 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java ---------------------------------------------------------------------- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java index 4f6985c..61c2eb4 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/storage/ColumnarStorageBench.java @@ -209,13 +209,13 @@ public class ColumnarStorageBench { case LIST: { List elementType = new ArrayList(); elementType.add(((ListTypeInfo) type).getListElementTypeInfo()); - fields[pos++] = createRecord(elementType); + fields[pos++] = record(createRecord(elementType)); } break; case MAP: { List keyValueType = new ArrayList(); keyValueType.add(((MapTypeInfo) type).getMapKeyTypeInfo()); keyValueType.add(((MapTypeInfo) type).getMapValueTypeInfo()); - fields[pos++] = record(createRecord(keyValueType)); + fields[pos++] = record(record(createRecord(keyValueType))); } break; case STRUCT: { List elementType = ((StructTypeInfo) type).getAllStructFieldTypeInfos(); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java index 80405bd..0fd538e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.io.parquet.convert; import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; @@ -82,8 +83,8 @@ public class HiveCollectionConverter extends HiveGroupConverter { @Override public void end() { - parent.set(index, new ArrayWritable( - Writable.class, list.toArray(new Writable[0]))); + parent.set(index, wrapList(new ArrayWritable( + Writable.class, list.toArray(new Writable[list.size()])))); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java index c6d03a1..4809f9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java @@ -13,6 +13,7 @@ */ package org.apache.hadoop.hive.ql.io.parquet.convert; +import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; import parquet.io.api.Converter; import parquet.io.api.GroupConverter; @@ -71,6 +72,15 @@ public abstract class HiveGroupConverter extends GroupConverter implements Conve return getConverterFromDescription(type.asGroupType(), index, parent); } + /** + * The original list and map conversion didn't remove the synthetic layer and + * the ObjectInspector had to remove it. This is a temporary fix that adds an + * extra layer for the ObjectInspector to remove. + */ + static ArrayWritable wrapList(ArrayWritable list) { + return new ArrayWritable(Writable.class, new Writable[] {list}); + } + public abstract void set(int index, Writable value); } http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java index 4becd20..0130aef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java @@ -125,8 +125,8 @@ public interface Repeated extends ConverterParent { @Override public void parentEnd() { - parent.set(index, new ArrayWritable( - Writable.class, list.toArray(new Writable[list.size()]))); + parent.set(index, HiveGroupConverter.wrapList(new ArrayWritable( + Writable.class, list.toArray(new Writable[list.size()])))); } @Override @@ -185,8 +185,8 @@ public interface Repeated extends ConverterParent { @Override public void parentEnd() { - parent.set(index, new ArrayWritable( - Writable.class, list.toArray(new Writable[list.size()]))); + parent.set(index, wrapList(new ArrayWritable( + Writable.class, list.toArray(new Writable[list.size()])))); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java index 49bf1c5..62c61fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java @@ -59,12 +59,15 @@ public abstract class AbstractParquetMapInspector implements SettableMapObjectIn } if (data instanceof ArrayWritable) { - final Writable[] mapArray = ((ArrayWritable) data).get(); - if (mapArray == null || mapArray.length == 0) { + final Writable[] mapContainer = ((ArrayWritable) data).get(); + + if (mapContainer == null || mapContainer.length == 0) { return null; } + final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get(); final Map map = new LinkedHashMap(); + for (final Writable obj : mapArray) { final ArrayWritable mapObj = (ArrayWritable) obj; final Writable[] arr = mapObj.get(); @@ -88,12 +91,12 @@ public abstract class AbstractParquetMapInspector implements SettableMapObjectIn } if (data instanceof ArrayWritable) { - final Writable[] mapArray = ((ArrayWritable) data).get(); + final Writable[] mapContainer = ((ArrayWritable) data).get(); - if (mapArray == null || mapArray.length == 0) { + if (mapContainer == null || mapContainer.length == 0) { return -1; } else { - return mapArray.length; + return ((ArrayWritable) mapContainer[0]).get().length; } } http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java index 143d72e..d38c641 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/DeepParquetHiveMapInspector.java @@ -40,12 +40,14 @@ public class DeepParquetHiveMapInspector extends AbstractParquetMapInspector { } if (data instanceof ArrayWritable) { - final Writable[] mapArray = ((ArrayWritable) data).get(); + final Writable[] mapContainer = ((ArrayWritable) data).get(); - if (mapArray == null || mapArray.length == 0) { + if (mapContainer == null || mapContainer.length == 0) { return null; } + final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get(); + for (final Writable obj : mapArray) { final ArrayWritable mapObj = (ArrayWritable) obj; final Writable[] arr = mapObj.get(); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java index bde0dcb..53ca31d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java @@ -56,13 +56,20 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { } if (data instanceof ArrayWritable) { - final Writable[] array = ((ArrayWritable) data).get(); - if (array == null || array.length == 0) { + final Writable[] listContainer = ((ArrayWritable) data).get(); + + if (listContainer == null || listContainer.length == 0) { + return null; + } + + final Writable subObj = listContainer[0]; + + if (subObj == null) { return null; } - if (index >= 0 && index < array.length) { - return array[index]; + if (index >= 0 && index < ((ArrayWritable) subObj).get().length) { + return ((ArrayWritable) subObj).get()[index]; } else { return null; } @@ -78,12 +85,19 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { } if (data instanceof ArrayWritable) { - final Writable[] array = ((ArrayWritable) data).get(); - if (array == null || array.length == 0) { + final Writable[] listContainer = ((ArrayWritable) data).get(); + + if (listContainer == null || listContainer.length == 0) { return -1; } - return array.length; + final Writable subObj = listContainer[0]; + + if (subObj == null) { + return 0; + } + + return ((ArrayWritable) subObj).get().length; } throw new UnsupportedOperationException("Cannot inspect " + data.getClass().getCanonicalName()); @@ -96,12 +110,21 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { } if (data instanceof ArrayWritable) { - final Writable[] array = ((ArrayWritable) data).get(); - if (array == null || array.length == 0) { + final Writable[] listContainer = ((ArrayWritable) data).get(); + + if (listContainer == null || listContainer.length == 0) { return null; } - final List list = new ArrayList(array.length); + final Writable subObj = listContainer[0]; + + if (subObj == null) { + return null; + } + + final Writable[] array = ((ArrayWritable) subObj).get(); + final List list = new ArrayList(); + for (final Writable obj : array) { list.add(obj); } http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java index 22250b3..5aa1448 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/StandardParquetHiveMapInspector.java @@ -37,12 +37,12 @@ public class StandardParquetHiveMapInspector extends AbstractParquetMapInspector return null; } if (data instanceof ArrayWritable) { - final Writable[] mapArray = ((ArrayWritable) data).get(); + final Writable[] mapContainer = ((ArrayWritable) data).get(); - if (mapArray == null || mapArray.length == 0) { + if (mapContainer == null || mapContainer.length == 0) { return null; } - + final Writable[] mapArray = ((ArrayWritable) mapContainer[0]).get(); for (final Writable obj : mapArray) { final ArrayWritable mapObj = (ArrayWritable) obj; final Writable[] arr = mapObj.get(); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java index e0ea262..f349aa0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java @@ -62,8 +62,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( - new IntWritable(34), new IntWritable(35), new IntWritable(36)); + ArrayWritable expected = record(list( + new IntWritable(34), new IntWritable(35), new IntWritable(36))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -109,9 +109,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new FloatWritable(1.0f), new FloatWritable(1.0f)), - record(new FloatWritable(2.0f), new FloatWritable(2.0f))); + record(new FloatWritable(2.0f), new FloatWritable(2.0f)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -148,8 +148,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( - new IntWritable(34), new IntWritable(35), new IntWritable(36)); + ArrayWritable expected = record(list( + new IntWritable(34), new IntWritable(35), new IntWritable(36))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -198,9 +198,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new LongWritable(1234L)), - record(new LongWritable(2345L))); + record(new LongWritable(2345L)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -237,8 +237,8 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( - new IntWritable(34), new IntWritable(35), new IntWritable(36)); + ArrayWritable expected = record(list( + new IntWritable(34), new IntWritable(35), new IntWritable(36))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -287,9 +287,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new LongWritable(1234L)), - record(new LongWritable(2345L))); + record(new LongWritable(2345L)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -339,9 +339,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( new LongWritable(1234L), - new LongWritable(2345L)); + new LongWritable(2345L))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -397,9 +397,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new DoubleWritable(0.0), new DoubleWritable(0.0)), - record(new DoubleWritable(0.0), new DoubleWritable(180.0))); + record(new DoubleWritable(0.0), new DoubleWritable(180.0)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -473,10 +473,10 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new DoubleWritable(0.0), new DoubleWritable(0.0)), null, - record(new DoubleWritable(0.0), new DoubleWritable(180.0))); + record(new DoubleWritable(0.0), new DoubleWritable(180.0)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -546,9 +546,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new DoubleWritable(0.0), new DoubleWritable(180.0)), - record(new DoubleWritable(0.0), new DoubleWritable(0.0))); + record(new DoubleWritable(0.0), new DoubleWritable(0.0)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -619,9 +619,9 @@ public class TestArrayCompatibility extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new DoubleWritable(0.0), new DoubleWritable(180.0)), - record(new DoubleWritable(0.0), new DoubleWritable(0.0))); + record(new DoubleWritable(0.0), new DoubleWritable(0.0)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java index 73425e3..8f03c5b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java @@ -311,11 +311,13 @@ public class TestDataWritableWriter { + "}\n"; ArrayWritable hiveRecord = createGroup( - createArray( - createInt(1), - createNull(), - createInt(2) - ) + createGroup( + createArray( + createInt(1), + createNull(), + createInt(2) + ) + ) ); // Write record to Parquet format @@ -361,16 +363,18 @@ public class TestDataWritableWriter { ArrayWritable hiveRecord = createGroup( createGroup( createArray( - createString("key1"), - createInt(1) - ), - createArray( - createString("key2"), - createInt(2) - ), - createArray( - createString("key3"), - createNull() + createArray( + createString("key1"), + createInt(1) + ), + createArray( + createString("key2"), + createInt(2) + ), + createArray( + createString("key3"), + createNull() + ) ) ) ); @@ -428,10 +432,14 @@ public class TestDataWritableWriter { + "}\n"; ArrayWritable hiveRecord = createGroup( - createArray( + createGroup( createArray( - createInt(1), - createInt(2) + createGroup( + createArray( + createInt(1), + createInt(2) + ) + ) ) ) ); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java index 32264a4..6fc83e0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java @@ -82,9 +82,9 @@ public class TestMapStructures extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new Text("lettuce"), new IntWritable(34)), - record(new Text("cabbage"), new IntWritable(18))); + record(new Text("cabbage"), new IntWritable(18)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -149,10 +149,10 @@ public class TestMapStructures extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( + ArrayWritable expected = record(list( record(new Text("lettuce"), new IntWritable(34)), record(new Text("kale"), null), - record(new Text("cabbage"), new IntWritable(18))); + record(new Text("cabbage"), new IntWritable(18)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -230,9 +230,9 @@ public class TestMapStructures extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( - record(new Text("green"), record(new Text("lettuce"), new Text("kale"), null)), - record(new Text("brown"), null)); + ArrayWritable expected = record(list( + record(new Text("green"), list(new Text("lettuce"), new Text("kale"), null)), + record(new Text("brown"), null))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -325,9 +325,9 @@ public class TestMapStructures extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( - record(new Text("low"), record(new IntWritable(34), new IntWritable(35), null)), - record(new Text("high"), record(new IntWritable(340), new IntWritable(360)))); + ArrayWritable expected = record(list( + record(new Text("low"), list(new IntWritable(34), new IntWritable(35), null)), + record(new Text("high"), list(new IntWritable(340), new IntWritable(360))))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -386,9 +386,9 @@ public class TestMapStructures extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list(record( + ArrayWritable expected = record(list(record( record(new IntWritable(7), new IntWritable(22)), - new DoubleWritable(3.14))); + new DoubleWritable(3.14)))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -447,9 +447,9 @@ public class TestMapStructures extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list(record( + ArrayWritable expected = record(list(record( new DoubleWritable(3.14), - record(new IntWritable(7), new IntWritable(22)))); + record(new IntWritable(7), new IntWritable(22))))); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); @@ -542,13 +542,13 @@ public class TestMapStructures extends AbstractTestParquetDirect { } }); - ArrayWritable expected = list( - record(new Text("a"), record( + ArrayWritable expected = record(list( + record(new Text("a"), list( record(new Text("b"), new IntWritable(1)))), - record(new Text("b"), record( + record(new Text("b"), list( record(new Text("a"), new IntWritable(-1)), record(new Text("b"), new IntWritable(-2)))) - ); + )); List records = read(test); Assert.assertEquals("Should have only one record", 1, records.size()); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java index dbb2795..21f889a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetSerDe.java @@ -55,6 +55,7 @@ public class TestParquetSerDe extends TestCase { arr[4] = new DoubleWritable((double) 5.3); arr[5] = new BytesWritable("hive and hadoop and parquet. Big family.".getBytes("UTF-8")); arr[6] = new BytesWritable("parquetSerde binary".getBytes("UTF-8")); + final Writable[] mapContainer = new Writable[1]; final Writable[] map = new Writable[3]; for (int i = 0; i < 3; ++i) { final Writable[] pair = new Writable[2]; @@ -62,13 +63,16 @@ public class TestParquetSerDe extends TestCase { pair[1] = new IntWritable(i); map[i] = new ArrayWritable(Writable.class, pair); } - arr[7] = new ArrayWritable(Writable.class, map); + mapContainer[0] = new ArrayWritable(Writable.class, map); + arr[7] = new ArrayWritable(Writable.class, mapContainer); + final Writable[] arrayContainer = new Writable[1]; final Writable[] array = new Writable[5]; for (int i = 0; i < 5; ++i) { array[i] = new BytesWritable(("elem_" + i).getBytes("UTF-8")); } - arr[8] = new ArrayWritable(Writable.class, array); + arrayContainer[0] = new ArrayWritable(Writable.class, array); + arr[8] = new ArrayWritable(Writable.class, arrayContainer); final ArrayWritable arrWritable = new ArrayWritable(Writable.class, arr); // Test http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java index f5d9cb4..ef05150 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestAbstractParquetMapInspector.java @@ -71,9 +71,11 @@ public class TestAbstractParquetMapInspector extends TestCase { final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)}; final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)}; - final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{ + final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{ new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)}); + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap}); + final Map expected = new HashMap(); expected.put(new IntWritable(0), new IntWritable(1)); expected.put(new IntWritable(2), new IntWritable(3)); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java index 1ca6861..8646ff4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java @@ -58,9 +58,11 @@ public class TestDeepParquetHiveMapInspector extends TestCase { final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)}; final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)}; - final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{ + final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{ new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)}); + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap}); + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0))); assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2))); assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new ShortWritable((short) 0))); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java index 0ce654d..f3a24af 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java @@ -58,8 +58,9 @@ public class TestParquetHiveArrayInspector extends TestCase { @Test public void testRegularList() { - final ArrayWritable list = new ArrayWritable(Writable.class, + final ArrayWritable internalList = new ArrayWritable(Writable.class, new Writable[]{new IntWritable(3), new IntWritable(5), new IntWritable(1)}); + final ArrayWritable list = new ArrayWritable(ArrayWritable.class, new ArrayWritable[]{internalList}); final List expected = new ArrayList(); expected.add(new IntWritable(3)); http://git-wip-us.apache.org/repos/asf/hive/blob/db8067f9/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java index 950956a..278419f 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java @@ -57,9 +57,11 @@ public class TestStandardParquetHiveMapInspector extends TestCase { final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)}; final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)}; - final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{ + final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{ new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)}); + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap}); + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0))); assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2))); assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 0)));