parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject parquet-mr git commit: PARQUET-1133 Add int96 support by returning bytearray, Skip originalType comparison for map types when originalType is null
Date Tue, 10 Oct 2017 23:21:09 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master ddbeb4dd1 -> d55a572e5


PARQUET-1133 Add int96 support by returning bytearray, Skip originalType comparison for map
types when originalType is null

- PigSchemaConverter: Added a null check before comparing a mapKeyValueType's original type
with the static constant
- PigSchemaConverter: Changed the handling of int96 types - return bytearray instead of rejecting
input
- PigSchemaConverterTest: Added unit tests for int96 conversion and handling map entries without
original type specified

Author: Addisu Feyissa <addisu.feyissa@C1159.local>

Closes #422 from adisu-feyissa/hotfix/remove_originalType_check_for_maps_and_add_int96_support
and squashes the following commits:

e6fa3444 [Addisu Feyissa] - PigSchemaConverter: Added a null check before comparing a mapKeyValueType's
original type with the static constant - PigSchemaConverter: Changed the handling of int96
types - return bytearray instead of rejecting input - PigSchemaConverTest: Added unit tests
for int96 conversion and handling map entries without original type specified


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/d55a572e
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/d55a572e
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/d55a572e

Branch: refs/heads/master
Commit: d55a572e5867832f6d5755fcd46101da51a38aa4
Parents: ddbeb4d
Author: Addisu Feyissa <addisu.feyissa@C1159.local>
Authored: Tue Oct 10 16:20:55 2017 -0700
Committer: Julien Le Dem <julien@ledem.net>
Committed: Tue Oct 10 16:20:55 2017 -0700

----------------------------------------------------------------------
 .../apache/parquet/pig/PigSchemaConverter.java  |  5 ++--
 .../parquet/pig/TestPigSchemaConverter.java     | 28 ++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/d55a572e/parquet-pig/src/main/java/org/apache/parquet/pig/PigSchemaConverter.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/main/java/org/apache/parquet/pig/PigSchemaConverter.java b/parquet-pig/src/main/java/org/apache/parquet/pig/PigSchemaConverter.java
index e560e42..cf99534 100644
--- a/parquet-pig/src/main/java/org/apache/parquet/pig/PigSchemaConverter.java
+++ b/parquet-pig/src/main/java/org/apache/parquet/pig/PigSchemaConverter.java
@@ -239,7 +239,8 @@ public class PigSchemaConverter {
       @Override
       public FieldSchema convertINT96(PrimitiveTypeName primitiveTypeName)
           throws FrontendException {
-        throw new FrontendException("NYI");
+        LOG.warn("Converting type " + primitiveTypeName + " to bytearray");
+        return new FieldSchema(fieldName, null, DataType.BYTEARRAY);
       }
 
       @Override
@@ -283,7 +284,7 @@ public class PigSchemaConverter {
         }
         GroupType mapKeyValType = parquetGroupType.getType(0).asGroupType();
         if (!mapKeyValType.isRepetition(Repetition.REPEATED) ||
-            !mapKeyValType.getOriginalType().equals(OriginalType.MAP_KEY_VALUE) ||
+            (mapKeyValType.getOriginalType() != null && !mapKeyValType.getOriginalType().equals(OriginalType.MAP_KEY_VALUE))
||
             mapKeyValType.getFieldCount()!=2) {
           throw new SchemaConversionException("Invalid map type " + parquetGroupType);
         }

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/d55a572e/parquet-pig/src/test/java/org/apache/parquet/pig/TestPigSchemaConverter.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/test/java/org/apache/parquet/pig/TestPigSchemaConverter.java
b/parquet-pig/src/test/java/org/apache/parquet/pig/TestPigSchemaConverter.java
index 1fe0ccd..646e117 100644
--- a/parquet-pig/src/test/java/org/apache/parquet/pig/TestPigSchemaConverter.java
+++ b/parquet-pig/src/test/java/org/apache/parquet/pig/TestPigSchemaConverter.java
@@ -244,6 +244,34 @@ public class TestPigSchemaConverter {
   }
 
   @Test
+  public void testMapWithFixedWithoutOriginalType() throws Exception {
+    testFixedConversion(
+      "message spark_schema {\n" +
+      "  optional binary a;\n" +
+      "  optional group b (MAP) {\n" +
+      "    repeated group map {\n" +
+      "      required binary key;\n" +
+      "      optional group value {\n" +
+      "        optional fixed_len_byte_array(5) c;\n" +
+      "        optional fixed_len_byte_array(7) d;\n" +
+      "      }\n" +
+      "    }\n" +
+      "  }\n" +
+      "}\n",
+      "a:bytearray, b:[(c:bytearray, d:bytearray)]");
+  }
+
+  @Test
+  public void testInt96() throws Exception {
+    testFixedConversion(
+      "message spark_schema {\n" +
+        "  optional int96 datetime;\n" +
+        "}",
+      "datetime:bytearray"
+    );
+  }
+
+  @Test
   public void testAnnonymousField() throws Exception {
     testConversion(
         "a:chararray, int",


Mime
View raw message