parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject [parquet-mr] branch master updated: PARQUET-1305: Backward incompatible change introduced in 1.8 (#483)
Date Wed, 07 Nov 2018 10:19:15 GMT
This is an automated email from the ASF dual-hosted git repository.

gabor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git


The following commit(s) were added to refs/heads/master by this push:
     new b6fd45e  PARQUET-1305: Backward incompatible change introduced in 1.8 (#483)
b6fd45e is described below

commit b6fd45ee08e08b490c35825953b51cfd8fb1b072
Author: nandorKollar <nandorKollar@users.noreply.github.com>
AuthorDate: Wed Nov 7 11:19:10 2018 +0100

    PARQUET-1305: Backward incompatible change introduced in 1.8 (#483)
---
 .../apache/parquet/thrift/struct/ThriftType.java   |  8 +++---
 .../parquet/thrift/TestThriftSchemaConverter.java  | 30 +++++++++++++++++++---
 .../parquet/thrift/struct/TestThriftType.java      | 10 ++++----
 3 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
index 3d64e8d..5f8afe6 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
@@ -1,4 +1,4 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -230,7 +230,7 @@ public abstract class ThriftType {
     public StructType(@JsonProperty("children") List<ThriftField> children,
                       @JsonProperty("structOrUnionType") StructOrUnionType structOrUnionType)
{
       super(STRUCT);
-      this.structOrUnionType = structOrUnionType == null ? StructOrUnionType.UNKNOWN : structOrUnionType;
+      this.structOrUnionType = structOrUnionType == null ? StructOrUnionType.STRUCT : structOrUnionType;
       this.children = children;
       int maxId = 0;
       if (children != null) {
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftSchemaConverter.java
b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftSchemaConverter.java
index cd5fc47..2cde15b 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftSchemaConverter.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftSchemaConverter.java
@@ -1,4 +1,4 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,7 @@ import org.apache.parquet.schema.MessageTypeParser;
 import org.apache.parquet.thrift.projection.StrictFieldProjectionFilter;
 import org.apache.parquet.thrift.projection.ThriftProjectionException;
 import org.apache.parquet.thrift.projection.deprecated.DeprecatedFieldProjectionFilter;
+import org.apache.parquet.thrift.struct.ThriftField;
 import org.apache.parquet.thrift.struct.ThriftType;
 import org.apache.parquet.thrift.struct.ThriftType.StructType;
 import org.apache.parquet.thrift.test.compat.MapStructV2;
@@ -34,7 +35,10 @@ import com.twitter.data.proto.tutorial.thrift.AddressBook;
 import com.twitter.data.proto.tutorial.thrift.Person;
 import com.twitter.elephantbird.thrift.test.TestStructInMap;
 
+import java.util.Arrays;
+
 import static org.apache.parquet.schema.MessageTypeParser.parseMessageType;
+import static org.apache.parquet.thrift.struct.ThriftField.Requirement.REQUIRED;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
@@ -288,6 +292,26 @@ public class TestThriftSchemaConverter {
     }
   }
 
+  @Test
+  public void testConvertStructCreatedViaDeprecatedConstructor() {
+    String expected = "message ParquetSchema {\n" +
+      "  required binary a (UTF8) = 1;\n" +
+      "  required binary b (UTF8) = 2;\n" +
+      "}\n";
+
+    ThriftSchemaConverter converter = new ThriftSchemaConverter();
+
+    StructType structType = new StructType(
+      Arrays.asList(
+        new ThriftField("a", (short) 1, REQUIRED, new ThriftType.StringType()),
+        new ThriftField("b", (short) 2, REQUIRED, new ThriftType.StringType())
+      )
+    );
+
+    final MessageType converted = converter.convert(structType);
+    assertEquals(MessageTypeParser.parseMessageType(expected), converted);
+  }
+
   public static void shouldGetProjectedSchema(String deprecatedFilterDesc, String strictFilterDesc,
String expectedSchemaStr, Class<? extends TBase<?,?>> thriftClass) {
     MessageType depRequestedSchema = getDeprecatedFilteredSchema(deprecatedFilterDesc, thriftClass);
     MessageType strictRequestedSchema = getStrictFilteredSchema(strictFilterDesc, thriftClass);
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java
b/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java
index e700042..88e8bf1 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java
@@ -1,4 +1,4 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -35,7 +35,7 @@ public class TestThriftType {
     assertEquals("{\n"
                 +"  \"id\" : \"STRUCT\",\n"
                 +"  \"children\" : [ ],\n"
-                +"  \"structOrUnionType\" : \"UNKNOWN\"\n"
+                +"  \"structOrUnionType\" : \"STRUCT\"\n"
                 +"}", st.toJSON());
 
     st = new StructType(new LinkedList<ThriftField>(), StructOrUnionType.UNION);
@@ -60,7 +60,7 @@ public class TestThriftType {
     st = (StructType) StructType.fromJSON("{\"id\": \"STRUCT\", \"children\":[], \"structOrUnionType\":
\"STRUCT\"}");
     assertEquals(st.getStructOrUnionType(), StructOrUnionType.STRUCT);
     st = (StructType) StructType.fromJSON("{\"id\": \"STRUCT\", \"children\":[]}");
-    assertEquals(st.getStructOrUnionType(), StructOrUnionType.UNKNOWN);
+    assertEquals(st.getStructOrUnionType(), StructOrUnionType.STRUCT);
     st = (StructType) StructType.fromJSON("{\"id\": \"STRUCT\", \"children\":[], \"structOrUnionType\":
\"UNKNOWN\"}");
     assertEquals(st.getStructOrUnionType(), StructOrUnionType.UNKNOWN);
   }


Mime
View raw message