parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ziva...@apache.org
Subject [parquet-mr] branch master updated: PARQUET-1576 Bump Apache Avro to 1.9.0 (#638)
Date Thu, 30 May 2019 12:12:29 GMT
This is an automated email from the ASF dual-hosted git repository.

zivanfi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git


The following commit(s) were added to refs/heads/master by this push:
     new 9d6fb45  PARQUET-1576 Bump Apache Avro to 1.9.0 (#638)
9d6fb45 is described below

commit 9d6fb45e54da65cbd407bb3e7bff0981aa9f8f9f
Author: Fokko Driesprong <fokko@driesprong.frl>
AuthorDate: Thu May 30 14:12:24 2019 +0200

    PARQUET-1576 Bump Apache Avro to 1.9.0 (#638)
---
 .../java/org/apache/parquet/avro/AvroTestUtil.java |  5 ++--
 .../parquet/avro/TestAvroSchemaConverter.java      | 22 +++++++--------
 .../parquet/avro/TestReflectLogicalTypes.java      |  7 ++---
 .../apache/parquet/avro/TestSpecificReadWrite.java | 32 ++++++++++++----------
 pom.xml                                            |  2 +-
 5 files changed, 35 insertions(+), 33 deletions(-)

diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
index 801f2bd..717e36e 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
@@ -24,6 +24,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+
+import org.apache.avro.JsonProperties;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
@@ -31,7 +33,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.parquet.hadoop.ParquetReader;
 import org.apache.parquet.hadoop.ParquetWriter;
-import org.codehaus.jackson.node.NullNode;
 import org.junit.Assert;
 import org.junit.rules.TemporaryFolder;
 
@@ -52,7 +53,7 @@ public class AvroTestUtil {
   }
 
   public static Schema.Field optionalField(String name, Schema schema) {
-    return new Schema.Field(name, optional(schema), null, NullNode.getInstance());
+    return new Schema.Field(name, optional(schema), null, JsonProperties.NULL_VALUE);
   }
 
   public static Schema array(Schema element) {
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
index 2548408..36bd7ba 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
@@ -20,6 +20,7 @@ package org.apache.parquet.avro;
 
 import com.google.common.collect.Lists;
 import com.google.common.io.Resources;
+import org.apache.avro.JsonProperties;
 import org.apache.avro.LogicalTypes;
 import org.apache.avro.Schema;
 import org.apache.hadoop.conf.Configuration;
@@ -28,7 +29,6 @@ import org.apache.parquet.schema.MessageTypeParser;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
 import org.apache.parquet.schema.Types;
-import org.codehaus.jackson.node.NullNode;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -296,8 +296,8 @@ public class TestAvroSchemaConverter {
   public void testOptionalFields() throws Exception {
     Schema schema = Schema.createRecord("record1", null, null, false);
     Schema optionalInt = optional(Schema.create(INT));
-    schema.setFields(Arrays.asList(
-        new Schema.Field("myint", optionalInt, null, NullNode.getInstance())
+    schema.setFields(Collections.singletonList(
+      new Schema.Field("myint", optionalInt, null, JsonProperties.NULL_VALUE)
     ));
     testRoundTripConversion(
         schema,
@@ -351,7 +351,7 @@ public class TestAvroSchemaConverter {
         Schema.create(INT),
         Schema.create(Schema.Type.FLOAT)));
     schema.setFields(Arrays.asList(
-        new Schema.Field("myunion", multipleTypes, null, NullNode.getInstance())));
+        new Schema.Field("myunion", multipleTypes, null, JsonProperties.NULL_VALUE)));
 
     // Avro union is modelled using optional data members of the different
     // types. This does not translate back into an Avro union
@@ -370,8 +370,8 @@ public class TestAvroSchemaConverter {
     Schema innerRecord = Schema.createRecord("element", null, null, false);
     Schema optionalString = optional(Schema.create(Schema.Type.STRING));
     innerRecord.setFields(Lists.newArrayList(
-        new Schema.Field("s1", optionalString, null, NullNode.getInstance()),
-        new Schema.Field("s2", optionalString, null, NullNode.getInstance())
+        new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
+        new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)
     ));
     Schema schema = Schema.createRecord("HasArray", null, null, false);
     schema.setFields(Lists.newArrayList(
@@ -397,8 +397,8 @@ public class TestAvroSchemaConverter {
     Schema innerRecord = Schema.createRecord("InnerRecord", null, null, false);
     Schema optionalString = optional(Schema.create(Schema.Type.STRING));
     innerRecord.setFields(Lists.newArrayList(
-        new Schema.Field("s1", optionalString, null, NullNode.getInstance()),
-        new Schema.Field("s2", optionalString, null, NullNode.getInstance())
+        new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
+        new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)
     ));
     Schema schema = Schema.createRecord("HasArray", null, null, false);
     schema.setFields(Lists.newArrayList(
@@ -424,7 +424,7 @@ public class TestAvroSchemaConverter {
         Schema.create(INT))));
     Schema schema = Schema.createRecord("AvroCompatListInList", null, null, false);
     schema.setFields(Lists.newArrayList(
-        new Schema.Field("listOfLists", listOfLists, null, NullNode.getInstance())
+        new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
     ));
     System.err.println("Avro schema: " + schema.toString(true));
 
@@ -453,7 +453,7 @@ public class TestAvroSchemaConverter {
         Schema.create(INT))));
     Schema schema = Schema.createRecord("ThriftCompatListInList", null, null, false);
     schema.setFields(Lists.newArrayList(
-        new Schema.Field("listOfLists", listOfLists, null, NullNode.getInstance())
+        new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
     ));
     System.err.println("Avro schema: " + schema.toString(true));
 
@@ -486,7 +486,7 @@ public class TestAvroSchemaConverter {
         Schema.create(INT))));
     Schema schema = Schema.createRecord("UnknownTwoLevelListInList", null, null, false);
     schema.setFields(Lists.newArrayList(
-        new Schema.Field("listOfLists", listOfLists, null, NullNode.getInstance())
+        new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
     ));
     System.err.println("Avro schema: " + schema.toString(true));
 
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
index 401e698..ede5209 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
@@ -120,7 +120,7 @@ public class TestReflectLogicalTypes {
   public void testDecimalBytes() throws IOException {
     Schema schema = REFLECT.getSchema(DecimalRecordBytes.class);
     Assert.assertEquals("Should have the correct record name",
-        "org.apache.parquet.avro.TestReflectLogicalTypes$",
+        "org.apache.parquet.avro.TestReflectLogicalTypes",
         schema.getNamespace());
     Assert.assertEquals("Should have the correct record name",
         "DecimalRecordBytes",
@@ -179,7 +179,7 @@ public class TestReflectLogicalTypes {
   public void testDecimalFixed() throws IOException {
     Schema schema = REFLECT.getSchema(DecimalRecordFixed.class);
     Assert.assertEquals("Should have the correct record name",
-        "org.apache.parquet.avro.TestReflectLogicalTypes$",
+        "org.apache.parquet.avro.TestReflectLogicalTypes",
         schema.getNamespace());
     Assert.assertEquals("Should have the correct record name",
         "DecimalRecordFixed",
@@ -260,7 +260,6 @@ public class TestReflectLogicalTypes {
   }
 
   @Test
-  @SuppressWarnings("unchecked")
   public void testPairRecord() throws IOException {
     ReflectData model = new ReflectData();
     model.addLogicalTypeConversion(new Conversion<Pair>() {
@@ -298,7 +297,7 @@ public class TestReflectLogicalTypes {
 
     Schema schema = model.getSchema(PairRecord.class);
     Assert.assertEquals("Should have the correct record name",
-        "org.apache.parquet.avro.TestReflectLogicalTypes$",
+        "org.apache.parquet.avro.TestReflectLogicalTypes",
         schema.getNamespace());
     Assert.assertEquals("Should have the correct record name",
         "PairRecord",
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java
index fa37d54..46e2f2c 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestSpecificReadWrite.java
@@ -1,4 +1,4 @@
-/* 
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,6 +19,8 @@
 package org.apache.parquet.avro;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNull;
 import static org.apache.parquet.filter.ColumnPredicates.equalTo;
 import static org.apache.parquet.filter.ColumnRecordFilter.column;
@@ -205,12 +207,12 @@ public class TestSpecificReadWrite {
 
     try(ParquetReader<Car> reader = new AvroParquetReader<Car>(conf, path)) {
       for (Car car = reader.read(); car != null; car = reader.read()) {
-        assertEquals(car.getDoors() != null, true);
-        assertEquals(car.getEngine() != null, true);
-        assertEquals(car.getMake() != null, true);
-        assertEquals(car.getModel() != null, true);
-        assertEquals(car.getYear() != null, true);
-        assertEquals(car.getVin() != null, true);
+        assertTrue(car.getDoors() == 4 || car.getDoors() == 5);
+        assertNotNull(car.getEngine());
+        assertNotNull(car.getMake());
+        assertNotNull(car.getModel());
+        assertEquals(2010, car.getYear());
+        assertNotNull(car.getVin());
         assertNull(car.getOptionalExtra());
         assertNull(car.getServiceHistory());
       }
@@ -225,12 +227,12 @@ public class TestSpecificReadWrite {
 
     try(ParquetReader<NewCar> reader = new AvroParquetReader<>(conf, path)) {
       for (NewCar car = reader.read(); car != null; car = reader.read()) {
-        assertEquals(car.getEngine() != null, true);
-        assertEquals(car.getBrand() != null, true);
-        assertEquals(car.getYear() != null, true);
-        assertEquals(car.getVin() != null, true);
-        assertEquals(car.getDescription() == null, true);
-        assertEquals(car.getOpt() == 5, true);
+        assertNotNull(car.getEngine());
+        assertNotNull(car.getBrand());
+        assertEquals(2010, car.getYear());
+        assertNotNull(car.getVin());
+        assertNull(car.getDescription());
+        assertEquals(5, car.getOpt());
       }
     }
   }
diff --git a/pom.xml b/pom.xml
index 0d52add..f8be2ea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,7 +97,7 @@
     <fastutil.version>7.0.13</fastutil.version>
     <semver.api.version>0.9.33</semver.api.version>
     <slf4j.version>1.7.22</slf4j.version>
-    <avro.version>1.8.2</avro.version>
+    <avro.version>1.9.0</avro.version>
     <guava.version>27.0.1-jre</guava.version>
     <brotli-codec.version>0.1.1</brotli-codec.version>
     <mockito.version>1.10.19</mockito.version>


Mime
View raw message