parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject parquet-mr git commit: PARQUET-387: Improve NPE message when avro arrays contain null.
Date Fri, 04 Dec 2015 19:47:49 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master e32aa6fe0 -> 14097c64d


PARQUET-387: Improve NPE message when avro arrays contain null.

Previously, the NPE had no error message but the Avro support accepts
schemas that have nullable array elements.

Author: Ryan Blue <blue@apache.org>

Closes #291 from rdblue/PARQUET-387-fix-npe-message and squashes the following commits:

39d3c83 [Ryan Blue] PARQUET-387: Update test case to verify help message.
d6b6bd8 [Ryan Blue] PARQUET-387: Improve NPE message when avro arrays contain null.


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/14097c64
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/14097c64
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/14097c64

Branch: refs/heads/master
Commit: 14097c64d243794610788d3ebb2e81ba8fd867c0
Parents: e32aa6f
Author: Ryan Blue <blue@apache.org>
Authored: Fri Dec 4 11:47:38 2015 -0800
Committer: Ryan Blue <blue@apache.org>
Committed: Fri Dec 4 11:47:38 2015 -0800

----------------------------------------------------------------------
 .../apache/parquet/avro/AvroWriteSupport.java   | 38 +++++++++++++++++---
 .../avro/TestReadWriteOldListBehavior.java      |  4 ++-
 2 files changed, 37 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/14097c64/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
----------------------------------------------------------------------
diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
index 35e3924..48fc01e 100644
--- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
+++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
@@ -452,8 +452,24 @@ public class AvroWriteSupport<T> extends WriteSupport<T>
{
                                 Collection<?> array) {
       if (array.size() > 0) {
         recordConsumer.startField(OLD_LIST_REPEATED_NAME, 0);
-        for (Object elt : array) {
-          writeValue(schema.getType(0), avroSchema.getElementType(), elt);
+        try {
+          for (Object elt : array) {
+            writeValue(schema.getType(0), avroSchema.getElementType(), elt);
+          }
+        } catch (NullPointerException e) {
+          // find the null element and throw a better error message
+          int i = 0;
+          for (Object elt : array) {
+            if (elt == null) {
+              throw new NullPointerException(
+                  "Array contains a null element at " + i + "\n" +
+                  "Set parquet.avro.write-old-list-structure=false to turn " +
+                  "on support for arrays with null elements.");
+            }
+            i += 1;
+          }
+          // no element was null, throw the original exception
+          throw e;
         }
         recordConsumer.endField(OLD_LIST_REPEATED_NAME, 0);
       }
@@ -464,8 +480,22 @@ public class AvroWriteSupport<T> extends WriteSupport<T>
{
                                     Object[] array) {
       if (array.length > 0) {
         recordConsumer.startField(OLD_LIST_REPEATED_NAME, 0);
-        for (Object element : array) {
-          writeValue(type.getType(0), schema.getElementType(), element);
+        try {
+          for (Object element : array) {
+            writeValue(type.getType(0), schema.getElementType(), element);
+          }
+        } catch (NullPointerException e) {
+          // find the null element and throw a better error message
+          for (int i = 0; i < array.length; i += 1) {
+            if (array[i] == null) {
+              throw new NullPointerException(
+                  "Array contains a null element at " + i + "\n" +
+                  "Set parquet.avro.write-old-list-structure=false to turn " +
+                  "on support for arrays with null elements.");
+            }
+          }
+          // no element was null, throw the original exception
+          throw e;
         }
         recordConsumer.endField(OLD_LIST_REPEATED_NAME, 0);
       }

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/14097c64/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
----------------------------------------------------------------------
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
index 7c2bc27..64caacc 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
@@ -43,6 +43,7 @@ import org.apache.parquet.hadoop.api.WriteSupport;
 import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.io.api.RecordConsumer;
 import org.apache.parquet.schema.MessageTypeParser;
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -347,7 +348,8 @@ public class TestReadWriteOldListBehavior {
       writer.write(record);
       fail("Should not succeed writing an array with null values");
     } catch (Exception e) {
-      // expected
+      Assert.assertTrue("Error message should provide context and help",
+          e.getMessage().contains("parquet.avro.write-old-list-structure"));
     } finally {
       writer.close();
     }


Mime
View raw message