parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nkol...@apache.org
Subject [parquet-mr] branch master updated: PARQUET-1303 correct ClassCastException for Avro @Stringable fields (#482)
Date Thu, 25 Jul 2019 13:15:41 GMT
This is an automated email from the ASF dual-hosted git repository.

nkollar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git


The following commit(s) were added to refs/heads/master by this push:
     new 93af6b4  PARQUET-1303 correct ClassCastException for Avro @Stringable fields (#482)
93af6b4 is described below

commit 93af6b4e8db84c813530cda763f378858e8c7700
Author: Zack Behringer <zebehringer@gmail.com>
AuthorDate: Thu Jul 25 09:15:35 2019 -0400

    PARQUET-1303 correct ClassCastException for Avro @Stringable fields (#482)
---
 .../org/apache/parquet/avro/AvroWriteSupport.java  |  4 +++-
 .../parquet/avro/TestReflectLogicalTypes.java      | 18 ++++++++++++----
 .../apache/parquet/avro/TestReflectReadWrite.java  | 24 +++++++++++++++++++++-
 3 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
index 2325dc5..859a133 100644
--- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
+++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroWriteSupport.java
@@ -367,8 +367,10 @@ public class AvroWriteSupport<T> extends WriteSupport<T>
{
     if (value instanceof Utf8) {
       Utf8 utf8 = (Utf8) value;
       return Binary.fromReusedByteArray(utf8.getBytes(), 0, utf8.getByteLength());
+    } else if (value instanceof CharSequence) {
+      return Binary.fromCharSequence((CharSequence) value);
     }
-    return Binary.fromCharSequence((CharSequence) value);
+    return Binary.fromCharSequence(value.toString());
   }
 
   private static GenericData getDataModel(Configuration conf) {
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
index ede5209..142191f 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectLogicalTypes.java
@@ -425,7 +425,7 @@ public class TestReflectLogicalTypes {
         read(REFLECT, nullableUuidStringSchema, test));
   }
 
-  @Test(expected = ClassCastException.class)
+  @Test
   public void testWriteUUIDMissingLogicalType() throws IOException {
     Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName())
         .fields().requiredString("uuid").endRecord();
@@ -439,6 +439,11 @@ public class TestReflectLogicalTypes {
     RecordWithUUID r2 = new RecordWithUUID();
     r2.uuid = u2;
 
+    List<RecordWithStringUUID> expected = Arrays.asList(
+        new RecordWithStringUUID(), new RecordWithStringUUID());
+    expected.get(0).uuid = u1.toString();
+    expected.get(1).uuid = u2.toString();
+
     // write without using REFLECT, which has the logical type
     File test = write(uuidSchema, r1, r2);
 
@@ -447,9 +452,14 @@ public class TestReflectLogicalTypes {
         .record(RecordWithStringUUID.class.getName())
         .fields().requiredString("uuid").endRecord();
 
-    // this fails with an AppendWriteException wrapping ClassCastException
-    // because the UUID isn't converted to a CharSequence expected internally
-    read(ReflectData.get(), uuidStringSchema, test);
+    Assert.assertEquals("Should read uuid as String without UUID conversion",
+        expected,
+        read(REFLECT, uuidStringSchema, test));
+
+    Assert.assertEquals("Should read uuid as String without UUID logical type",
+        expected,
+        read(ReflectData.get(), uuidStringSchema, test)
+        );
   }
 
   @Test
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java
index 2c88d05..539427b 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReflectReadWrite.java
@@ -30,6 +30,7 @@ import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.reflect.Stringable;
 import org.apache.avro.util.Utf8;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -101,6 +102,7 @@ public class TestReflectReadWrite {
         schema.getField("mystringarray").schema(), Lists.newArrayList(new Utf8("a"), new
Utf8("b"))));
     record.put("mylist", new GenericData.Array<Utf8>(
         schema.getField("mylist").schema(), Lists.newArrayList(new Utf8("a"), new Utf8("b"),
new Utf8("c"))));
+    record.put("mystringable", new StringableObj("blah blah"));
     return record;
   }
 
@@ -124,6 +126,7 @@ public class TestReflectReadWrite {
     object.myintarray = new int[] { 1, 2 };
     object.mystringarray = new String[] { "a", "b" };
     object.mylist = Lists.newArrayList("a", "b", "c");
+    object.mystringable = new StringableObj("blah blah");
     return object;
   }
 
@@ -154,6 +157,21 @@ public class TestReflectReadWrite {
     A, B
   }
 
+  public static class StringableObj {
+    private String value;
+    public StringableObj(String value) {
+      this.value = value;
+    }
+    @Override
+    public String toString() {
+      return this.value;
+    }
+    @Override
+    public boolean equals(Object other) {
+      return other instanceof StringableObj && this.value.equals(((StringableObj)other).value);
+    }
+  }
+
   public static class Pojo {
     public boolean myboolean;
     public byte mybyte;
@@ -171,6 +189,8 @@ public class TestReflectReadWrite {
     private int[] myintarray;
     private String[] mystringarray;
     private List<String> mylist;
+    @Stringable
+    private StringableObj mystringable;
 
     @Override
     public boolean equals(Object o) {
@@ -190,7 +210,8 @@ public class TestReflectReadWrite {
           && Arrays.equals(myshortarray, that.myshortarray)
           && Arrays.equals(myintarray, that.myintarray)
           && Arrays.equals(mystringarray, that.mystringarray)
-          && mylist.equals(that.mylist);
+          && mylist.equals(that.mylist)
+          && mystringable.equals(that.mystringable);
     }
 
     @Override
@@ -211,6 +232,7 @@ public class TestReflectReadWrite {
           ", myintarray=" + Arrays.toString(myintarray) +
           ", mystringarray=" + Arrays.toString(mystringarray) +
           ", mylist=" + mylist +
+          ", mystringable=" + mystringable.toString() +
           '}';
     }
   }


Mime
View raw message