hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1671695 - in /hive/trunk/serde/src: java/org/apache/hadoop/hive/serde2/avro/ test/org/apache/hadoop/hive/serde2/avro/
Date Mon, 06 Apr 2015 21:05:22 GMT
Author: gunther
Date: Mon Apr  6 21:05:22 2015
New Revision: 1671695

URL: http://svn.apache.org/r1671695
Log:
HIVE-7049: Unable to deserialize AVRO data when file schema and record schema are different
and nullable (Daniel Dai, reviewed by Ashutosh Chauhan)

Modified:
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java?rev=1671695&r1=1671694&r2=1671695&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java Mon
Apr  6 21:05:22 2015
@@ -189,7 +189,7 @@ class AvroDeserializer {
       String columnName = columnNames.get(i);
       Object datum = record.get(columnName);
       Schema datumSchema = record.getSchema().getField(columnName).schema();
-      Schema.Field field = fileSchema.getField(columnName);
+      Schema.Field field = AvroSerdeUtils.isNullableType(fileSchema)?AvroSerdeUtils.getOtherTypeFromNullableType(fileSchema).getField(columnName):fileSchema.getField(columnName);
       objectRow.add(worker(datum, field == null ? null : field.schema(), datumSchema, columnType));
     }
 

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java?rev=1671695&r1=1671694&r2=1671695&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
Mon Apr  6 21:05:22 2015
@@ -185,9 +185,7 @@ public class TestAvroDeserializer {
 
   }
 
-  @Test
-  public void canDeserializeRecords() throws SerDeException, IOException {
-    Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
+  public void canDeserializeRecordsInternal(Schema s, Schema fileSchema) throws SerDeException,
IOException {
     GenericData.Record record = new GenericData.Record(s);
     GenericData.Record innerRecord = new GenericData.Record(s.getField("aRecord").schema());
     innerRecord.put("int1", 42);
@@ -196,7 +194,7 @@ public class TestAvroDeserializer {
     record.put("aRecord", innerRecord);
     assertTrue(GENERIC_DATA.validate(s, record));
 
-    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
+    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record, fileSchema);
 
     AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
 
@@ -232,6 +230,19 @@ public class TestAvroDeserializer {
     assertEquals(42432234234l, innerRecord2OI.getStructFieldData(innerRecord2, allStructFieldRefs1.get(2)));
   }
 
+  @Test
+  public void canDeserializeRecords() throws SerDeException, IOException {
+    Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
+    canDeserializeRecordsInternal(s, s);
+  }
+
+  @Test
+  public void canDeserializeNullableRecords() throws SerDeException, IOException {
+    Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.RECORD_SCHEMA);
+    Schema fileSchema = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.NULLABLE_RECORD_SCHEMA);
+    canDeserializeRecordsInternal(s, fileSchema);
+  }
+
   private class ResultPair { // Because Pairs give Java the vapors.
     public final ObjectInspector oi;
     public final Object value;

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java?rev=1671695&r1=1671694&r2=1671695&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
(original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
Mon Apr  6 21:05:22 2015
@@ -100,6 +100,7 @@ public class TestAvroObjectInspectorGene
       "    }\n" +
       "  ]\n" +
       "}";
+  public static final String NULLABLE_RECORD_SCHEMA = "[\"null\", " + RECORD_SCHEMA + "]";
   public static final String UNION_SCHEMA = "{\n" +
       "  \"namespace\": \"test.a.rossa\",\n" +
       "  \"name\": \"oneUnion\",\n" +

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java?rev=1671695&r1=1671694&r2=1671695&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/Utils.java Mon Apr  6 21:05:22
2015
@@ -24,6 +24,7 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 import java.rmi.server.UID;
 
+import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 
 class Utils {
@@ -31,10 +32,15 @@ class Utils {
   // chance to muck with the bytes and we're working against real Avro data.
   public static AvroGenericRecordWritable
   serializeAndDeserializeRecord(GenericData.Record record) throws IOException {
+    return serializeAndDeserializeRecord(record, record.getSchema());
+  }
+
+  public static AvroGenericRecordWritable
+  serializeAndDeserializeRecord(GenericData.Record record, Schema fileSchema) throws IOException
{
     AvroGenericRecordWritable garw = new AvroGenericRecordWritable(record);
     garw.setRecordReaderID(new UID());
     // Assuming file schema is the same as record schema for testing purpose.
-    garw.setFileSchema(record.getSchema());
+    garw.setFileSchema(fileSchema);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DataOutputStream daos = new DataOutputStream(baos);
     garw.write(daos);



Mime
View raw message