hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject hive git commit: HIVE-10187: Avro backed tables don't handle cyclical or recursive records (Mark Wagner via cws)
Date Fri, 12 Feb 2016 20:57:07 GMT
Repository: hive
Updated Branches:
  refs/heads/master 28cf89266 -> 3bf2f679c


HIVE-10187: Avro backed tables don't handle cyclical or recursive records (Mark Wagner via
cws)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3bf2f679
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3bf2f679
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3bf2f679

Branch: refs/heads/master
Commit: 3bf2f679cc7b1da7d0c09041c9751c6bcd7e0d7e
Parents: 28cf892
Author: Carl Steinbach <csteinba@linkedin.com>
Authored: Fri Feb 12 12:55:55 2016 -0800
Committer: Carl Steinbach <csteinba@linkedin.com>
Committed: Fri Feb 12 12:55:55 2016 -0800

----------------------------------------------------------------------
 pom.xml                                         |  6 +++
 .../hive/serde2/avro/SchemaToTypeInfo.java      |  4 +-
 .../hive/serde2/avro/TestAvroSerializer.java    | 44 ----------------
 .../hive/serde2/avro/TestSchemaToTypeInfo.java  | 54 ++++++++++++++++++++
 4 files changed, 63 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 15e3522..af2e3d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -258,6 +258,12 @@
         <groupId>com.google.code.tempus-fugit</groupId>
         <artifactId>tempus-fugit</artifactId>
         <version>${tempus-fugit.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-core</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>com.googlecode.javaewah</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
index e60168c..35d83bd 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
@@ -217,7 +217,9 @@ class SchemaToTypeInfo {
     if (seenSchemas == null) {
         seenSchemas = Collections.newSetFromMap(new IdentityHashMap<Schema, Boolean>());
     } else if (seenSchemas.contains(schema)) {
-        return primitiveTypeToTypeInfo.get(Schema.Type.NULL);
+      throw new AvroSerdeException(
+          "Recursive schemas are not supported. Recursive schema was " + schema
+              .getFullName());
     }
     seenSchemas.add(schema);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
index 0b94dc5..45be2dd 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
@@ -491,48 +491,4 @@ public class TestAvroSerializer {
 
     assertArrayEquals(fixed.bytes(), ((GenericData.Fixed) r.get("fixed1")).bytes());
   }
-
-  @Test
-  public void canSerializeCyclesInSchema() throws SerDeException, IOException {
-    // Create parent-child avro-record and avro-schema
-    AvroCycleParent parent = new AvroCycleParent();
-    AvroCycleChild child = new AvroCycleChild();
-    parent.setChild (child);
-    Schema parentS = ReflectData.AllowNull.get().getSchema(AvroCycleParent.class);
-    GenericData.Record parentRec = new GenericData.Record(parentS);
-    Schema childS = ReflectData.AllowNull.get().getSchema(AvroCycleChild.class);
-    GenericData.Record childRec  = new GenericData.Record(childS);
-    parentRec.put("child", childRec);
-
-    // Initialize Avro SerDe
-    AvroSerializer as = new AvroSerializer();
-    AvroDeserializer ad = new AvroDeserializer();
-    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(parentS);
-    ObjectInspector oi = aoig.getObjectInspector();
-    List<String> columnNames = aoig.getColumnNames();
-    List<TypeInfo> columnTypes = aoig.getColumnTypes();
-
-    // Check serialization and deserialization
-    AvroGenericRecordWritable agrw = Utils.serializeAndDeserializeRecord(parentRec);
-    Object obj = ad.deserialize(columnNames, columnTypes, agrw, parentS);
-
-    Writable result = as.serialize(obj, oi, columnNames, columnTypes, parentS);
-    assertTrue(result instanceof AvroGenericRecordWritable);
-    GenericRecord r2 = ((AvroGenericRecordWritable) result).getRecord();
-    assertEquals(parentS, r2.getSchema());
-  }
-
-  private static class AvroCycleParent {
-    AvroCycleChild child;
-    public AvroCycleChild getChild () {return child;}
-    public void setChild (AvroCycleChild child) {this.child = child;}
-  }
-
-  private static class AvroCycleChild {
-    AvroCycleParent parent;
-    AvroCycleChild next;
-    Map <String, AvroCycleParent> map;
-    public AvroCycleParent getParent () {return parent;}
-    public void setParent (AvroCycleParent parent) {this.parent = parent;}
-  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java
new file mode 100644
index 0000000..af258c7
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.avro;
+
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+
+public class TestSchemaToTypeInfo {
+
+  @Rule
+  public ExpectedException expect = ExpectedException.none();
+
+  @Test
+  public void testDisallowRecursiveSchema()
+      throws AvroSerdeException {
+
+    expect.expect(AvroSerdeException.class);
+    expect.expectMessage("Recursive schemas are not supported");
+
+    final String schemaString = "{\n"
+        + "  \"type\" : \"record\",\n"
+        + "  \"name\" : \"Cycle\",\n"
+        + "  \"namespace\" : \"org.apache.hadoop.hive.serde2.avro\",\n"
+        + "  \"fields\" : [ {\n"
+        + "    \"name\" : \"child\",\n"
+        + "    \"type\" : [ \"null\", \"Cycle\"],\n"
+        + "    \"default\" : null\n"
+        + "  } ]\n"
+        + "}";
+
+    List<TypeInfo> types = SchemaToTypeInfo.generateColumnTypes(new Schema.Parser().parse(schemaString));
+  }
+}
\ No newline at end of file


Mime
View raw message