pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r1369740 - in /pig/trunk: ./ contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/ contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/ contrib/piggybank/java/src/test/java/org/apache/pig/pi...
Date Mon, 06 Aug 2012 07:44:07 GMT
Author: sms
Date: Mon Aug  6 07:44:06 2012
New Revision: 1369740

URL: http://svn.apache.org/viewvc?rev=1369740&view=rev
Log:
PIG-2837: AvroStorage throws StackOverFlowError (cheolsoo via sms)

Added:
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro
  (with props)
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro
  (with props)
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1369740&r1=1369739&r2=1369740&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Aug  6 07:44:06 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2837: AvroStorage throws StackOverFlowError (cheolsoo via sms)
+
 PIG-2856: AvroStorage doesn't load files in the directories when a glob pattern matches both
files and directories. (cheolsoo via sms)
 
 PIG-2569: Fix org.apache.pig.test.TestInvoker.testSpeed (aklochkov via dvryaboy)

Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java?rev=1369740&r1=1369739&r2=1369740&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java
Mon Aug  6 07:44:06 2012
@@ -47,12 +47,12 @@ public class AvroSchema2Pig {
      */
     public static ResourceSchema convert(Schema schema) throws IOException {
 
-        if (AvroStorageUtils.containsGenericUnion(schema))
-            throw new IOException ("We don't accept schema containing generic unions.");
-        
         if (AvroStorageUtils.containsRecursiveRecord(schema))
             throw new IOException ("We don't accept schema containing recursive records.");
         
+        if (AvroStorageUtils.containsGenericUnion(schema))
+            throw new IOException ("We don't accept schema containing generic unions.");
+
         ResourceFieldSchema inSchema = inconvert(schema, FIELD);
 
         ResourceSchema tupleSchema;

Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1369740&r1=1369739&r2=1369740&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Mon Aug  6 07:44:06 2012
@@ -32,6 +32,7 @@ import org.apache.pig.backend.executione
 import org.apache.pig.backend.executionengine.ExecJob;
 import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobCreationException;
+import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.piggybank.storage.avro.AvroStorage;
 import org.apache.pig.piggybank.storage.avro.PigSchema2Avro;
 import org.apache.pig.test.Util;
@@ -81,6 +82,8 @@ public class TestAvroStorage {
     final private String testArrayFile = getInputFile("test_array.avro");
     final private String testRecordFile = getInputFile("test_record.avro");
     final private String testRecordSchema = getInputFile("test_record.avsc");
+    final private String testRecursiveSchemaFile = getInputFile("test_recursive_schema.avro");
+    final private String testGenericUnionSchemaFile = getInputFile("test_generic_union_schema.avro");
     final private String testTextFile = getInputFile("test_record.txt");
     final private String testSingleTupleBagFile = getInputFile("messages.avro");
     final private String testNoExtensionFile = getInputFile("test_no_extension");
@@ -97,6 +100,48 @@ public class TestAvroStorage {
     }
 
     @Test
+    public void testRecursiveSchema() throws IOException {
+        // Verify that a FrontendException is thrown if schema is recursive.
+        String output= outbasedir + "testRecursiveSchema";
+        deleteDirectory(new File(output));
+        String [] queries = {
+          " in = LOAD '" + testRecursiveSchemaFile +
+              "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+          " STORE in INTO '" + output +
+              "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
+           };
+        try {
+            testAvroStorage(queries);
+            Assert.fail();
+        } catch (FrontendException e) {
+            // The IOException thrown by AvroStorage for recursive schema is caught
+            // by the Pig frontend, and FrontendException is re-thrown.
+            assertTrue(e.getMessage().contains("Cannot get schema"));
+        }
+    }
+
+    @Test
+    public void testGenericUnionSchema() throws IOException {
+        // Verify that a FrontendException is thrown if schema has generic union.
+        String output= outbasedir + "testGenericUnionSchema";
+        deleteDirectory(new File(output));
+        String [] queries = {
+          " in = LOAD '" + testGenericUnionSchemaFile +
+              "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+          " STORE in INTO '" + output +
+              "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
+           };
+        try {
+            testAvroStorage(queries);
+            Assert.fail();
+        } catch (FrontendException e) {
+            // The IOException thrown by AvroStorage for generic union is caught
+            // by the Pig frontend, and FrontendException is re-thrown.
+            assertTrue(e.getMessage().contains("Cannot get schema"));
+        }
+    }
+
+    @Test
     public void testDir() throws IOException {
         // Verify that all files in a directory including its sub-directories are loaded.
         String output= outbasedir + "testDir";

Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro?rev=1369740&view=auto
==============================================================================
Binary file - no diff available.

Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro?rev=1369740&view=auto
==============================================================================
Binary file - no diff available.

Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream



Mime
View raw message