pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject svn commit: r1582387 - in /pig/trunk: ./ src/org/apache/pig/builtin/ test/org/apache/pig/builtin/ test/org/apache/pig/builtin/avro/code/pig/ test/org/apache/pig/builtin/avro/data/json/ test/org/apache/pig/builtin/avro/schema/
Date Thu, 27 Mar 2014 16:34:42 GMT
Author: cheolsoo
Date: Thu Mar 27 16:34:41 2014
New Revision: 1582387

URL: http://svn.apache.org/r1582387
Log:
PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly
in foreach statement (jeongjinku via cheolsoo)

Added:
    pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
    pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
    pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/builtin/AvroStorage.java
    pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1582387&r1=1582386&r2=1582387&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 27 16:34:41 2014
@@ -103,6 +103,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly in foreach
statement (jeongjinku via cheolsoo)
+
 PIG-3836: Pig signature has has guava version dependency (amatsukawa via cheolsoo)
 
 PIG-3807: Pig creates wrong schema after dereferencing nested tuple fields with sorts (daijy)

Modified: pig/trunk/src/org/apache/pig/builtin/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/AvroStorage.java?rev=1582387&r1=1582386&r2=1582387&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/AvroStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/AvroStorage.java Thu Mar 27 16:34:41 2014
@@ -210,6 +210,7 @@ public class AvroStorage extends LoadFun
   public final void setUDFContextSignature(final String signature) {
     udfContextSignature = signature;
     super.setUDFContextSignature(signature);
+    updateSchemaFromInputAvroSchema();
   }
 
   /**
@@ -620,16 +621,24 @@ public class AvroStorage extends LoadFun
    */
   public final Schema getInputAvroSchema() {
     if (schema == null) {
-      String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
-      if (schemaString != null) {
-        Schema s = new Schema.Parser().parse(schemaString);
-        schema = s;
-      }
+      updateSchemaFromInputAvroSchema();
     }
     return schema;
   }
 
-  /*
+  /**
+   * Utility function that gets the input avro schema from the udf
+   * properties and updates schema for this instance.
+   */
+  private final void updateSchemaFromInputAvroSchema() {
+    String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
+    if (schemaString != null) {
+      Schema s = new Schema.Parser().parse(schemaString);
+      schema = s;
+    }
+  }
+
+  /**
    * @see org.apache.pig.LoadFunc#getInputFormat()
    */
   @Override

Modified: pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java?rev=1582387&r1=1582386&r2=1582387&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java Thu Mar 27 16:34:41 2014
@@ -105,6 +105,7 @@ public class TestAvroStorage {
         "recordWithRepeatedSubRecords",
         "recursiveRecord",
         "projectionTest",
+        "projectionTestWithSchema",
         "recordsWithSimpleUnion",
         "recordsWithSimpleUnionOutput",
     };
@@ -335,10 +336,9 @@ public class TestAvroStorage {
                  "AVROSTORAGE_OUT_1", "records",
                  "AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin -f " + basedir + "schema/recordsSubSchema.avsc",
                  "OUTFILE",           createOutputName())
-          );
-        verifyResults(createOutputName(),check);
-      }
-
+        );
+      verifyResults(createOutputName(),check);
+    }
 
     @Test public void testProjection() throws Exception {
         final String input = basedir + "data/avro/uncompressed/records.avro";
@@ -349,10 +349,23 @@ public class TestAvroStorage {
                 "AVROSTORAGE_OUT_1", "projectionTest",
                 "AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin",
                 "OUTFILE",          createOutputName())
-          );
-        verifyResults(createOutputName(),check);
-      }
+        );
+      verifyResults(createOutputName(),check);
+    }
 
+    @Test public void testProjectionWithSchema() throws Exception {
+        final String input = basedir + "data/avro/uncompressed/records.avro";
+        final String check = basedir + "data/avro/uncompressed/projectionTestWithSchema.avro";
+        testAvroStorage(true, basedir + "code/pig/projection_test_with_schema.pig",
+                ImmutableMap.of(
+                        "INFILE",           input,
+                        "AVROSTORAGE_IN_2",  "-f " + basedir + "schema/records.avsc",
+                        "AVROSTORAGE_OUT_1", "projectionTest",
+                        "AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin",
+                        "OUTFILE",          createOutputName())
+        );
+      verifyResults(createOutputName(),check);
+    }
 
     @Test public void testDates() throws Exception {
       final String input = basedir + "data/avro/uncompressed/records.avro";
@@ -768,7 +781,6 @@ public class TestAvroStorage {
         pigServerLocal.registerQuery("C = FOREACH B generate maps#'key1';");
         pigServerLocal.registerQuery("STORE C INTO 'out' USING mock.Storage();");
 
-
         List<Tuple> out = data.get("out");
         assertEquals(tuple("v11"), out.get(0));
         assertEquals(tuple("v21"), out.get(1));
@@ -891,7 +903,7 @@ public class TestAvroStorage {
             assertEquals(expected.size(), count);
           }
         }
-      }
+    }
 
     private Set<GenericData.Record> getExpected (String pathstr ) throws IOException
{
 
@@ -929,7 +941,7 @@ public class TestAvroStorage {
             }
         }
         return ret;
-  }
+    }
 
 }
 

Added: pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig?rev=1582387&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig Thu
Mar 27 16:34:41 2014
@@ -0,0 +1,4 @@
+in = LOAD '$INFILE' USING AvroStorage('','$AVROSTORAGE_IN_2');
+out = FOREACH in GENERATE $0, $1, $3;
+RMF $OUTFILE;
+STORE out INTO '$OUTFILE' USING AvroStorage('$AVROSTORAGE_OUT_1','$AVROSTORAGE_OUT_2');
\ No newline at end of file

Added: pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json?rev=1582387&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json Thu
Mar 27 16:34:41 2014
@@ -0,0 +1,15 @@
+{
+ "key" : "A",
+ "intValue" : 1,
+ "booleanValue" : true
+}
+{
+ "key" : "B",
+ "intValue" : 2,
+ "booleanValue" : true
+}
+{
+ "key" : "C",
+ "intValue" : 3,
+ "booleanValue" : false
+}
\ No newline at end of file

Added: pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc?rev=1582387&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc Thu Mar
27 16:34:41 2014
@@ -0,0 +1,10 @@
+{
+  "name" : "projectionTestWithSchema",
+  "namespace" : "org.apache.pig.test.builtin",
+  "type" : "record",
+  "fields" : [
+    {"name" : "key", "type" : "string"},
+    {"name" : "intValue", "type" : "int"},
+    {"name" : "booleanValue", "type" : "boolean"}
+  ]
+}
\ No newline at end of file



Mime
View raw message