pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject svn commit: r1582793 - in /pig/branches/branch-0.12: ./ src/org/apache/pig/builtin/ test/org/apache/pig/builtin/ test/org/apache/pig/builtin/avro/code/pig/ test/org/apache/pig/builtin/avro/data/json/ test/org/apache/pig/builtin/avro/schema/
Date Fri, 28 Mar 2014 16:13:52 GMT
Author: cheolsoo
Date: Fri Mar 28 16:13:51 2014
New Revision: 1582793

URL: http://svn.apache.org/r1582793
Log:
PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly
in foreach statement (jeongjinku via cheolsoo)

Added:
    pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
    pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
    pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
Modified:
    pig/branches/branch-0.12/CHANGES.txt
    pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java
    pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java

Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1582793&r1=1582792&r2=1582793&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Fri Mar 28 16:13:51 2014
@@ -34,6 +34,8 @@ PIG-3480: TFile-based tmpfile compressio
 
 BUG FIXES
 
+PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly in foreach
statement (jeongjinku via cheolsoo)
+
 PIG-3794: pig -useHCatalog fails using pig command line interface on HDInsight (ehans via
daijy)
 
 PIG-3827: Custom partitioner is not picked up with secondary sort optimization (daijy)

Modified: pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java?rev=1582793&r1=1582792&r2=1582793&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java Fri Mar 28 16:13:51
2014
@@ -210,6 +210,7 @@ public class AvroStorage extends LoadFun
   public final void setUDFContextSignature(final String signature) {
     udfContextSignature = signature;
     super.setUDFContextSignature(signature);
+    updateSchemaFromInputAvroSchema();
   }
 
   /**
@@ -620,16 +621,24 @@ public class AvroStorage extends LoadFun
    */
   public final Schema getInputAvroSchema() {
     if (schema == null) {
-      String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
-      if (schemaString != null) {
-        Schema s = new Schema.Parser().parse(schemaString);
-        schema = s;
-      }
+      updateSchemaFromInputAvroSchema();
     }
     return schema;
   }
 
-  /*
+  /**
+   * Utility function that gets the input avro schema from the udf
+   * properties and updates schema for this instance.
+   */
+  private final void updateSchemaFromInputAvroSchema() {
+    String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
+    if (schemaString != null) {
+      Schema s = new Schema.Parser().parse(schemaString);
+      schema = s;
+    }
+  }
+
+  /**
    * @see org.apache.pig.LoadFunc#getInputFormat()
    */
   @Override

Modified: pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java?rev=1582793&r1=1582792&r2=1582793&view=diff
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java (original)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java Fri Mar 28 16:13:51
2014
@@ -100,6 +100,7 @@ public class TestAvroStorage {
         "recordWithRepeatedSubRecords",
         "recursiveRecord",
         "projectionTest",
+        "projectionTestWithSchema",
         "recordsWithSimpleUnion",
         "recordsWithSimpleUnionOutput",
     };
@@ -332,9 +333,8 @@ public class TestAvroStorage {
                  "OUTFILE",           createOutputName())
           );
         verifyResults(createOutputName(),check);
-      }
-
-    
+    }
+ 
     @Test public void testProjection() throws Exception {
         final String input = basedir + "data/avro/uncompressed/records.avro";
         final String check = basedir + "data/avro/uncompressed/projectionTest.avro";
@@ -346,9 +346,22 @@ public class TestAvroStorage {
                 "OUTFILE",          createOutputName())
           );
         verifyResults(createOutputName(),check);
-      }
+    }
+
+    @Test public void testProjectionWithSchema() throws Exception {
+        final String input = basedir + "data/avro/uncompressed/records.avro";
+        final String check = basedir + "data/avro/uncompressed/projectionTestWithSchema.avro";
+        testAvroStorage(true, basedir + "code/pig/projection_test_with_schema.pig",
+                ImmutableMap.of(
+                        "INFILE",           input,
+                        "AVROSTORAGE_IN_2",  "-f " + basedir + "schema/records.avsc",
+                        "AVROSTORAGE_OUT_1", "projectionTest",
+                        "AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin",
+                        "OUTFILE",          createOutputName())
+          );
+        verifyResults(createOutputName(),check);
+    }
 
-    
     @Test public void testDates() throws Exception {
       final String input = basedir + "data/avro/uncompressed/records.avro";
       final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPigWithDates.avro";
@@ -762,7 +775,6 @@ public class TestAvroStorage {
         pigServerLocal.registerQuery("B = LOAD '" + createOutputName() + "' USING AvroStorage();");
         pigServerLocal.registerQuery("C = FOREACH B generate maps#'key1';");
         pigServerLocal.registerQuery("STORE C INTO 'out' USING mock.Storage();");
-        
 
         List<Tuple> out = data.get("out");
         assertEquals(tuple("v11"), out.get(0));
@@ -836,7 +848,7 @@ public class TestAvroStorage {
             assertEquals(expected.size(), count);
           }
         }
-      }
+    }
 
     private Set<GenericData.Record> getExpected (String pathstr ) throws IOException
{
 
@@ -874,7 +886,7 @@ public class TestAvroStorage {
             }
         }
         return ret;
-  }
+    }
 
 }
 

Added: pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig?rev=1582793&view=auto
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
(added)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
Fri Mar 28 16:13:51 2014
@@ -0,0 +1,4 @@
+in = LOAD '$INFILE' USING AvroStorage('','$AVROSTORAGE_IN_2');
+out = FOREACH in GENERATE $0, $1, $3;
+RMF $OUTFILE;
+STORE out INTO '$OUTFILE' USING AvroStorage('$AVROSTORAGE_OUT_1','$AVROSTORAGE_OUT_2');
\ No newline at end of file

Added: pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json?rev=1582793&view=auto
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
(added)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
Fri Mar 28 16:13:51 2014
@@ -0,0 +1,15 @@
+{
+ "key" : "A",
+ "intValue" : 1,
+ "booleanValue" : true
+}
+{
+ "key" : "B",
+ "intValue" : 2,
+ "booleanValue" : true
+}
+{
+ "key" : "C",
+ "intValue" : 3,
+ "booleanValue" : false
+}
\ No newline at end of file

Added: pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc?rev=1582793&view=auto
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
(added)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
Fri Mar 28 16:13:51 2014
@@ -0,0 +1,10 @@
+{
+  "name" : "projectionTestWithSchema",
+  "namespace" : "org.apache.pig.test.builtin",
+  "type" : "record",
+  "fields" : [
+    {"name" : "key", "type" : "string"},
+    {"name" : "intValue", "type" : "int"},
+    {"name" : "booleanValue", "type" : "boolean"}
+  ]
+}
\ No newline at end of file



Mime
View raw message