Author: cheolsoo
Date: Thu Mar 27 16:34:41 2014
New Revision: 1582387
URL: http://svn.apache.org/r1582387
Log:
PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly
in foreach statement (jeongjinku via cheolsoo)
Added:
pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/builtin/AvroStorage.java
pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1582387&r1=1582386&r2=1582387&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 27 16:34:41 2014
@@ -103,6 +103,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly in foreach
statement (jeongjinku via cheolsoo)
+
PIG-3836: Pig signature has has guava version dependency (amatsukawa via cheolsoo)
PIG-3807: Pig creates wrong schema after dereferencing nested tuple fields with sorts (daijy)
Modified: pig/trunk/src/org/apache/pig/builtin/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/AvroStorage.java?rev=1582387&r1=1582386&r2=1582387&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/AvroStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/AvroStorage.java Thu Mar 27 16:34:41 2014
@@ -210,6 +210,7 @@ public class AvroStorage extends LoadFun
public final void setUDFContextSignature(final String signature) {
udfContextSignature = signature;
super.setUDFContextSignature(signature);
+ updateSchemaFromInputAvroSchema();
}
/**
@@ -620,16 +621,24 @@ public class AvroStorage extends LoadFun
*/
public final Schema getInputAvroSchema() {
if (schema == null) {
- String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
- if (schemaString != null) {
- Schema s = new Schema.Parser().parse(schemaString);
- schema = s;
- }
+ updateSchemaFromInputAvroSchema();
}
return schema;
}
- /*
+ /**
+ * Utility function that gets the input avro schema from the udf
+ * properties and updates schema for this instance.
+ */
+ private final void updateSchemaFromInputAvroSchema() {
+ String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
+ if (schemaString != null) {
+ Schema s = new Schema.Parser().parse(schemaString);
+ schema = s;
+ }
+ }
+
+ /**
* @see org.apache.pig.LoadFunc#getInputFormat()
*/
@Override
Modified: pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java?rev=1582387&r1=1582386&r2=1582387&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java Thu Mar 27 16:34:41 2014
@@ -105,6 +105,7 @@ public class TestAvroStorage {
"recordWithRepeatedSubRecords",
"recursiveRecord",
"projectionTest",
+ "projectionTestWithSchema",
"recordsWithSimpleUnion",
"recordsWithSimpleUnionOutput",
};
@@ -335,10 +336,9 @@ public class TestAvroStorage {
"AVROSTORAGE_OUT_1", "records",
"AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin -f " + basedir + "schema/recordsSubSchema.avsc",
"OUTFILE", createOutputName())
- );
- verifyResults(createOutputName(),check);
- }
-
+ );
+ verifyResults(createOutputName(),check);
+ }
@Test public void testProjection() throws Exception {
final String input = basedir + "data/avro/uncompressed/records.avro";
@@ -349,10 +349,23 @@ public class TestAvroStorage {
"AVROSTORAGE_OUT_1", "projectionTest",
"AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin",
"OUTFILE", createOutputName())
- );
- verifyResults(createOutputName(),check);
- }
+ );
+ verifyResults(createOutputName(),check);
+ }
+ @Test public void testProjectionWithSchema() throws Exception {
+ final String input = basedir + "data/avro/uncompressed/records.avro";
+ final String check = basedir + "data/avro/uncompressed/projectionTestWithSchema.avro";
+ testAvroStorage(true, basedir + "code/pig/projection_test_with_schema.pig",
+ ImmutableMap.of(
+ "INFILE", input,
+ "AVROSTORAGE_IN_2", "-f " + basedir + "schema/records.avsc",
+ "AVROSTORAGE_OUT_1", "projectionTest",
+ "AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin",
+ "OUTFILE", createOutputName())
+ );
+ verifyResults(createOutputName(),check);
+ }
@Test public void testDates() throws Exception {
final String input = basedir + "data/avro/uncompressed/records.avro";
@@ -768,7 +781,6 @@ public class TestAvroStorage {
pigServerLocal.registerQuery("C = FOREACH B generate maps#'key1';");
pigServerLocal.registerQuery("STORE C INTO 'out' USING mock.Storage();");
-
List<Tuple> out = data.get("out");
assertEquals(tuple("v11"), out.get(0));
assertEquals(tuple("v21"), out.get(1));
@@ -891,7 +903,7 @@ public class TestAvroStorage {
assertEquals(expected.size(), count);
}
}
- }
+ }
private Set<GenericData.Record> getExpected (String pathstr ) throws IOException
{
@@ -929,7 +941,7 @@ public class TestAvroStorage {
}
}
return ret;
- }
+ }
}
Added: pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig?rev=1582387&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig Thu
Mar 27 16:34:41 2014
@@ -0,0 +1,4 @@
+in = LOAD '$INFILE' USING AvroStorage('','$AVROSTORAGE_IN_2');
+out = FOREACH in GENERATE $0, $1, $3;
+RMF $OUTFILE;
+STORE out INTO '$OUTFILE' USING AvroStorage('$AVROSTORAGE_OUT_1','$AVROSTORAGE_OUT_2');
\ No newline at end of file
Added: pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json?rev=1582387&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json Thu
Mar 27 16:34:41 2014
@@ -0,0 +1,15 @@
+{
+ "key" : "A",
+ "intValue" : 1,
+ "booleanValue" : true
+}
+{
+ "key" : "B",
+ "intValue" : 2,
+ "booleanValue" : true
+}
+{
+ "key" : "C",
+ "intValue" : 3,
+ "booleanValue" : false
+}
\ No newline at end of file
Added: pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc?rev=1582387&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc Thu Mar
27 16:34:41 2014
@@ -0,0 +1,10 @@
+{
+ "name" : "projectionTestWithSchema",
+ "namespace" : "org.apache.pig.test.builtin",
+ "type" : "record",
+ "fields" : [
+ {"name" : "key", "type" : "string"},
+ {"name" : "intValue", "type" : "int"},
+ {"name" : "booleanValue", "type" : "boolean"}
+ ]
+}
\ No newline at end of file
|