pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject svn commit: r1419206 - in /pig/trunk: ./ contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/ contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/ contrib/piggybank/java/src/test/java/org/apache/pig/pi...
Date Mon, 10 Dec 2012 06:10:59 GMT
Author: cheolsoo
Date: Mon Dec 10 06:10:58 2012
New Revision: 1419206

URL: http://svn.apache.org/viewvc?rev=1419206&view=rev
Log:
PIG-3075: Allow AvroStorage STORE Operations To Use Schema Specified By URI (nwhite via cheolsoo)

Added:
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro
  (with props)
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1419206&r1=1419205&r2=1419206&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Dec 10 06:10:58 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-3075: Allow AvroStorage STORE Operations To Use Schema Specified By URI (nwhite via cheolsoo)
+
 PIG-3062: Change HBaseStorage to permit overriding pushProjection (billgraham)
 
 PIG-3016: Modernize more tests (jcoveney via cheolsoo)

Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java?rev=1419206&r1=1419205&r2=1419206&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
Mon Dec 10 06:10:58 2012
@@ -477,6 +477,7 @@ public class AvroStorage extends FileInp
                              || name.equalsIgnoreCase("same")
                              || name.equalsIgnoreCase("schema")
                              || name.equalsIgnoreCase("schema_file")
+                             || name.equalsIgnoreCase("schema_uri")
                              || name.matches("field\\d+")) {
                     /* store value as string */
                     map.put(name, value);
@@ -541,6 +542,12 @@ public class AvroStorage extends FileInp
                 nullable = (Boolean) value;
             } else if (name.equalsIgnoreCase("schema")) {
                 outputAvroSchema = Schema.parse((String) value);
+            } else if (name.equalsIgnoreCase("schema_uri")) {
+                /* use the contents of the specified path as output schema */
+                Path path = new Path( ((String) value).trim());
+                AvroStorageLog.details("schema_uri path=" + path.toUri().toString());
+                FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+                outputAvroSchema = getSchemaFromFile(path, fs);
             } else if (name.matches("field\\d+")) {
                 /*set schema of dth field */
                 if (fields == null)

Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1419206&r1=1419205&r2=1419206&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Mon Dec 10 06:10:58 2012
@@ -80,6 +80,7 @@ public class TestAvroStorage {
     final private String testDir21AllFiles = getInputFile("{test_dir2,test_dir1}/test_glob*.avro");
     final private String testNoMatchedFiles = getInputFile("test_dir{1,2}/file_that_does_not_exist*.avro");
     final private String testArrayFile = getInputFile("test_array.avro");
+    final private String testArraySchema = getInputFile("test_array.avsc");
     final private String testRecordFile = getInputFile("test_record.avro");
     final private String testRecordSchema = getInputFile("test_record.avsc");
     final private String testGenericUnionFile = getInputFile("test_generic_union.avro");
@@ -687,17 +688,31 @@ public class TestAvroStorage {
         verifyResults(output, expected);
     }
 
-
     @Test
     public void testArrayWithSchema() throws IOException {
         String output= outbasedir + "testArrayWithSchema";
         String expected = basedir + "expected_testArrayWithSchema.avro";
         deleteDirectory(new File(output));
         String [] queries = {
+                " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage
();",
+                " STORE in INTO '" + output +
+                "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( "  +
+                "   'schema', '{\"type\":\"array\",\"items\":\"float\"}'  );"
+        };
+        testAvroStorage( queries);
+        verifyResults(output, expected);
+    }
+
+    @Test
+    public void testArrayWithSchemaURI() throws IOException {
+        String output= outbasedir + "testArrayWithSchemaURI";
+        String expected = basedir + "expected_testArrayWithSchemaURI.avro"; // doubles (not
floats) stored
+        deleteDirectory(new File(output));
+        String [] queries = {
            " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage
();",
            " STORE in INTO '" + output +
                "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( "  +
-               "   'schema', '{\"type\":\"array\",\"items\":\"float\"}'  );"
+               "   'schema_uri', '" + testArraySchema  + "'  );"
             };
         testAvroStorage( queries);
         verifyResults(output, expected);

Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro?rev=1419206&view=auto
==============================================================================
Binary file - no diff available.

Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro
------------------------------------------------------------------------------
    svn:executable = *

Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc?rev=1419206&view=auto
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc
(added)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc
Mon Dec 10 06:10:58 2012
@@ -0,0 +1 @@
+{"type":"array","items":"double"}
\ No newline at end of file



Mime
View raw message