pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r1369723 - in /pig/trunk: CHANGES.txt contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Date Mon, 06 Aug 2012 05:18:41 GMT
Author: sms
Date: Mon Aug  6 05:18:41 2012
New Revision: 1369723

URL: http://svn.apache.org/viewvc?rev=1369723&view=rev
Log:
PIG-2856: AvroStorage doesn't load files in the directories when a glob pattern matches both
files and directories. (cheolsoo via sms)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1369723&r1=1369722&r2=1369723&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Aug  6 05:18:41 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2856: AvroStorage doesn't load files in the directories when a glob pattern matches both
files and directories. (cheolsoo via sms)
+
 PIG-2569: Fix org.apache.pig.test.TestInvoker.testSpeed (aklochkov via dvryaboy)
 
 PIG-2858: Improve PlanHelper to allow finding any PhysicalOperator in a plan (dvryaboy)

Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java?rev=1369723&r1=1369722&r2=1369723&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
Mon Aug  6 05:18:41 2012
@@ -124,7 +124,7 @@ public class AvroStorageUtils {
         }
         for (FileStatus file : matchedFiles) {
             if (file.isDir()) {
-                for (FileStatus sub : fs.listStatus(path)) {
+                for (FileStatus sub : fs.listStatus(file.getPath())) {
                     getAllSubDirs(sub.getPath(), job, paths);
                 }
             } else {

Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1369723&r1=1369722&r2=1369723&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Mon Aug  6 05:18:41 2012
@@ -29,6 +29,8 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.executionengine.ExecJob;
+import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobCreationException;
 import org.apache.pig.piggybank.storage.avro.AvroStorage;
 import org.apache.pig.piggybank.storage.avro.PigSchema2Avro;
@@ -41,6 +43,7 @@ import org.junit.Test;
 import java.io.File;
 import java.io.IOException;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Properties;
 import java.util.Set;
 
@@ -112,7 +115,7 @@ public class TestAvroStorage {
     public void testGlob1() throws IOException {
         // Verify that the a glob pattern matches files properly.
         String output = outbasedir + "testGlob1";
-        String expected = basedir + "expected_test_dir_1.avro";
+        String expected = basedir + "expected_testDir.avro";
         deleteDirectory(new File(output));
         String [] queries = {
            " in = LOAD '" + testDir1AllFiles + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage
();",
@@ -492,11 +495,14 @@ public class TestAvroStorage {
             if (query != null && query.length() > 0)
                 pigServerLocal.registerQuery(query);
         }
-        pigServerLocal.executeBatch();
+        List<ExecJob> jobs = pigServerLocal.executeBatch();
+        for (ExecJob job : jobs) {
+            assertEquals(JOB_STATUS.COMPLETED, job.getStatus());
+        }
     }
     
     private void verifyResults(String outPath, String expectedOutpath) throws IOException
{
-      verifyResults(outPath, expectedOutpath, null);
+        verifyResults(outPath, expectedOutpath, null);
     }
 
     private void verifyResults(String outPath, String expectedOutpath, String expectedCodec)
throws IOException {



Mime
View raw message