pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1358051 - in /pig/trunk: CHANGES.txt src/org/apache/pig/builtin/PigStorage.java test/org/apache/pig/test/TestPigStorage.java
Date Fri, 06 Jul 2012 06:32:29 GMT
Author: daijy
Date: Fri Jul  6 06:32:28 2012
New Revision: 1358051

URL: http://svn.apache.org/viewvc?rev=1358051&view=rev
Log:
Revert PIG-2746 due to TestBZip.testBz2 failure

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/builtin/PigStorage.java
    pig/trunk/test/org/apache/pig/test/TestPigStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1358051&r1=1358050&r2=1358051&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Jul  6 06:32:28 2012
@@ -32,8 +32,6 @@ PIG-2750: add artifacts to the ivy.xml f
 
 PIG-2748: Change the names of the jar produced in the build folder to match maven conventions
(julien)
 
-PIG-2746: Pig doesn't detect all forms of compression extensions properly (qwertymaniac via
daijy)
-
 PIG-2770: Allow easy inclusion of custom build targets (julien)
 
 PIG-2697: pretty print schema via pig.pretty.print.schema (rangadi via jcoveney)

Modified: pig/trunk/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=1358051&r1=1358050&r2=1358051&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/PigStorage.java Fri Jul  6 06:32:28 2012
@@ -33,8 +33,9 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
 import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.OutputFormat;
@@ -403,13 +404,15 @@ LoadPushDown, LoadMetadata, StoreMetadat
     }
 
     private void setCompression(Path path, Job job) {
-        CompressionCodecFactory factory = new CompressionCodecFactory(job.getConfiguration());
-        CompressionCodec codec = factory.getCodec(path);
-        if (codec == null) {
-            FileOutputFormat.setCompressOutput(job, false);
-        } else {
+     	String location=path.getName();
+        if (location.endsWith(".bz2") || location.endsWith(".bz")) {
+            FileOutputFormat.setCompressOutput(job, true);
+            FileOutputFormat.setOutputCompressorClass(job,  BZip2Codec.class);
+        }  else if (location.endsWith(".gz")) {
             FileOutputFormat.setCompressOutput(job, true);
-            FileOutputFormat.setOutputCompressorClass(job, codec.getClass());
+            FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
+        } else {
+            FileOutputFormat.setCompressOutput( job, false);
         }
     }
 

Modified: pig/trunk/test/org/apache/pig/test/TestPigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStorage.java?rev=1358051&r1=1358050&r2=1358051&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStorage.java Fri Jul  6 06:32:28 2012
@@ -529,28 +529,5 @@ public class TestPigStorage  {
             String inputFileName = (String)tuple.get(0);
             assertEquals("tagsource value must be part-m-00000", inputFileName, storeFileName);
         }
-    }
-
-    @Test
-    public void testPigStorageCompression() throws Exception {
-        final String storeFileName = "part-m-00000.bz2";
-        pigContext.connect();
-
-        String query = "a = LOAD '" + datadir + "' using PigStorage('\\t') " +
-        "as (f1:chararray, f2:int);";
-        pig.registerQuery(query);
-        // Storing in 'aout.bz2' directory will store contents in part-m-00000.bz2
-        // (With BZ2 compression applied automatically)
-        pig.store("a", datadir + "aout.bz2", "PigStorage('\\t', '-schema')");
-
-        // Verify input source tag is present when using -tagsource
-        pig.registerQuery("b = LOAD '" + datadir + "aout.bz2' using PigStorage('\\t', '-tagsource');");
-        pig.registerQuery("c = foreach b generate INPUT_FILE_NAME;");
-        Iterator<Tuple> iter = pig.openIterator("c");
-        while(iter.hasNext()) {
-            Tuple tuple = iter.next();
-            String inputFileName = (String)tuple.get(0);
-            assertEquals("tagsource value must be part-m-00000.bz2", inputFileName, storeFileName);
-        }
-    }
+    }    
 }



Mime
View raw message