pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rd...@apache.org
Subject svn commit: r982704 - in /hadoop/pig/trunk: src/org/apache/pig/builtin/PigStorage.java test/org/apache/pig/test/TestBZip.java
Date Thu, 05 Aug 2010 17:33:12 GMT
Author: rding
Date: Thu Aug  5 17:33:11 2010
New Revision: 982704

URL: http://svn.apache.org/viewvc?rev=982704&view=rev
Log:
PIG-1533: Compression codec should be a per-store property

Modified:
    hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java

Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=982704&r1=982703&r2=982704&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java Thu Aug  5 17:33:11 2010
@@ -238,6 +238,8 @@ LoadPushDown {
         }  else if (location.endsWith(".gz")) {
             FileOutputFormat.setCompressOutput(job, true);
             FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
+        } else {
+            FileOutputFormat.setCompressOutput(job, false);
         }
     }
 

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=982704&r1=982703&r2=982704&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java Thu Aug  5 17:33:11 2010
@@ -17,6 +17,7 @@
  */
 package org.apache.pig.test;
 
+import static org.junit.Assert.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotSame;
 
@@ -353,4 +354,31 @@ public class TestBZip {
         assertEquals(expectedCount, result);
         
     }
+    
+    @Test
+    public void testBzipStoreInMultiQuery() throws Exception {
+        String[] inputData = new String[] {
+                "1\t2\r3\t4"
+        };
+        
+        String inputFileName = "input.txt";
+        Util.createInputFile(cluster, inputFileName, inputData);
+        
+        PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster
+                .getProperties());
+        
+        pig.setBatchOn();
+        pig.registerQuery("a = load '" +  inputFileName + "';");
+        pig.registerQuery("store a into 'output.bz2';");
+        pig.registerQuery("store a into 'output';");
+        pig.executeBatch();
+        
+        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
+                pig.getPigContext().getProperties()));
+        FileStatus stat = fs.getFileStatus(new Path("output/part-m-00000"));        
+        assertTrue(stat.getLen() > 0);     
+        
+        stat = fs.getFileStatus(new Path("output.bz2/part-m-00000.bz2"));
+        assertTrue(stat.getLen() > 0);     
+    }
 }



Mime
View raw message