pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jcove...@apache.org
Subject svn commit: r1394301 - in /pig/trunk: CHANGES.txt contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
Date Thu, 04 Oct 2012 22:26:31 GMT
Author: jcoveney
Date: Thu Oct  4 22:26:31 2012
New Revision: 1394301

URL: http://svn.apache.org/viewvc?rev=1394301&view=rev
Log:
PIG-2737: [piggybank] TestIndexedStorage is failing, should be refactored (jcoveney)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1394301&r1=1394300&r2=1394301&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Oct  4 22:26:31 2012
@@ -28,7 +28,7 @@ IMPROVEMENTS
 PIG-2923: Lazily register bags with SpillableMemoryManager (dvryaboy)
 
 PIG-2929: Improve documentation around AVG, CONCAT, MIN, MAX (cheolsoo via billgraham)
- 
+
 PIG-2852: Update documentation regarding parallel local mode execution (cheolsoo via jcoveney)
 
 PIG-2879: Pig current releases lack a UDF startsWith.This UDF tests if a given string starts
with the specified prefix. (initialcontext via azaroth)
@@ -285,6 +285,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-2737: [piggybank] TestIndexedStorage is failing, should be refactored (jcoveney)
+
 PIG-2935: Catch NoSuchMethodError when StoreFuncInterface's new cleanupOnSuccess method isn't
implemented. (gates via dvryaboy)
 
 PIG-2920: e2e tests override PERL5LIB environment variable (azaroth)

Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java?rev=1394301&r1=1394300&r2=1394301&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
(original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
Thu Oct  4 22:26:31 2012
@@ -18,6 +18,10 @@
 
 package org.apache.pig.piggybank.test.storage;
 
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.io.Files;
+
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -45,90 +49,85 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
 import org.apache.pig.piggybank.storage.IndexedStorage;
 import org.apache.pig.test.Util;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 /**
  * Tests IndexedStorage.
  */
 public class TestIndexedStorage {
+    private File outputDir;
+
     public TestIndexedStorage () throws IOException {
     }
 
-    @BeforeClass
+    @Before
     /**
      * Creates indexed data sets.
-     */ 
-    public static void setUpBeforeClass() throws Exception {
+     */
+    public void setUp() throws Exception {
         PigServer pigServer = new PigServer(ExecType.LOCAL);
-        
+
+        outputDir = Files.createTempDir();
+        outputDir.deleteOnExit();
+
         String[] input1 = new String[] {
             "2\t2", "3\t3", "4\t3", "5\t5", "6\t6", "10\t10"
         };
-        
-        createInputFile(pigServer, input1, 1);
-        
+
+        createInputFile(pigServer, input1, 1, outputDir);
+
         String[] input2 = new String[] {
             "3\t2", "4\t4", "5\t5", "11\t11", "13\t13"
         };
-        
-        createInputFile(pigServer, input2, 2);
+
+        createInputFile(pigServer, input2, 2, outputDir);
 
         String[] input3 = new String[] {
             "7\t7", "8\t8", "9\t9"
         };
-        
-        createInputFile(pigServer, input3, 3);
+
+        createInputFile(pigServer, input3, 3, outputDir);
     }
-    
-    private static void createInputFile(PigServer pigServer, String[] inputs, int id) throws
IOException {
+
+    private static void createInputFile(PigServer pigServer, String[] inputs, int id, File
outputDir) throws IOException {
         File input = File.createTempFile("tmp", "");
         input.delete();
         Util.createLocalInputFile(input.getAbsolutePath(), inputs);
-        
+
         pigServer.registerQuery("A = load '" + input.getAbsolutePath() + "' as (a0:int, a1:int);");
-        
-        File output = File.createTempFile("tmp", "");
-        output.delete();
+
+        File output = new File(outputDir, "/" + id);
         pigServer.store("A", output.getAbsolutePath(), "org.apache.pig.piggybank.storage.IndexedStorage('\t','0,1')");
-        
-        File outputFile = new File(output.getAbsoluteFile()+"/part-m-00000");
-        new File("out/"+id).mkdirs();
-        outputFile.renameTo(new File("out/"+id+"/part-m-00000"));
-        
-        File outputIndexFile = new File(output.getAbsoluteFile()+"/.part-m-00000.index");
-        outputIndexFile.renameTo(new File("out/"+id+"/.part-m-00000.index"));
     }
 
-    @AfterClass
+    @After
     /**
      * Deletes all data directories.
-     */ 
-    public static void tearDownAfterClass() throws Exception {
-        Configuration conf = new Configuration();
-        LocalFileSystem fs = FileSystem.getLocal(conf);
-        fs.delete(new Path("out"), true);
+     */
+    public void tearDown() throws Exception {
+        outputDir.delete();
     }
 
 
     @Test
     /**
      * Tests getNext called repeatedly returns items in sorted order.
-     */ 
+     */
     public void testGetNext() throws IOException, InterruptedException {
         IndexedStorage storage = new IndexedStorage("\t","0,1");
         Configuration conf = new Configuration();
         conf.set("fs.default.name", "file:///");
         LocalFileSystem fs = FileSystem.getLocal(conf);
-        
+
         TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 1, true, 1, 1);
         conf.set("mapred.task.id", taskId.toString());
-        
-        conf.set("mapred.input.dir","out");
+
+        conf.set("mapred.input.dir", outputDir.getAbsolutePath());
         storage.initialize(conf);
-       
-        Integer key; 
+
+        Integer key;
         int [][] correctValues = {{2,2},{3,2},{3,3},{4,3},{4,4},{5,5},{5,5},{6,6},{7,7},{8,8},{9,9},{10,10},{11,11},{13,13}};
         for (int [] outer : correctValues) {
             System.out.println("Testing: (" + outer[0] + "," + outer[1] + ")");
@@ -136,28 +135,28 @@ public class TestIndexedStorage {
             System.out.println("Read: " + read);
 
             key = Integer.decode(((DataByteArray)read.get(0)).toString());
-            Assert.assertTrue("GetNext did not return the correct value.  Received: " + read,
key.equals(outer[0]));
+            assertTrue("GetNext did not return the correct value.  Received: " + read, key.equals(outer[0]));
             key = Integer.decode(((DataByteArray)read.get(1)).toString());
-            Assert.assertTrue("GetNext did not return the correct value.  Received: " + read,
key.equals(outer[1]));
+            assertTrue("GetNext did not return the correct value.  Received: " + read, key.equals(outer[1]));
         }
         Tuple read = storage.getNext();
-        Assert.assertTrue("GetNext did not return the correct value", (read == null));
+        assertTrue("GetNext did not return the correct value", (read == null));
     }
 
     @Test
     /**
      * Tests seekNear and getNext work correctly.
-     */ 
+     */
     public void testSeek() throws IOException, InterruptedException {
         IndexedStorage storage = new IndexedStorage("\t","0,1");
         Configuration conf = new Configuration();
         conf.set("fs.default.name", "file:///");
         LocalFileSystem fs = FileSystem.getLocal(conf);
-        
+
         TaskAttemptID taskId =  HadoopShims.createTaskAttemptID("jt", 2, true, 2, 2);
         conf.set("mapred.task.id", taskId.toString());
-        
-        conf.set("mapred.input.dir","out");
+
+        conf.set("mapred.input.dir", outputDir.getAbsolutePath());
         storage.initialize(conf);
 
         TupleFactory tupleFactory = TupleFactory.getInstance();
@@ -166,82 +165,82 @@ public class TestIndexedStorage {
         Tuple read;
 
         //Seek to 1,1 (not in index). getNext should return 2
-        seek.set(0, new Integer(1)); 
-        seek.set(1, new Integer(1)); 
+        seek.set(0, new Integer(1));
+        seek.set(1, new Integer(1));
         storage.seekNear(seek);
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(2));
+        assertTrue("GetNext did not return the correct value", key.equals(2));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(2));
+        assertTrue("GetNext did not return the correct value", key.equals(2));
 
         //Seek to 3,2.  getNext should return 3,2
-        seek.set(0, new Integer(3)); 
-        seek.set(1, new Integer(2)); 
+        seek.set(0, new Integer(3));
+        seek.set(1, new Integer(2));
         storage.seekNear(seek);
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(3));
+        assertTrue("GetNext did not return the correct value", key.equals(3));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(2));
+        assertTrue("GetNext did not return the correct value", key.equals(2));
 
         //getNext should return 3,3
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(3));
+        assertTrue("GetNext did not return the correct value", key.equals(3));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(3));
+        assertTrue("GetNext did not return the correct value", key.equals(3));
 
         //Seek to 6,6.  getNext should return 6,6
-        seek.set(0, new Integer(6)); 
-        seek.set(1, new Integer(6)); 
+        seek.set(0, new Integer(6));
+        seek.set(1, new Integer(6));
         storage.seekNear(seek);
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(6));
+        assertTrue("GetNext did not return the correct value", key.equals(6));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(6));
+        assertTrue("GetNext did not return the correct value", key.equals(6));
 
         //getNext should return 7,7
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(7));
+        assertTrue("GetNext did not return the correct value", key.equals(7));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(7));
+        assertTrue("GetNext did not return the correct value", key.equals(7));
 
         //Seek to 9,9.  getNext should return 9,9
-        seek.set(0, new Integer(9)); 
-        seek.set(1, new Integer(9)); 
+        seek.set(0, new Integer(9));
+        seek.set(1, new Integer(9));
         storage.seekNear(seek);
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(9));
+        assertTrue("GetNext did not return the correct value", key.equals(9));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(9));
+        assertTrue("GetNext did not return the correct value", key.equals(9));
 
         //getNext should return 10,10
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(10));
+        assertTrue("GetNext did not return the correct value", key.equals(10));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(10));
+        assertTrue("GetNext did not return the correct value", key.equals(10));
 
 
         //Seek to 13,12 (Not in index).  getNext should return 13,13
-        seek.set(0, new Integer(13)); 
-        seek.set(1, new Integer(12)); 
+        seek.set(0, new Integer(13));
+        seek.set(1, new Integer(12));
         storage.seekNear(seek);
         read = storage.getNext();
         key = Integer.decode(((DataByteArray)read.get(0)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(13));
+        assertTrue("GetNext did not return the correct value", key.equals(13));
         key = Integer.decode(((DataByteArray)read.get(1)).toString());
-        Assert.assertTrue("GetNext did not return the correct value", key.equals(13));
+        assertTrue("GetNext did not return the correct value", key.equals(13));
 
-        //Seek to 20 (Not in index). getNext should return null 
-        seek.set(0, new Integer(20)); 
-        seek.set(1, new Integer(20)); 
+        //Seek to 20 (Not in index). getNext should return null
+        seek.set(0, new Integer(20));
+        seek.set(1, new Integer(20));
         storage.seekNear(seek);
         read = storage.getNext();
-        Assert.assertTrue("GetNext did not return the correct value", (read == null));
+        assertTrue("GetNext did not return the correct value", (read == null));
     }
 }



Mime
View raw message