pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dvrya...@apache.org
Subject svn commit: r1222488 [1/2] - in /pig/trunk: ./ test/org/apache/pig/test/
Date Thu, 22 Dec 2011 22:14:07 GMT
Author: dvryaboy
Date: Thu Dec 22 22:14:06 2011
New Revision: 1222488

URL: http://svn.apache.org/viewvc?rev=1222488&view=rev
Log:
PIG-2448: Convert more tests to use LOCAL mode

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/test/org/apache/pig/test/TestBatchAliases.java
    pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java
    pig/trunk/test/org/apache/pig/test/TestPONegative.java
    pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
    pig/trunk/test/org/apache/pig/test/TestUTF8.java
    pig/trunk/test/org/apache/pig/test/TestUnion.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1222488&r1=1222487&r2=1222488&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Dec 22 22:14:06 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2448: Convert more tests to use LOCAL mode (dvryaboy)
+
 PIG-2438: Do not hardcode commons-lang version in build.xml (azaroth)
 
 PIG-2422: Add log messages for Jython schema definitions (vivekp via gates)

Modified: pig/trunk/test/org/apache/pig/test/TestBatchAliases.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBatchAliases.java?rev=1222488&r1=1222487&r2=1222488&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBatchAliases.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBatchAliases.java Thu Dec 22 22:14:06 2011
@@ -27,8 +27,6 @@ import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecJob;
 import org.apache.pig.impl.io.FileLocalizer;
-import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -36,41 +34,25 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class TestBatchAliases extends TestCase {
 
-    private static final MiniCluster cluster = MiniCluster.buildCluster();
-
     private PigServer myPig;
 
     @Override
     @Before
     public void setUp() throws Exception {
-        cluster.setProperty("opt.multiquery", ""+true);
-        myPig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+        System.setProperty("opt.multiquery", ""+true);
+        myPig = new PigServer(ExecType.LOCAL, System.getProperties());
         deleteOutputFiles();
     }
 
-    @Override
-    @After
-    public void tearDown() throws Exception {
-        myPig = null;
-        Util.deleteFile(cluster, "passwd");
-    }
-    
-    @AfterClass
-    public static void oneTimeTearDown() throws Exception {
-        cluster.shutDown();
-    }
-    
     @Test
     public void testBatchAliases() throws IOException {
 
         // test case: key ('group') isn't part of foreach output
         // and keys have the same type.
-        Util.copyFromLocalToCluster(cluster, 
-                "test/org/apache/pig/test/data/passwd", "passwd");
         try {
             myPig.setBatchOn();
 
-            myPig.registerQuery("a = load 'passwd' " +
+            myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " +
                                 "using PigStorage(':') as (uname:chararray, passwd:chararray,
uid:int, gid:int);");
             myPig.registerQuery("b = group a by uid;");
             myPig.registerQuery("c = group a by gid;");
@@ -89,11 +71,11 @@ public class TestBatchAliases extends Te
             }
             assertTrue(foundD);
             assertTrue(foundE);
-            
+
         } catch (Exception e) {
             e.printStackTrace();
             Assert.fail();
-        } 
+        }
     }
 
     private void deleteOutputFiles() {

Modified: pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java?rev=1222488&r1=1222487&r2=1222488&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java Thu Dec 22 22:14:06 2011
@@ -38,7 +38,6 @@ import org.apache.pig.data.SingleTupleBa
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.MultiMap;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -51,33 +50,26 @@ import junit.framework.TestCase;
  */
 @RunWith(JUnit4.class)
 public class TestDataBagAccess extends TestCase {
-
-    static MiniCluster cluster = MiniCluster.buildCluster();
     private PigServer pigServer;
 
     @Before
     @Override
     public void setUp() throws Exception{
-        pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
-    }
-    
-    @AfterClass
-    public static void oneTimeTearDown() throws Exception {
-        cluster.shutDown();
+        pigServer = new PigServer(ExecType.LOCAL, System.getProperties());
     }
-    
+
     @Test
     public void testSingleTupleBagAcess() throws Exception {
         Tuple inputTuple = new DefaultTuple();
         inputTuple.append("a");
         inputTuple.append("b");
-        
+
         SingleTupleBag bg = new SingleTupleBag(inputTuple);
         Iterator<Tuple> it = bg.iterator();
         assertEquals(inputTuple, it.next());
         assertFalse(it.hasNext());
     }
-    
+
     @Test
     public void testNonSpillableDataBag() throws Exception {
         String[][] tupleContents = new String[][] {{"a", "b"},{"c", "d" }, { "e", "f"} };
@@ -94,12 +86,12 @@ public class TestDataBagAccess extends T
         }
         assertEquals(tupleContents.length, j);
     }
-    
+
     @Test
     public void testBagConstantAccess() throws IOException, ExecException {
-        File input = Util.createInputFile("tmp", "", 
+        File input = Util.createInputFile("tmp", "",
                 new String[] {"sampledata\tnot_used"});
-        pigServer.registerQuery("a = load '" 
+        pigServer.registerQuery("a = load '"
                 + Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())
+ "';");
         pigServer.registerQuery("b = foreach a generate {(16, 4.0e-2, 'hello', -101)} as
mybag:{t:(i: int, d: double, c: chararray, e : int)};");
         pigServer.registerQuery("c = foreach b generate mybag.i, mybag.d, mybag.c, mybag.e;");
@@ -114,15 +106,15 @@ public class TestDataBagAccess extends T
             assertEquals(resultClasses[i], bag.iterator().next().get(0).getClass());
         }
     }
-    
+
     @Test
     public void testBagConstantAccessFailure() throws IOException, ExecException {
-        File input = Util.createInputFile("tmp", "", 
+        File input = Util.createInputFile("tmp", "",
                 new String[] {"sampledata\tnot_used"});
         boolean exceptionOccured = false;
         pigServer.setValidateEachStatement(true);
         try {
-            pigServer.registerQuery("a = load '" 
+            pigServer.registerQuery("a = load '"
                     + Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())
+ "';");
             pigServer.registerQuery("b = foreach a generate {(16, 4.0e-2, 'hello')} as mybag:{t:(i:
int, d: double, c: chararray)};");
             pigServer.registerQuery("c = foreach b generate mybag.t;");
@@ -134,12 +126,12 @@ public class TestDataBagAccess extends T
         }
         assertTrue(exceptionOccured);
     }
-    
+
     @Test
     public void testBagConstantFlatten1() throws IOException, ExecException {
-        File input = Util.createInputFile("tmp", "", 
+        File input = Util.createInputFile("tmp", "",
                 new String[] {"sampledata\tnot_used"});
-        pigServer.registerQuery("A = load '" 
+        pigServer.registerQuery("A = load '"
                 + Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())
+ "';");
         pigServer.registerQuery("B = foreach A generate {(('p1-t1-e1', 'p1-t1-e2'),('p1-t2-e1',
'p1-t2-e2'))," +
                 "(('p2-t1-e1', 'p2-t1-e2'), ('p2-t2-e1', 'p2-t2-e2'))};");
@@ -158,13 +150,13 @@ public class TestDataBagAccess extends T
         assertEquals("p2-t2-e1", (String)t.get(1));
         assertFalse(it.hasNext());
     }
-    
+
     @Test
     public void testBagConstantFlatten2() throws IOException, ExecException {
-        File input = Util.createInputFile("tmp", "", 
+        File input = Util.createInputFile("tmp", "",
                 new String[] {"somestring\t10\t{(a,10),(b,20)}"});
-        pigServer.registerQuery("a = load '" 
-                + Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())

+        pigServer.registerQuery("a = load '"
+                + Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())
                 + "' " + "as (str:chararray, intval:int, bg:bag{t:tuple(s:chararray, i:int)});");
         pigServer.registerQuery("b = foreach a generate str, intval, flatten(bg);");
         pigServer.registerQuery("c = foreach b generate str, intval, s, i;");
@@ -182,7 +174,7 @@ public class TestDataBagAccess extends T
             i++;
         }
         assertEquals(results.length, i);
-        
+
         pigServer.registerQuery("c = foreach b generate str, intval, bg::s, bg::i;");
         it = pigServer.openIterator("c");
         i = 0;
@@ -199,15 +191,15 @@ public class TestDataBagAccess extends T
 
     @Test
     public void testBagStoreLoad() throws IOException, ExecException {
-        File input = Util.createInputFile("tmp", "", 
+        File input = Util.createInputFile("tmp", "",
                 new String[] {"a\tid1", "a\tid2", "a\tid3", "b\tid4", "b\tid5", "b\tid6"});
-        pigServer.registerQuery("a = load '" 
-                + Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())

+        pigServer.registerQuery("a = load '"
+                + Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())
                 + "' " + "as (s:chararray, id:chararray);");
         pigServer.registerQuery("b = group a by s;");
         Class[] loadStoreClasses = new Class[] { BinStorage.class, PigStorage.class };
         for (int i = 0; i < loadStoreClasses.length; i++) {
-            String output = "/pig/out/TestDataBagAccess-testBagStoreLoad-" + 
+            String output = "/pig/out/TestDataBagAccess-testBagStoreLoad-" +
                              loadStoreClasses[i].getName() + ".txt";
             pigServer.deleteFile(output);
             pigServer.store("b", output, loadStoreClasses[i].getName());
@@ -256,7 +248,7 @@ public class TestDataBagAccess extends T
             // make sure we saw the right number of high
             // level tuples
             assertEquals(results.keySet().size(), j);
-            
+
             pigServer.registerQuery("d = foreach c generate gp, flatten(bg);");
             // results should be
             // a a id1
@@ -289,7 +281,7 @@ public class TestDataBagAccess extends T
             }
             // check we got expected number of tuples
             assertEquals(resultTuples.size(), j);
-            
+
             // same output as above - but projection based on aliases
             pigServer.registerQuery("e = foreach d generate gp, sReLoaded, idReLoaded;");
             it = pigServer.openIterator("e");
@@ -328,7 +320,7 @@ public class TestDataBagAccess extends T
             // check we got expected number of tuples
             assertEquals(resultTuples.size(), j);
 
-        
+
         }
     }
 }

Modified: pig/trunk/test/org/apache/pig/test/TestPONegative.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPONegative.java?rev=1222488&r1=1222487&r2=1222488&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPONegative.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPONegative.java Thu Dec 22 22:14:06 2011
@@ -34,7 +34,6 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONegative;
 import org.apache.pig.impl.plan.PlanException;
-import org.junit.AfterClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -42,19 +41,12 @@ import org.junit.runners.JUnit4;
 import junit.framework.TestCase;
 @RunWith(JUnit4.class)
 public class TestPONegative extends TestCase {
-    
+
     DataBag bag = BagFactory.getInstance().newDefaultBag();
     Random r = new Random();
     TupleFactory tf = TupleFactory.getInstance();
-    static MiniCluster miniCluster = MiniCluster.buildCluster();
     final int MAX = 10;
-    
-    @AfterClass
-    public static void oneTimeTearDown() throws Exception {
-        miniCluster.shutDown();
-    }
-    
-    
+
     @Test
     public void testPONegInt () throws PlanException, ExecException {
         for(int i = 0; i < MAX; i++) {
@@ -62,16 +54,16 @@ public class TestPONegative extends Test
             t.append(r.nextInt());
             bag.add(t);
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.INTEGER);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.INTEGER);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
@@ -80,8 +72,8 @@ public class TestPONegative extends Test
             assertEquals(expected.intValue(), output);
         }
     }
-    
-    @Test    
+
+    @Test
     public void testPONegIntAndNull () throws PlanException, ExecException {
         for(int i = 0; i < MAX; i++) {
             Tuple t = tf.newTuple();
@@ -93,20 +85,20 @@ public class TestPONegative extends Test
 	            bag.add(t);
             }
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.INTEGER);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.INTEGER);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
-            
+
             if(t.get(0) == null) {
                 Integer output = (Integer)pn.getNext((Integer)null).result;
                 assertEquals(null, output);
@@ -118,24 +110,24 @@ public class TestPONegative extends Test
             }
           }
     }
-    
-    @Test    
+
+    @Test
     public void testPONegLong () throws PlanException, ExecException {
         for(int i = 0; i < MAX; i++) {
             Tuple t = tf.newTuple();
             t.append(r.nextLong());
             bag.add(t);
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.LONG);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.LONG);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
@@ -144,8 +136,8 @@ public class TestPONegative extends Test
             assertEquals(expected.longValue(), output);
         }
     }
-    
-    @Test    
+
+    @Test
     public void testPONegLongAndNull () throws PlanException, ExecException {
         for(int i = 0; i < MAX; i++) {
             Tuple t = tf.newTuple();
@@ -157,20 +149,20 @@ public class TestPONegative extends Test
 	            bag.add(t);
             }
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.LONG);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.LONG);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
-            
+
             if(t.get(0) == null) {
                 Long output = (Long)pn.getNext((Long)null).result;
                 assertEquals(null, output);
@@ -182,7 +174,7 @@ public class TestPONegative extends Test
             }
         }
     }
-    
+
     @Test
     public void testPONegDouble() throws PlanException, ExecException {
         for(int i = 0; i < MAX; i++) {
@@ -190,16 +182,16 @@ public class TestPONegative extends Test
             t.append(r.nextDouble());
             bag.add(t);
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.DOUBLE);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.DOUBLE);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
@@ -208,7 +200,7 @@ public class TestPONegative extends Test
 			assertEquals(expected.doubleValue(), output);
         }
     }
-  
+
     @Test
     public void testPONegDoubleAndNull() throws PlanException, ExecException {
         for(int i = 0; i < MAX; i++) {
@@ -221,20 +213,20 @@ public class TestPONegative extends Test
 	            bag.add(t);
             }
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.DOUBLE);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.DOUBLE);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
-            
+
             if(t.get(0) == null) {
             	Double output = (Double )pn.getNext((Double )null).result;
                 assertEquals(null, output);
@@ -253,16 +245,16 @@ public class TestPONegative extends Test
             t.append(r.nextFloat());
             bag.add(t);
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.FLOAT);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.FLOAT);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
@@ -271,7 +263,7 @@ public class TestPONegative extends Test
             assertEquals(expected.floatValue(), output);
         }
     }
-    
+
     @Test
     public void testPONegFloatAndNull() throws PlanException, ExecException {
         for(int i = 0; i < MAX; i++) {
@@ -284,20 +276,20 @@ public class TestPONegative extends Test
 	            bag.add(t);
             }
         }
-        
+
         POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
         prj.setResultType(DataType.FLOAT);
         PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
         pn.setResultType(DataType.FLOAT);
-        
+
         PhysicalPlan plan = new PhysicalPlan();
         plan.add(prj); plan.add(pn);
         plan.connect(prj, pn);
-        
+
         for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
             Tuple t = it.next();
             plan.attachInput(t);
-            
+
             if(t.get(0) == null) {
             	Float output = (Float)pn.getNext((Float)null).result;
                 assertEquals(null, output);
@@ -308,12 +300,12 @@ public class TestPONegative extends Test
             }
         }
     }
-    
+
     @Test
     public void testPONegType() throws Exception {
-        PigServer pig = new PigServer(ExecType.MAPREDUCE, miniCluster.getProperties());
+        PigServer pig = new PigServer(ExecType.LOCAL, System.getProperties());
         File f = Util.createInputFile("tmp", "", new String[] {"a", "b", "c"});
-        pig.registerQuery("a = load '" 
+        pig.registerQuery("a = load '"
                 + Util.generateURI(f.toString(), pig.getPigContext()) + "';");
         // -1 is modeled as POnegative with Constant(1)
         pig.registerQuery("b = foreach a generate SIZE(-1);");



Mime
View raw message