pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1359495 - in /pig/trunk: CHANGES.txt test/org/apache/pig/newplan/logical/relational/LOTestHelper.java test/org/apache/pig/test/TestNewPlanOperatorPlan.java test/org/apache/pig/test/TestParser.java test/org/apache/pig/test/TestPigStorage.java
Date Tue, 10 Jul 2012 07:05:16 GMT
Author: daijy
Date: Tue Jul 10 07:05:16 2012
New Revision: 1359495

URL: http://svn.apache.org/viewvc?rev=1359495&view=rev
Log:
PIG-2807: TestParser TestPigStorage TestNewPlanOperatorPlan broken by PIG-2699

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java
    pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java
    pig/trunk/test/org/apache/pig/test/TestParser.java
    pig/trunk/test/org/apache/pig/test/TestPigStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1359495&r1=1359494&r2=1359495&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jul 10 07:05:16 2012
@@ -188,6 +188,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-2807: TestParser TestPigStorage TestNewPlanOperatorPlan broken by PIG-2699 (julienledem
via daijy)
+
 PIG-2782: Specifying sorting field(s) at nightly.conf (cheolsoo via daijy)
 
 PIG-2790: After Pig-2699 the script schema (LOAD ... USING ... AS {script schema}) is passed
after getSchema is called (daijy)

Modified: pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java?rev=1359495&r1=1359494&r2=1359495&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java (original)
+++ pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java Tue Jul 10
07:05:16 2012
@@ -7,6 +7,10 @@ import org.apache.pig.impl.io.FileSpec;
 
 public class LOTestHelper {
     public static LOLoad newLOLoad(FileSpec loader, LogicalSchema schema, LogicalPlan plan,
Configuration conf) {
-        return new LOLoad(loader, schema, plan, conf, (LoadFunc)PigContext.instantiateFuncFromSpec(loader.getFuncSpec()),
null);
+        LoadFunc loadFunc = null;
+        if (loader != null) {
+            loadFunc = (LoadFunc)PigContext.instantiateFuncFromSpec(loader.getFuncSpec());
+        }
+        return new LOLoad(loader, schema, plan, conf, loadFunc, null);
     }
 }

Modified: pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java?rev=1359495&r1=1359494&r2=1359495&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java Tue Jul 10 07:05:16 2012
@@ -64,6 +64,12 @@ import org.junit.Test;
 
 public class TestNewPlanOperatorPlan extends TestCase {
     
+    public static class FooLoad extends PigStorage {
+        public FooLoad(String[] params) {
+        }
+
+    }
+
     private static class SillyPlan extends BaseOperatorPlan {
         
         SillyPlan() {
@@ -1100,7 +1106,7 @@ public class TestNewPlanOperatorPlan ext
        jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad A1 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema1, lp, conf);
        lp.add(A1);
         
         // B = load
@@ -1136,7 +1142,7 @@ public class TestNewPlanOperatorPlan ext
         jaschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
         LOLoad A2 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema2, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema2, lp, conf);
         lp.add(A2);
         
         // B = load
@@ -1177,7 +1183,7 @@ public class TestNewPlanOperatorPlan ext
        jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad A1 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema1, lp, conf);
        lp.add(A1);
         
         // B = load
@@ -1215,7 +1221,7 @@ public class TestNewPlanOperatorPlan ext
         jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
         LOLoad A3 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema3, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema3, lp, conf);
         lp.add(A3);
         
         // B = load
@@ -1257,7 +1263,7 @@ public class TestNewPlanOperatorPlan ext
        jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad A1 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema1, lp, conf);
        lp.add(A1);
         
         // B = load
@@ -1293,7 +1299,7 @@ public class TestNewPlanOperatorPlan ext
         jaschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
         LOLoad A5 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema5, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema5, lp, conf);
         lp.add(A5);
         
         // B = load
@@ -1348,7 +1354,7 @@ public class TestNewPlanOperatorPlan ext
         jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
         LOLoad A6 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema6, lp, conf);
         lp.add(A6);
         
         // B = load
@@ -1389,7 +1395,7 @@ public class TestNewPlanOperatorPlan ext
         jaschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
         LOLoad A7 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema7, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema7, lp, conf);
         lp.add(A7);
         
         // B = load
@@ -1437,7 +1443,7 @@ public class TestNewPlanOperatorPlan ext
         jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
         LOLoad A6 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema6, lp, conf);
         lp.add(A6);
         
         // B = load
@@ -1478,7 +1484,7 @@ public class TestNewPlanOperatorPlan ext
         jaschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
         LOLoad A8 = newLOLoad(new FileSpec("/abc",
-           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema8, lp, conf);
+           new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[]
{"x", "y"})), jaschema8, lp, conf);
         lp.add(A8);
         
         // B = load

Modified: pig/trunk/test/org/apache/pig/test/TestParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParser.java?rev=1359495&r1=1359494&r2=1359495&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestParser.java Tue Jul 10 07:05:16 2012
@@ -33,6 +33,8 @@ import org.apache.pig.PigServer;
 import org.apache.pig.ExecType;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.builtin.mock.Storage;
+import org.apache.pig.builtin.mock.Storage.Data;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -134,41 +136,41 @@ protected final Log log = LogFactory.get
     
     @Test
     public void testRemoteServerList2() throws ExecException, IOException {
-        try {
-            Properties pigProperties = pigServer.getPigContext().getProperties();
-            pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020");
-            Configuration conf;
-            
-            pigServer.setBatchOn();
-            
-            pigServer.registerQuery("a = load '/user/pig/1.txt';");
-            pigServer.registerQuery("store a into '/user/pig/1.txt';");
-            
-            System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
-            conf = ConfigurationUtil.toConfiguration(pigProperties);
-            assertTrue(conf.get("mapreduce.job.hdfs-servers")==null||
-                    conf.get("mapreduce.job.hdfs-servers").equals("hdfs://a.com:8020"));
-                       
-            pigServer.registerQuery("store a into 'hdfs://b.com/user/pig/1.txt';");
-            System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
-            conf = ConfigurationUtil.toConfiguration(pigProperties);
-            assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null &&
-                    conf.get("mapreduce.job.hdfs-servers").contains("hdfs://b.com"));
-                        
-            pigServer.registerQuery("store a into 'har://hdfs-c.com:8020/user/pig/1.txt';");
-            System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
-            conf = ConfigurationUtil.toConfiguration(pigProperties);
-            assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null &&
-                    conf.get("mapreduce.job.hdfs-servers").contains("hdfs://c.com:8020"));
-                        
-            pigServer.registerQuery("store a into 'hdfs://d.com:8020/user/pig/1.txt';");
-            System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
-            conf = ConfigurationUtil.toConfiguration(pigProperties);
-            assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null &&
-                    conf.get("mapreduce.job.hdfs-servers").contains("hdfs://d.com:8020"));
 
-        } catch (IOException io) {
-            fail(io.getMessage());
-        }
+        Properties pigProperties = pigServer.getPigContext().getProperties();
+        pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020");
+        Configuration conf;
+
+        pigServer.setBatchOn();
+
+        Data data = Storage.resetData(pigServer.getPigContext());
+        data.set("/user/pig/1.txt");// no data
+        
+        pigServer.registerQuery("a = load '/user/pig/1.txt' using mock.Storage;");
+        pigServer.registerQuery("store a into '/user/pig/1.txt';");
+
+        System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
+        conf = ConfigurationUtil.toConfiguration(pigProperties);
+        assertTrue(conf.get("mapreduce.job.hdfs-servers")==null||
+                conf.get("mapreduce.job.hdfs-servers").equals("hdfs://a.com:8020"));
+
+        pigServer.registerQuery("store a into 'hdfs://b.com/user/pig/1.txt';");
+        System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
+        conf = ConfigurationUtil.toConfiguration(pigProperties);
+        assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null &&
+                conf.get("mapreduce.job.hdfs-servers").contains("hdfs://b.com"));
+
+        pigServer.registerQuery("store a into 'har://hdfs-c.com:8020/user/pig/1.txt';");
+        System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
+        conf = ConfigurationUtil.toConfiguration(pigProperties);
+        assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null &&
+                conf.get("mapreduce.job.hdfs-servers").contains("hdfs://c.com:8020"));
+
+        pigServer.registerQuery("store a into 'hdfs://d.com:8020/user/pig/1.txt';");
+        System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers"));
+        conf = ConfigurationUtil.toConfiguration(pigProperties);
+        assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null &&
+                conf.get("mapreduce.job.hdfs-servers").contains("hdfs://d.com:8020"));
+
     }
 }

Modified: pig/trunk/test/org/apache/pig/test/TestPigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStorage.java?rev=1359495&r1=1359494&r2=1359495&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStorage.java Tue Jul 10 07:05:16 2012
@@ -28,6 +28,7 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -108,7 +109,7 @@ public class TestPigStorage  {
     @Test
     public void testBlockBoundary() throws ExecException {
 
-        // This tests PigStorage loader with records exectly
+        // This tests PigStorage loader with records exactly
         // on the boundary of the file blocks.
         Properties props = new Properties();
         for (Entry<Object, Object> entry : cluster.getProperties().entrySet()) {
@@ -199,10 +200,11 @@ public class TestPigStorage  {
         pigContext.connect();
         String query = "a = LOAD '" + datadir + "originput' using PigStorage('\\t', '-schema')
" +
         "as (f1:chararray, f2:int);";
-        pig.registerQuery(query);
         try{
+            pig.registerQuery(query);
             pig.dumpSchema("a");
         }catch(FrontendException ex){
+            assertEquals(ex.toString(), 1000, ex.getErrorCode());
             return;
         }
         fail("no exception caught");



Mime
View raw message