Return-Path: X-Original-To: apmail-pig-commits-archive@www.apache.org Delivered-To: apmail-pig-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7D0A09651 for ; Tue, 10 Jul 2012 07:05:44 +0000 (UTC) Received: (qmail 61859 invoked by uid 500); 10 Jul 2012 07:05:43 -0000 Delivered-To: apmail-pig-commits-archive@pig.apache.org Received: (qmail 61766 invoked by uid 500); 10 Jul 2012 07:05:43 -0000 Mailing-List: contact commits-help@pig.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@pig.apache.org Delivered-To: mailing list commits@pig.apache.org Received: (qmail 61738 invoked by uid 99); 10 Jul 2012 07:05:42 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Jul 2012 07:05:42 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Jul 2012 07:05:38 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 4602B238889B for ; Tue, 10 Jul 2012 07:05:17 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1359495 - in /pig/trunk: CHANGES.txt test/org/apache/pig/newplan/logical/relational/LOTestHelper.java test/org/apache/pig/test/TestNewPlanOperatorPlan.java test/org/apache/pig/test/TestParser.java test/org/apache/pig/test/TestPigStorage.java Date: Tue, 10 Jul 2012 07:05:16 -0000 To: commits@pig.apache.org From: daijy@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120710070517.4602B238889B@eris.apache.org> Author: daijy Date: Tue Jul 10 07:05:16 2012 New Revision: 1359495 URL: http://svn.apache.org/viewvc?rev=1359495&view=rev Log: PIG-2807: TestParser TestPigStorage TestNewPlanOperatorPlan broken by PIG-2699 Modified: pig/trunk/CHANGES.txt pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java pig/trunk/test/org/apache/pig/test/TestParser.java pig/trunk/test/org/apache/pig/test/TestPigStorage.java Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1359495&r1=1359494&r2=1359495&view=diff ============================================================================== --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Tue Jul 10 07:05:16 2012 @@ -188,6 +188,8 @@ OPTIMIZATIONS BUG FIXES +PIG-2807: TestParser TestPigStorage TestNewPlanOperatorPlan broken by PIG-2699 (julienledem via daijy) + PIG-2782: Specifying sorting field(s) at nightly.conf (cheolsoo via daijy) PIG-2790: After Pig-2699 the script schema (LOAD ... USING ... AS {script schema}) is passed after getSchema is called (daijy) Modified: pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java?rev=1359495&r1=1359494&r2=1359495&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java (original) +++ pig/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java Tue Jul 10 07:05:16 2012 @@ -7,6 +7,10 @@ import org.apache.pig.impl.io.FileSpec; public class LOTestHelper { public static LOLoad newLOLoad(FileSpec loader, LogicalSchema schema, LogicalPlan plan, Configuration conf) { - return new LOLoad(loader, schema, plan, conf, (LoadFunc)PigContext.instantiateFuncFromSpec(loader.getFuncSpec()), null); + LoadFunc loadFunc = null; + if (loader != null) { + loadFunc = (LoadFunc)PigContext.instantiateFuncFromSpec(loader.getFuncSpec()); + } + return new LOLoad(loader, schema, plan, conf, loadFunc, null); } } Modified: pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java?rev=1359495&r1=1359494&r2=1359495&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java (original) +++ pig/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java Tue Jul 10 07:05:16 2012 @@ -64,6 +64,12 @@ import org.junit.Test; public class TestNewPlanOperatorPlan extends TestCase { + public static class FooLoad extends PigStorage { + public FooLoad(String[] params) { + } + + } + private static class SillyPlan extends BaseOperatorPlan { SillyPlan() { @@ -1100,7 +1106,7 @@ public class TestNewPlanOperatorPlan ext jaschema1.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A1 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema1, lp, conf); lp.add(A1); // B = load @@ -1136,7 +1142,7 @@ public class TestNewPlanOperatorPlan ext jaschema2.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A2 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema2, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema2, lp, conf); lp.add(A2); // B = load @@ -1177,7 +1183,7 @@ public class TestNewPlanOperatorPlan ext jaschema1.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A1 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema1, lp, conf); lp.add(A1); // B = load @@ -1215,7 +1221,7 @@ public class TestNewPlanOperatorPlan ext jaschema3.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A3 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema3, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema3, lp, conf); lp.add(A3); // B = load @@ -1257,7 +1263,7 @@ public class TestNewPlanOperatorPlan ext jaschema1.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A1 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema1, lp, conf); lp.add(A1); // B = load @@ -1293,7 +1299,7 @@ public class TestNewPlanOperatorPlan ext jaschema5.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A5 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema5, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema5, lp, conf); lp.add(A5); // B = load @@ -1348,7 +1354,7 @@ public class TestNewPlanOperatorPlan ext jaschema6.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A6 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema6, lp, conf); lp.add(A6); // B = load @@ -1389,7 +1395,7 @@ public class TestNewPlanOperatorPlan ext jaschema7.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A7 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema7, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema7, lp, conf); lp.add(A7); // B = load @@ -1437,7 +1443,7 @@ public class TestNewPlanOperatorPlan ext jaschema6.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A6 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema6, lp, conf); lp.add(A6); // B = load @@ -1478,7 +1484,7 @@ public class TestNewPlanOperatorPlan ext jaschema8.addField(new LogicalSchema.LogicalFieldSchema( "x", null, DataType.INTEGER)); LOLoad A8 = newLOLoad(new FileSpec("/abc", - new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema8, lp, conf); + new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {"x", "y"})), jaschema8, lp, conf); lp.add(A8); // B = load Modified: pig/trunk/test/org/apache/pig/test/TestParser.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParser.java?rev=1359495&r1=1359494&r2=1359495&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestParser.java (original) +++ pig/trunk/test/org/apache/pig/test/TestParser.java Tue Jul 10 07:05:16 2012 @@ -33,6 +33,8 @@ import org.apache.pig.PigServer; import org.apache.pig.ExecType; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; +import org.apache.pig.builtin.mock.Storage; +import org.apache.pig.builtin.mock.Storage.Data; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -134,41 +136,41 @@ protected final Log log = LogFactory.get @Test public void testRemoteServerList2() throws ExecException, IOException { - try { - Properties pigProperties = pigServer.getPigContext().getProperties(); - pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020"); - Configuration conf; - - pigServer.setBatchOn(); - - pigServer.registerQuery("a = load '/user/pig/1.txt';"); - pigServer.registerQuery("store a into '/user/pig/1.txt';"); - - System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); - conf = ConfigurationUtil.toConfiguration(pigProperties); - assertTrue(conf.get("mapreduce.job.hdfs-servers")==null|| - conf.get("mapreduce.job.hdfs-servers").equals("hdfs://a.com:8020")); - - pigServer.registerQuery("store a into 'hdfs://b.com/user/pig/1.txt';"); - System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); - conf = ConfigurationUtil.toConfiguration(pigProperties); - assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null && - conf.get("mapreduce.job.hdfs-servers").contains("hdfs://b.com")); - - pigServer.registerQuery("store a into 'har://hdfs-c.com:8020/user/pig/1.txt';"); - System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); - conf = ConfigurationUtil.toConfiguration(pigProperties); - assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null && - conf.get("mapreduce.job.hdfs-servers").contains("hdfs://c.com:8020")); - - pigServer.registerQuery("store a into 'hdfs://d.com:8020/user/pig/1.txt';"); - System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); - conf = ConfigurationUtil.toConfiguration(pigProperties); - assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null && - conf.get("mapreduce.job.hdfs-servers").contains("hdfs://d.com:8020")); - } catch (IOException io) { - fail(io.getMessage()); - } + Properties pigProperties = pigServer.getPigContext().getProperties(); + pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020"); + Configuration conf; + + pigServer.setBatchOn(); + + Data data = Storage.resetData(pigServer.getPigContext()); + data.set("/user/pig/1.txt");// no data + + pigServer.registerQuery("a = load '/user/pig/1.txt' using mock.Storage;"); + pigServer.registerQuery("store a into '/user/pig/1.txt';"); + + System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); + conf = ConfigurationUtil.toConfiguration(pigProperties); + assertTrue(conf.get("mapreduce.job.hdfs-servers")==null|| + conf.get("mapreduce.job.hdfs-servers").equals("hdfs://a.com:8020")); + + pigServer.registerQuery("store a into 'hdfs://b.com/user/pig/1.txt';"); + System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); + conf = ConfigurationUtil.toConfiguration(pigProperties); + assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null && + conf.get("mapreduce.job.hdfs-servers").contains("hdfs://b.com")); + + pigServer.registerQuery("store a into 'har://hdfs-c.com:8020/user/pig/1.txt';"); + System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); + conf = ConfigurationUtil.toConfiguration(pigProperties); + assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null && + conf.get("mapreduce.job.hdfs-servers").contains("hdfs://c.com:8020")); + + pigServer.registerQuery("store a into 'hdfs://d.com:8020/user/pig/1.txt';"); + System.out.println("hdfs-servers: " + pigProperties.getProperty("mapreduce.job.hdfs-servers")); + conf = ConfigurationUtil.toConfiguration(pigProperties); + assertTrue(conf.get("mapreduce.job.hdfs-servers")!=null && + conf.get("mapreduce.job.hdfs-servers").contains("hdfs://d.com:8020")); + } } Modified: pig/trunk/test/org/apache/pig/test/TestPigStorage.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStorage.java?rev=1359495&r1=1359494&r2=1359495&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestPigStorage.java (original) +++ pig/trunk/test/org/apache/pig/test/TestPigStorage.java Tue Jul 10 07:05:16 2012 @@ -28,6 +28,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -108,7 +109,7 @@ public class TestPigStorage { @Test public void testBlockBoundary() throws ExecException { - // This tests PigStorage loader with records exectly + // This tests PigStorage loader with records exactly // on the boundary of the file blocks. Properties props = new Properties(); for (Entry entry : cluster.getProperties().entrySet()) { @@ -199,10 +200,11 @@ public class TestPigStorage { pigContext.connect(); String query = "a = LOAD '" + datadir + "originput' using PigStorage('\\t', '-schema') " + "as (f1:chararray, f2:int);"; - pig.registerQuery(query); try{ + pig.registerQuery(query); pig.dumpSchema("a"); }catch(FrontendException ex){ + assertEquals(ex.toString(), 1000, ex.getErrorCode()); return; } fail("no exception caught");