pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billgra...@apache.org
Subject svn commit: r1372669 - in /pig/trunk: CHANGES.txt src/org/apache/pig/parser/QueryParserDriver.java test/org/apache/pig/test/TestPigServer.java test/org/apache/pig/test/TestPigServerWithMacros.java
Date Mon, 13 Aug 2012 23:48:46 GMT
Author: billgraham
Date: Mon Aug 13 23:48:46 2012
New Revision: 1372669

URL: http://svn.apache.org/viewvc?rev=1372669&view=rev
Log:
PIG-2866: PigServer fails with macros without a script file (billgraham)

Added:
    pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
    pig/trunk/test/org/apache/pig/test/TestPigServer.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1372669&r1=1372668&r2=1372669&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Aug 13 23:48:46 2012
@@ -232,6 +232,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-2866: PigServer fails with macros without a script file (billgraham)
+
 PIG-2860: [piggybank] TestAvroStorageUtils.testGetConcretePathFromGlob fails on some version
of hadoop (cheolsoo via jcoveney)
 
 PIG-2861: PlanHelper imports org.python.google.common.collect.Lists instead of org.google.common.collect.Lists
(jcoveney)

Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1372669&r1=1372668&r2=1372669&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Mon Aug 13 23:48:46 2012
@@ -407,10 +407,15 @@ public class QueryParserDriver {
         String body = bodyNode.getChild(0).getText();
 
         body = body.substring(1, body.length() - 1);
-        
-        FetchFileRet localFileRet = getMacroFile(fname);
 
-        PigMacro pm = new PigMacro(mn, localFileRet.file.getAbsolutePath(), params, returns,
body, seen);
+        // sometimes the script has no filename, like when a string is passed to PigServer
for
+        // example. See PIG-2866.
+        if (fname != null) {
+            FetchFileRet localFileRet = getMacroFile(fname);
+            fname = localFileRet.file.getAbsolutePath();
+        }
+
+        PigMacro pm = new PigMacro(mn, fname, params, returns, body, seen);
         
         try {
             pm.validate();

Modified: pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=1372669&r1=1372668&r2=1372669&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServer.java Mon Aug 13 23:48:46 2012
@@ -45,6 +45,7 @@ import java.util.Properties;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.pig.builtin.mock.Storage;
 import org.apache.pig.impl.PigContext;
 
 
@@ -54,7 +55,6 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.util.LogUtils;
 import org.apache.pig.impl.util.PropertiesUtil;
 import org.apache.pig.impl.util.Utils;
 import org.junit.After;
@@ -398,57 +398,6 @@ public class TestPigServer {
         // clean-up
         Assert.assertTrue(fs.delete(new Path(jarLocation), true));
     }
-    
-    @Test
-    public void testRegisterRemoteMacro() throws Throwable {
-        String macroName = "util.pig";
-        File macroFile = File.createTempFile("tmp", "");
-        PrintWriter pw = new PrintWriter(new FileWriter(macroFile));
-        pw.println("DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate
COUNT($X); };");
-        pw.close();
-        
-        FileSystem fs = cluster.getFileSystem();
-        fs.copyFromLocalFile(new Path(macroFile.getAbsolutePath()), new Path(macroName));
-        
-        // find the absolute path for the directory so that it does not
-        // depend on configuration
-        String absPath = fs.getFileStatus(new Path(macroName)).getPath().toString();
-        
-        Util.createInputFile(cluster, "testRegisterRemoteMacro_input", new String[]{"1",
"2"});
-        
-        pig.registerQuery("import '" + absPath + "';");
-        pig.registerQuery("a = load 'testRegisterRemoteMacro_input';");
-        pig.registerQuery("b = row_count(a);");
-        Iterator<Tuple> iter = pig.openIterator("b");
-        
-        Assert.assertTrue(((Long)iter.next().get(0))==2);
-    }
-    
-    @Test
-    public void testRegisterRemoteScript() throws Throwable {
-        String scriptName = "script.py";
-        File scriptFile = File.createTempFile("tmp", "");
-        PrintWriter pw = new PrintWriter(new FileWriter(scriptFile));
-        pw.println("@outputSchema(\"word:chararray\")\ndef helloworld():\n    return 'Hello,
World'");
-        pw.close();
-        
-        FileSystem fs = cluster.getFileSystem();
-        fs.copyFromLocalFile(new Path(scriptFile.getAbsolutePath()), new Path(scriptName));
-        
-        // find the absolute path for the directory so that it does not
-        // depend on configuration
-        String absPath = fs.getFileStatus(new Path(scriptName)).getPath().toString();
-        
-        Util.createInputFile(cluster, "testRegisterRemoteScript_input", new String[]{"1",
"2"});
-        pig.registerCode(absPath, "jython", "pig");
-        pig.registerQuery("a = load 'testRegisterRemoteScript_input';");
-        pig.registerQuery("b = foreach a generate pig.helloworld($0);");
-        Iterator<Tuple> iter = pig.openIterator("b");
-        
-        Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
-        Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
-        Assert.assertFalse(iter.hasNext());
-    }
 
     @Test
     public void testDescribeLoad() throws Throwable {

Added: pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java?rev=1372669&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java Mon Aug 13 23:48:46 2012
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.builtin.mock.Storage;
+import org.apache.pig.data.Tuple;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.util.Iterator;
+
+import static org.apache.pig.builtin.mock.Storage.resetData;
+import static org.apache.pig.builtin.mock.Storage.tuple;
+
+public class TestPigServerWithMacros {
+    private PigServer pig = null;
+
+    @Before
+    public void setUp() throws Exception{
+        pig = new PigServer(ExecType.LOCAL);
+    }
+
+    @After
+    public void tearDown() throws Exception{
+        pig = null;
+    }
+
+    @Test
+    public void testRegisterRemoteMacro() throws Throwable {
+        String macroName = "util.pig";
+        File macroFile = File.createTempFile("tmp", "");
+        PrintWriter pw = new PrintWriter(new FileWriter(macroFile));
+        pw.println("DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate
COUNT($X); };");
+        pw.close();
+
+        Path macroPath = new Path(macroName);
+        FileSystem fs = macroPath.getFileSystem(new Configuration());
+
+        fs.copyFromLocalFile(new Path(macroFile.getAbsolutePath()), macroPath);
+
+        // find the absolute path for the directory so that it does not
+        // depend on configuration
+        String absPath = fs.getFileStatus(new Path(macroName)).getPath().toString();
+
+        pig = new PigServer(ExecType.LOCAL);
+        Storage.Data data = resetData(pig);
+        data.set("some_path", "(l:chararray)", tuple("first row"), tuple("second row"));
+
+        pig.registerQuery("import '" + absPath + "';");
+        pig.registerQuery("a = load 'some_path' USING mock.Storage();");
+        pig.registerQuery("b = row_count(a);");
+        Iterator<Tuple> iter = pig.openIterator("b");
+
+        Assert.assertEquals(2L, iter.next().get(0));
+    }
+
+    @Test
+    public void testInlineMacro() throws Throwable {
+        Storage.Data data = resetData(pig);
+        data.set("some_path", "(l:chararray)", tuple("first row"), tuple("second row"));
+
+        pig.registerQuery("DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach
Y generate COUNT($X); };");
+        pig.registerQuery("a = load 'some_path' USING mock.Storage();");
+        pig.registerQuery("b = row_count(a);");
+        Iterator<Tuple> iter = pig.openIterator("b");
+
+        Assert.assertEquals(2L, iter.next().get(0));
+    }
+
+    @Test
+    public void testRegisterRemoteScript() throws Throwable {
+        String scriptName = "script.py";
+        File scriptFile = File.createTempFile("tmp", "");
+        PrintWriter pw = new PrintWriter(new FileWriter(scriptFile));
+        pw.println("@outputSchema(\"word:chararray\")\ndef helloworld():\n    return 'Hello,
World'");
+        pw.close();
+
+        Path scriptPath = new Path(scriptName);
+        FileSystem fs = scriptPath.getFileSystem(new Configuration());
+        fs.copyFromLocalFile(new Path(scriptFile.getAbsolutePath()), scriptPath);
+
+        // find the absolute path for the directory so that it does not
+        // depend on configuration
+        String absPath = fs.getFileStatus(scriptPath).getPath().toString();
+
+        Storage.Data data = resetData(pig);
+        data.set("some_path", "(l:chararray)", tuple(tuple("first row")), tuple(tuple("second
row")));
+
+        pig.registerCode(absPath, "jython", "pig");
+        pig.registerQuery("a = load 'some_path' USING mock.Storage();");
+        pig.registerQuery("b = foreach a generate pig.helloworld($0);");
+        Iterator<Tuple> iter = pig.openIterator("b");
+
+        Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
+        Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
+        Assert.assertFalse(iter.hasNext());
+    }
+}



Mime
View raw message