pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1368271 - in /pig/trunk: CHANGES.txt src/org/apache/pig/parser/QueryParserDriver.java src/org/apache/pig/parser/QueryParserUtils.java test/org/apache/pig/test/TestMacroExpansion.java
Date Wed, 01 Aug 2012 21:40:34 GMT
Author: daijy
Date: Wed Aug  1 21:40:33 2012
New Revision: 1368271

URL: http://svn.apache.org/viewvc?rev=1368271&view=rev
Log:
PIG-2729: Macro expansion does not use pig.import.search.path - UnitTest borked

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
    pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
    pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Aug  1 21:40:33 2012
@@ -475,6 +475,8 @@ PIG-2228: support partial aggregation in
 
 BUG FIXES
 
+PIG-2729: Macro expansion does not use pig.import.search.path - UnitTest borked (johannesch
via daijy)
+
 PIG-2783: Fix Iterator_1 e2e test for Hadoop 23 (rohini via daijy)
 
 PIG-2761: With hadoop23 importing modules inside python script does not work (rohini via
daijy)

Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Wed Aug  1 21:40:33 2012
@@ -19,6 +19,8 @@
 package org.apache.pig.parser;
 
 import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -344,10 +346,13 @@ public class QueryParserDriver {
     private FetchFileRet getMacroFile(String fname) {
         FetchFileRet localFileRet = null;
         try {
-            if (fnameMap.get(fname)!=null) {
+            if (fnameMap.get(fname) != null) {
                 localFileRet = fnameMap.get(fname);
             } else {
-                localFileRet = FileLocalizer.fetchFile(pigContext.getProperties(), fname);
+                File localFile = QueryParserUtils.getFileFromImportSearchPath(fname);
+                localFileRet = localFile == null ?
+                        FileLocalizer.fetchFile(pigContext.getProperties(), fname)
+                        : new FetchFileRet(localFile.getCanonicalFile(), false);
                 fnameMap.put(fname, localFileRet);
             }
         } catch (IOException e) {
@@ -443,7 +448,7 @@ public class QueryParserDriver {
         
         BufferedReader in = null;
         try {
-            in = QueryParserUtils.getImportScriptAsReader(localFileRet.file.getAbsolutePath());
+            in = new BufferedReader(new FileReader(localFileRet.file));
         } catch (FileNotFoundException e) {
             String msg = getErrorMessage(fname, t,
                     "Failed to import file '" + fname + "'", e.getMessage());

Modified: pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java Wed Aug  1 21:40:33 2012
@@ -18,10 +18,7 @@
 
 package org.apache.pig.parser;
 
-import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -54,7 +51,7 @@ import org.apache.pig.tools.pigstats.Scr
 public class QueryParserUtils {
     private static Log log = LogFactory.getLog( LogicalPlanGenerator.class );
 
-	public static String removeQuotes(String str) {
+    public static String removeQuotes(String str) {
         if (str.startsWith("\u005c'") && str.endsWith("\u005c'"))
             return str.substring(1, str.length() - 1);
         else
@@ -201,12 +198,11 @@ public class QueryParserUtils {
         }
     }
 
-    static BufferedReader getImportScriptAsReader(String scriptPath)
-            throws FileNotFoundException {
+     static File getFileFromImportSearchPath(String scriptPath) {
         File f = new File(scriptPath);
         if (f.exists() || f.isAbsolute() || scriptPath.startsWith("./")
                 || scriptPath.startsWith("../")) {
-            return new BufferedReader(new FileReader(f));
+            return f;
         }
 
         ScriptState state = ScriptState.get();
@@ -218,14 +214,13 @@ public class QueryParserUtils {
                 for (String path : paths) {
                     File f1 = new File(path + File.separator + scriptPath);
                     if (f1.exists()) {
-                        return new BufferedReader(new FileReader(f1));
+                        return f1;
                     }
                 }
             }
         }
 
-        throw new FileNotFoundException("Can't find the Specified file "
-                + scriptPath);
+        return null;
     }
     
     static QueryParser createParser(CommonTokenStream tokens) {

Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Wed Aug  1 21:40:33 2012
@@ -19,6 +19,7 @@
 package org.apache.pig.test;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -44,6 +45,10 @@ import org.junit.Test;
 
 public class TestMacroExpansion {
     
+    private static String groupAndCountMacro = "define group_and_count (A,group_key, reducers)
returns B {\n" +
+            "    $B = distinct $A partition BY org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
+            "};\n";
+    private static String garbageMacroContent = "&?:#garbage-!";
     static File command;
     
     @BeforeClass
@@ -102,10 +107,6 @@ public class TestMacroExpansion {
     
     @Test
     public void distinctTest() throws Exception {
-        String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
-            "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
-            "};\n";
-        
         String script = 
             "alpha = load 'users' as (user, age, zip);\n" +
             "gamma = group_and_count (alpha, user, 23);\n" +
@@ -120,7 +121,7 @@ public class TestMacroExpansion {
             "store gamma INTO 'byuser';\n" +
             "store delta INTO 'byage';\n";
         
-        verify(macro + script, expected);
+        verify(groupAndCountMacro + script, expected);
     }   
     
     @Test
@@ -658,10 +659,6 @@ public class TestMacroExpansion {
     
     @Test
     public void duplicationTest() throws Throwable {
-        String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
-            "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
-            "};\n";
-        
         String script = 
             "alpha = load 'users' as (user, age, zip);\n" +
             "gamma = group_and_count (alpha, user, 23);\n" +
@@ -671,21 +668,12 @@ public class TestMacroExpansion {
         
         String expectedErr = "Reason: Duplicated macro name 'group_and_count'";
         
-        validateFailure(macro + macro + script, expectedErr);
+        validateFailure(groupAndCountMacro + groupAndCountMacro + script, expectedErr);
     }
     
     @Test
     public void simpleImportTest() throws Exception {
-        String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
-            "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
-            "};\n";
-        
-        File f = new File("mytest.pig");
-        f.deleteOnExit();
-        
-        FileWriter fw = new FileWriter(f);
-        fw.append(macro);
-        fw.close();
+        createFile("mytest.pig", groupAndCountMacro);
         
         String script =
             "import 'mytest.pig';\n" +
@@ -703,66 +691,55 @@ public class TestMacroExpansion {
     
     @Test 
     public void importUsingSearchPathTest() throws Exception {
-        String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
-            "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
-            "};\n";
-        
-        File f = new File("/tmp/mytest2.pig");
-        f.deleteOnExit();
-        
-        FileWriter fw = new FileWriter(f);
-        fw.append(macro);
-        fw.close();
-        
-        String script =
-            "import '/tmp/mytest2.pig';\n" +
-            "alpha = load 'users' as (user, age, zip);\n" +
-            "gamma = group_and_count (alpha, user, 23);\n" +
-            "store gamma into 'byuser';\n";
-        
-        File f1 = new File("myscript.pig");
-        f1.deleteOnExit();
-        
-        FileWriter fw1 = new FileWriter(f1);
-        fw1.append(script);
-        fw1.close();
-        
-        String[] args = { "-Dpig.import.search.path=/tmp", "-x", "local", "-c", "myscript.pig"
};
-        PigStats stats = PigRunner.run(args, null);
- 
-        assertTrue(stats.isSuccessful());
+        createFile("/tmp/mytest2.pig", garbageMacroContent);
+        assertTrue(verifyImportUsingSearchPath("/tmp/mytest2.pig", "mytest2.pig", "/tmp"));
+    }
+
+    @Test
+    public void importUsingSearchPathTest2() throws Exception {
+        createFile("/tmp/mytest2.pig", garbageMacroContent);
+        assertTrue(verifyImportUsingSearchPath("mytest2.pig", "./mytest2.pig", "/tmp"));
+    }
+
+    @Test
+    public void importUsingSearchPathTest3() throws Exception {
+        createFile("/tmp/mytest2.pig", garbageMacroContent);
+        assertTrue(verifyImportUsingSearchPath("../mytest2.pig", "../mytest2.pig", "/tmp"));
+    }
+
+    @Test
+    public void importUsingSearchPathTest4() throws Exception {
+        createFile("/tmp/pigtestdir/tmp/mytest2.pig", garbageMacroContent);
+        assertTrue(verifyImportUsingSearchPath("/tmp/mytest2.pig", "/tmp/mytest2.pig", "/tmp/pigtestdir/"));
+    }
+
+    @Test
+    public void importUsingSearchPathTest5() throws Exception {
+        createFile("/tmp/mytest2a.pig", garbageMacroContent);
+        createFile("mytest2a.pig", groupAndCountMacro);
+        assertTrue(verifyImportUsingSearchPath("/tmp/mytest2a.pig", "mytest2a.pig", "/tmp",
false));
     }
     
     @Test
-    public void negtiveUsingSearchPathTest() throws Exception {
-        String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
-            "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
-            "};\n";
-        
-        File f = new File("/tmp/mytest2.pig");
-        f.deleteOnExit();
-        
-        FileWriter fw = new FileWriter(f);
-        fw.append(macro);
-        fw.close();
-        
-        String script =
-            "import 'mytest2.pig';\n" +
-            "alpha = load 'users' as (user, age, zip);\n" +
-            "gamma = group_and_count (alpha, user, 23);\n" +
-            "store gamma into 'byuser';\n";
-        
-        File f1 = new File("myscript.pig");
-        f1.deleteOnExit();
-                
-        FileWriter fw1 = new FileWriter(f1);
-        fw1.append(script);
-        fw1.close();
-        
-        String[] args = { "-x", "local", "-c", "myscript.pig" };
-        PigStats stats = PigRunner.run(args, null);
- 
-        assertTrue(!stats.isSuccessful());
+    public void negativeUsingSearchPathTest() throws Exception {
+        /* Delete the mytest3.pig file in negativeUsingSearchPathTest, just in
+         * case negativeUsingSearchPathTest2 is executed first and a garbage mytest3.pig
+         * file is created.
+         */
+        new File("mytest3.pig").delete();
+        assertFalse(verifyImportUsingSearchPath("/tmp/mytest3.pig", "mytest3.pig", null));
+    }
+
+    @Test
+    public void negativeUsingSearchPathTest2() throws Exception {
+        createFile("mytest3.pig", garbageMacroContent);
+        assertFalse(verifyImportUsingSearchPath("/tmp/mytest3.pig", "mytest3.pig", "/tmp"));
+    }
+
+    @Test
+    public void negativeUsingSearchPathTest3() throws Exception {
+        createFile("/tmp/mytest3a.pig", garbageMacroContent);
+        assertFalse(verifyImportUsingSearchPath("/tmp/mytest3a.pig", "mytest3a.pig", "/tmp",
false));
     }
     
     @Test
@@ -774,12 +751,7 @@ public class TestMacroExpansion {
             "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
             "};\n";
         
-        File f = new File("mytest.pig");
-        f.deleteOnExit();
-        
-        FileWriter fw = new FileWriter(f);
-        fw.append(macro);
-        fw.close();
+        createFile("mytest.pig", macro);
         
         String script =
             "import 'mytest.pig';\n" +
@@ -804,28 +776,16 @@ public class TestMacroExpansion {
         String macro1 = "define group_and_count (A, reducers) returns B {\n" +
             "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
             "};\n";
-        
         String macro2 = "define distinct_with_reducer(A, reducers) returns B {\n" +
             "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
             "};\n";
         
-        File f1 = new File("mytest1.pig");
-        f1.deleteOnExit();
-        
-        FileWriter fw1 = new FileWriter(f1);
-        fw1.append(macro1);
-        fw1.close();
-        
-        File f2 = new File("mytest2.pig");
-        f2.deleteOnExit();
-        
-        FileWriter fw2 = new FileWriter(f2);
-        fw2.append(macro2);
-        fw2.close();
+        createFile("mytest4.pig", macro1);
+        createFile("mytest5.pig", macro2);
         
         String script =
-            "import 'mytest1.pig';\n" +
-            "import 'mytest2.pig';\n" +
+            "import 'mytest4.pig';\n" +
+            "import 'mytest5.pig';\n" +
             "alpha = load 'users' as (user, age, zip);\n" +
             "gamma = group_and_count (alpha, 23);\n" +
             "beta = distinct_with_reducer(alpha, 32);\n" +
@@ -2004,12 +1964,7 @@ public class TestMacroExpansion {
     // PIG-1988
     @Test
     public void test36() throws Exception {
-        File f = new File("mymacro.pig");
-        f.deleteOnExit();
-        
-        FileWriter fw = new FileWriter(f);
-        fw.append(" ");
-        fw.close();
+        createFile("mymacro.pig", " ");
 
         String query = "import 'mymacro.pig';" +
             "define macro1() returns void {}; " + 
@@ -2029,17 +1984,8 @@ public class TestMacroExpansion {
     // PIG-1987
     @Test
     public void test37() throws Exception {
-        String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
-        "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner
parallel $reducers;\n" +
-        "};\n";
-    
-        File f = new File("mytest.pig");
-        f.deleteOnExit();
-        
-        FileWriter fw = new FileWriter(f);
-        fw.append(macro);
-        fw.close();
-        
+        createFile("mytest.pig", groupAndCountMacro);
+
         String script =
             "set default_parallel 10\n" +
             "import 'mytest.pig';\n" +
@@ -2124,12 +2070,7 @@ public class TestMacroExpansion {
     }
     
     private void verify(String s, String expected) throws Exception {
-        File f1 = new File("myscript.pig");
-        f1.deleteOnExit();
-        
-        FileWriter fw1 = new FileWriter(f1);
-        fw1.append(s);
-        fw1.close();
+        createFile("myscript.pig", s);
         
         String[] args = { "-Dpig.import.search.path=/tmp", "-x", "local", "-c", "myscript.pig"
};
         PigStats stats = PigRunner.run(args, null);
@@ -2231,5 +2172,54 @@ public class TestMacroExpansion {
     private void validateFailure(String piglatin, String expectedErr) throws Throwable {
         validateFailure(piglatin, expectedErr, "Reason:");
     }
+
+    private void createFile(String filePath, String content) throws Exception {
+        createFile(filePath, content, true);
+    }
+
+    private void createFile(String filePath, String content, boolean deleteOnExit) throws
Exception {
+        File f = new File(filePath);
+        if (f.getParent() != null && !(new File(f.getParent())).exists()) {
+            (new File(f.getParent())).mkdirs();
+        }
+        
+        if (deleteOnExit) {
+            f.deleteOnExit();
+        }
+
+        FileWriter fw = new FileWriter(f);
+        fw.append(content);
+        fw.close();
+    }
+
+    private boolean verifyImportUsingSearchPath(String macroFilePath, String importFilePath,
+            String importSearchPath) throws Exception {
+        return verifyImportUsingSearchPath(macroFilePath, importFilePath,
+                importSearchPath, true);
+    }
+
+    private boolean verifyImportUsingSearchPath(String macroFilePath, String importFilePath,
+            String importSearchPath, boolean createMacroFilePath) throws Exception {
+
+        if (createMacroFilePath) {
+            createFile(macroFilePath, groupAndCountMacro);
+        }
+
+        String script =
+            "import '" + importFilePath + "';\n" +
+            "alpha = load 'users' as (user, age, zip);\n" +
+            "gamma = group_and_count (alpha, user, 23);\n" +
+            "store gamma into 'byuser';\n";
+
+        createFile("myscript.pig", script);
+
+        String[] args = {
+                (importSearchPath != null ? "-Dpig.import.search.path=" + importSearchPath
: ""),
+                "-x", "local", "-c", "myscript.pig"
+        };
+        PigStats stats = PigRunner.run(args, null);
+
+        return stats.isSuccessful();
+    }
     
 }



Mime
View raw message