pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From knogu...@apache.org
Subject svn commit: r1743583 - in /pig/trunk: CHANGES.txt src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj test/org/apache/pig/test/TestPigScriptParser.java
Date Thu, 12 May 2016 21:47:37 GMT
Author: knoguchi
Date: Thu May 12 21:47:37 2016
New Revision: 1743583

URL: http://svn.apache.org/viewvc?rev=1743583&view=rev
Log:
PIG-4889: Replacing backslash fails as lexical error (knoguchi)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
    pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1743583&r1=1743582&r2=1743583&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 12 21:47:37 2016
@@ -123,6 +123,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4889: Replacing backslash fails as lexical error (knoguchi)
+
 PIG-4880: Overlapping of parameter substitution names inside&outside a macro fails with
NPE (knoguchi)
 
 PIG-4881: TestBuiltin.testUniqueID failing on hadoop-1.x (knoguchi)

Modified: pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1743583&r1=1743582&r2=1743583&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Thu May 12 21:47:37
2016
@@ -23,6 +23,7 @@ options {
   STATIC = false;
   // Case is ignored in keywords
   IGNORE_CASE = true;
+  // DEBUG_PARSER = true;
   JAVA_UNICODE_ESCAPE = true;
 }
 
@@ -292,7 +293,8 @@ TOKEN_MGR_DECLS : {
 
 <IN_STRING> MORE :
 {
-	<"\\'">
+	<"\\\\">
+|	<"\\'">
 |	<"'"> { SwitchTo(prevState);}
 |	<("\n" | "\r" | "\r\n")> {secondary_prompt();}
 |	<(~[])>

Modified: pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java?rev=1743583&r1=1743582&r2=1743583&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java Thu May 12 21:47:37 2016
@@ -30,6 +30,9 @@ import java.util.Properties;
 
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
+import org.apache.pig.builtin.mock.Storage;
+import org.apache.pig.builtin.mock.Storage.Data;
+import static org.apache.pig.builtin.mock.Storage.tuple;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -156,6 +159,79 @@ public class TestPigScriptParser {
         }
     }
 
+    @Test
+    public void testBackSlashOnly() throws Exception {
+        PigServer pig = new PigServer(Util.getLocalTestMode());
+        Data data = Storage.resetData(pig);
+        data.set("input", tuple("abc"), tuple("\\bcd"), tuple("'cde"), tuple("def\\\\"));
+
+        String query =
+            "A = load 'input' USING mock.Storage() as (a0:chararray);\n"
+            // java String is escaping "\" so the following line is equivalent of
+            // B = FILTER A by STARTSWITH(a0,'\\'); in the pig script
+            + "B = FILTER A by STARTSWITH(a0,'\\\\');\n"
+            + "store B into 'out' using mock.Storage;" ;
+
+        Util.registerMultiLineQuery(pig, query);
+        List<Tuple> list = data.get("out");
+
+        assertEquals("There should be only one match", 1, list.size());
+        Tuple t = list.get(0);
+        assertEquals("result should have only one field", 1, t.size() );
+        assertEquals("\\bcd",(String) t.get(0));
+    }
+
+
+    @Test
+    public void testBackSlashSingleQuote() throws Exception {
+        PigServer pig = new PigServer(Util.getLocalTestMode());
+        Data data = Storage.resetData(pig);
+        data.set("input", tuple("abc"), tuple("\\bcd"), tuple("'cde"), tuple("def\\\\"));
+
+        String query =
+            "A = load 'input' USING mock.Storage() as (a0:chararray);\n"
+            // java String is escaping "\" so the following line is equivalent of
+            // B = FILTER A by STARTSWITH(a0,'\''); in the pig script
+            + "B = FILTER A by STARTSWITH(a0,'\\'');\n"
+            + "store B into 'out' using mock.Storage;" ;
+
+        Util.registerMultiLineQuery(pig, query);
+        List<Tuple> list = data.get("out");
+
+        assertEquals("There should be only one match", 1, list.size());
+        Tuple t = list.get(0);
+        assertEquals("result should have only one field", 1, t.size() );
+        assertEquals("'cde",(String) t.get(0));
+    }
+
+    @Test
+    public void testBackSlashReplace() throws Exception {
+        PigServer pig = new PigServer(Util.getLocalTestMode());
+        Data data = Storage.resetData(pig);
+        //After java escaping, these tuples have
+        //'abc', '\bcd' and 'def\\' respectively
+        data.set("input", tuple("abc"), tuple("\\bcd"), tuple("def\\\\"));
+
+        String query =
+            "A = load 'input' USING mock.Storage() as (a0:chararray);\n"
+            // java String is escaping "\" so the following line is equivalent of
+            //"B = FOREACH A GENERATE REPLACE(a0,'\\\\','+');\n"
+            + "B = FOREACH A GENERATE REPLACE(a0,'\\\\\\\\','+');\n"
+            + "store B into 'out' using mock.Storage;" ;
+
+            // REPLACE(a0,'\\\\','+')
+            // --> Pig parser unescape and pass "\\" to REPLACE UDF.
+            // --> REPLACE UDF calls, Pattern.compile("\\"); which
+            // matches "\"
+
+        Util.registerMultiLineQuery(pig, query);
+        List<Tuple> list = data.get("out");
+
+        List<Tuple> expectedRes =
+                Util.getTuplesFromConstantTupleStrings(
+                        new String[] {"('abc')","('+bcd')", "('def++')"});
+        Util.checkQueryOutputsAfterSort(list, expectedRes);
+    }
     private void checkParsedConstContent(PigServer pigServer,
                                          PigContext pigContext,
                                          String query,



Mime
View raw message