pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From knogu...@apache.org
Subject svn commit: r1748622 - in /pig/trunk: CHANGES.txt src/org/apache/pig/tools/parameters/PreprocessorContext.java test/org/apache/pig/test/TestParamSubPreproc.java
Date Wed, 15 Jun 2016 19:20:30 GMT
Author: knoguchi
Date: Wed Jun 15 19:20:30 2016
New Revision: 1748622

URL: http://svn.apache.org/viewvc?rev=1748622&view=rev
Log:
PIG-4896: Param substitution ignored when redefined (knoguchi)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
    pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1748622&r1=1748621&r2=1748622&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 15 19:20:30 2016
@@ -30,6 +30,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-4896: Param substitution ignored when redefined (knoguchi)
+
 PIG-2315: Make as clause work in generate (daijy via knoguchi)
 
 PIG-4921: Kill running jobs on InterruptedException (rohini)

Modified: pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1748622&r1=1748621&r2=1748622&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Wed Jun 15 19:20:30
2016
@@ -46,10 +46,6 @@ public class PreprocessorContext {
 
     private Map<String, String> param_val;
 
-    // used internally to detect when a param is set multiple times,
-    // but it set with the same value so it's ok not to log a warning
-    private Map<String, String> param_source;
-    
     private PigContext pigContext;
 
     public Map<String, String> getParamVal() {
@@ -64,23 +60,16 @@ public class PreprocessorContext {
      */
     public PreprocessorContext(int limit) {
         param_val = new Hashtable<String, String> (limit);
-        param_source = new Hashtable<String, String> (limit);
     }
 
     public PreprocessorContext(Map<String, String> paramVal) {
         param_val = paramVal;
-        param_source = new Hashtable<String, String>(paramVal);
     }
 
     public void setPigContext(PigContext context) {
         this.pigContext = context;
     }
 
-    /*
-    public  void processLiteral(String key, String val) {
-        processLiteral(key, val, true);
-    } */
-
     /**
      * This method generates parameter value by running specified command
      *
@@ -102,21 +91,6 @@ public class PreprocessorContext {
         processOrdLine(key, val, true);
     }
 
-    /*
-    public  void processLiteral(String key, String val, Boolean overwrite) {
-
-        if (param_val.containsKey(key)) {
-            if (overwrite) {
-                log.warn("Warning : Multiple values found for " + key + ". Using value "
+ val);
-            } else {
-                return;
-            }
-        }
-
-        String sub_val = substitute(val);
-        param_val.put(key, sub_val);
-    } */
-
     /**
      * This method generates parameter value by running specified command
      *
@@ -129,20 +103,20 @@ public class PreprocessorContext {
             filter.validate(PigCommandFilter.Command.SH);
         }
 
-        if (param_val.containsKey(key)) {
-            if (param_source.get(key).equals(val) || !overwrite) {
-                return;
-            } else {
-                log.warn("Warning : Multiple values found for " + key
-                        + ". Using value " + val);
-            }
+        if (param_val.containsKey(key) && !overwrite) {
+            return;
         }
 
-        param_source.put(key, val);
-
         val = val.substring(1, val.length()-1); //to remove the backticks
         String sub_val = substitute(val);
         sub_val = executeShellCommand(sub_val);
+
+        if (param_val.containsKey(key) && !param_val.get(key).equals(sub_val) ) {
+            //(boolean overwrite is always true here)
+            log.warn("Warning : Multiple values found for " + key + " command `" + val +
"`. "
+                     + "Previous value " + param_val.get(key) + ", now using value " + sub_val);
+        }
+
         param_val.put(key, sub_val);
     }
 
@@ -175,17 +149,17 @@ public class PreprocessorContext {
      */
     public  void processOrdLine(String key, String val, Boolean overwrite)  throws ParameterSubstitutionException
{
 
+        String sub_val = substitute(val, key);
         if (param_val.containsKey(key)) {
-            if (param_source.get(key).equals(val) || !overwrite) {
+            if (param_val.get(key).equals(sub_val) || !overwrite) {
                 return;
             } else {
-                log.warn("Warning : Multiple values found for " + key + ". Using value "
+ val);
+                log.warn("Warning : Multiple values found for " + key
+                         + ". Previous value " + param_val.get(key)
+                         + ", now using value " + sub_val);
             }
         }
 
-        param_source.put(key, val);
-
-        String sub_val = substitute(val, key);
         param_val.put(key, sub_val);
     }
 

Modified: pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1748622&r1=1748621&r2=1748622&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Wed Jun 15 19:20:30 2016
@@ -19,6 +19,7 @@
 package org.apache.pig.test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -943,6 +944,79 @@ public class TestParamSubPreproc {
         log.info("Done");
     }
 
+    @Test
+    public void testSubstitutionWithRedeclaration() throws Exception{
+        log.info("Starting test testSubstitutionWithRedeclaration()");
+        final String queryString =
+              "%declare output '/tmp/abc';\n" +
+              "%declare actualoutput '$output.out';\n" +
+              "A = load 'input.txt' ;\n" +
+              "store A into '$actualoutput';\n" +
+              "%declare output '/tmp/def';\n" +
+              "%declare actualoutput '$output.out';\n" +
+              "store A into '$actualoutput';";
+
+
+        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+        pigIStream = new BufferedReader(
+                                        new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
+        pigOStream = new FileWriter(basedir + "/output1.pig");
+
+        String[] arg = {"output = 'output.txt'"};
+        String[] argFiles = null;
+        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+
+        String expectedString = queryString.replaceAll("%declare [0-9a-zA-Z.'/\\$; ]*\n",";\n")
+                                .replaceAll("\\$","")
+                                .replaceFirst("actualoutput","/tmp/abc.out")
+                                .replaceFirst("actualoutput","/tmp/def.out");
+        InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
+
+        compareResults(expected, pigResultStream);
+
+        log.info("Done");
+    }
+
+    @Test
+    public void testSubstitutionWithRedeclaredShell() throws Exception{
+        log.info("Starting test testSubstitutionWithRedeclaredShell()");
+        final String queryString =
+              "A = load 'input.txt' ;\n" +
+              "%declare now `bash -c \"date +'%Y%m%d_%H:%M:%S'; sleep 1;\"`;\n" +
+              "store A into '$now';\n" +
+              "%declare now `bash -c \"date +'%Y%m%d_%H:%M:%S'; sleep 1;\"`;\n" +
+              "store A into '$now';\n";
+
+        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+        pigIStream = new BufferedReader(
+                                        new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
+        pigOStream = new FileWriter(basedir + "/output1.pig");
+
+        String[] arg = {"output = 'output.txt'"};
+        String[] argFiles = null;
+        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+        BufferedReader pigresult = new BufferedReader(new InputStreamReader(new FileInputStream(basedir
+ "/output1.pig")));
+
+
+        String [] filenames = new String [2];
+        int index=0;
+        String line;
+        while ((line = pigresult.readLine())!=null) {
+            if( line.startsWith("store A into") ) {
+                filenames[index++] = line.split(" ")[3];
+            }
+        }
+
+        assertEquals("There should be 2 store statements", 2, index);
+        assertNotEquals("Identical shell param should be reexecuted.",
+                     filenames[0],
+                     filenames[1]);
+        log.info("Done");
+    }
+
     @SuppressWarnings("resource")
     private BufferedReader WithConditionalReplacement(String filename, String orig, String
dest, boolean replace) throws IOException {
         BufferedReader pigOrigIStream = new BufferedReader(new FileReader(filename));



Mime
View raw message