pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roh...@apache.org
Subject svn commit: r1817125 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/tools/grunt/ src/org/apache/pig/tools/pigstats/ test/org/apache/pig/pigunit/pig/ test/org/apache/pig/test/
Date Mon, 04 Dec 2017 20:28:18 GMT
Author: rohini
Date: Mon Dec  4 20:28:18 2017
New Revision: 1817125

URL: http://svn.apache.org/viewvc?rev=1817125&view=rev
Log:
PIG-5315: pig.script is not set for scripts run via PigServer (satishsaley via rohini)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/PigServer.java
    pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
    pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
    pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java
    pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Dec  4 20:28:18 2017
@@ -60,6 +60,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5315: pig.script is not set for scripts run via PigServer (satishsaley via rohini)
+
 PIG-5310: MergeJoin throwing NullPointer Exception (satishsaley via rohini)
 
 PIG-5314: Abort method is not implemented in PigProcessor (satishsaley via rohini)

Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Mon Dec  4 20:28:18 2017
@@ -43,6 +43,7 @@ import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -775,7 +776,10 @@ public class PigServer {
      */
     public void registerScript(InputStream in, Map<String,String> params,List<String>
paramsFiles) throws IOException {
         try {
-            String substituted = pigContext.doParamSubstitution(in, paramMapToList(params),
paramsFiles);
+            String script = IOUtils.toString(in);
+            ScriptState.get().setScript(script);
+            String substituted = pigContext.doParamSubstitution(new BufferedReader(new StringReader(script)),
+                    paramMapToList(params), paramsFiles);
             GruntParser grunt = new GruntParser(new StringReader(substituted), this);
             grunt.setInteractive(false);
             grunt.parseStopOnError(true);
@@ -855,6 +859,7 @@ public class PigServer {
         FileInputStream fis = null;
         try{
             fis = new FileInputStream(fileName);
+            ScriptState.get().setFileName(fileName);
             registerScript(fis, params, paramsFiles);
         }catch (FileNotFoundException e){
             log.error(e.getLocalizedMessage());

Modified: pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Mon Dec  4 20:28:18 2017
@@ -73,6 +73,7 @@ import org.apache.pig.tools.pigscript.pa
 import org.apache.pig.tools.pigstats.JobStats;
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStats.JobGraph;
+import org.apache.pig.tools.pigstats.ScriptState;
 import org.apache.pig.validator.BlackAndWhitelistFilter;
 import org.apache.pig.validator.PigCommandFilter;
 import org.fusesource.jansi.Ansi;
@@ -494,9 +495,12 @@ public class GruntParser extends PigScri
                 setBatchOn();
                 mPigServer.setJobName(script);
                 try {
-                    loadScript(script, true, false, mLoadOnly, params, files);
+                    FetchFileRet scriptFile = FileLocalizer.fetchFile(mConf, script);
+                    ScriptState.get().beginNestedScript(scriptFile.file);
+                    loadScript(scriptFile, script, true, false, mLoadOnly, params, files);
                     executeBatch();
                 } finally {
+                    ScriptState.get().endNestedScript();
                     discardBatch();
                 }
             } else {
@@ -508,6 +512,11 @@ public class GruntParser extends PigScri
     }
 
     private void loadScript(String script, boolean batch, boolean loadOnly, boolean illustrate,
+            List<String> params, List<String> files) throws IOException, ParseException
{
+        loadScript(FileLocalizer.fetchFile(mConf, script), script, batch, loadOnly, illustrate,
params, files);
+    }
+
+    private void loadScript(FetchFileRet fetchFile, String script, boolean batch, boolean
loadOnly, boolean illustrate,
                             List<String> params, List<String> files)
         throws IOException, ParseException {
 
@@ -524,7 +533,6 @@ public class GruntParser extends PigScri
         pc.setParamFiles(files);
 
         try {
-            FetchFileRet fetchFile = FileLocalizer.fetchFile(mConf, script);
             String cmds = runPreprocessor(fetchFile.file.getAbsolutePath(), params, files);
 
             if (mInteractive && !batch) { // Write prompt and echo commands

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Mon Dec  4 20:28:18 2017
@@ -26,6 +26,7 @@ import java.util.BitSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Stack;
 import java.util.UUID;
 import java.util.jar.Attributes;
 import java.util.jar.JarFile;
@@ -182,6 +183,8 @@ public abstract class ScriptState {
 
     protected List<PigProgressNotificationListener> listeners = Lists.newArrayList();
 
+    private Stack<ScriptInfo> scripts = new Stack<>();
+
     protected ScriptState(String id) {
         this.id = id;
         this.serializedScript = "";
@@ -439,6 +442,31 @@ public abstract class ScriptState {
         return featureLongToString(scriptFeatures);
     }
 
+    /**
+     * Stores information about the current script and pushes it onto a stack.
+     *
+     * @param scriptFile
+     * @throws IOException
+     */
+    public void beginNestedScript(File scriptFile) throws IOException {
+        ScriptInfo scriptInfo = new ScriptInfo();
+        scriptInfo.fileName = this.fileName;
+        scriptInfo.serializedScript = this.serializedScript;
+        scriptInfo.truncatedScript = this.truncatedScript;
+        scripts.push(scriptInfo);
+        this.setScript(scriptFile);
+        this.setFileName(scriptFile.getName());
+    }
+
+    public void endNestedScript() {
+        if (!scripts.isEmpty()) {
+            ScriptInfo scriptInfo = scripts.pop();
+            // Change the current script information
+            this.fileName = scriptInfo.fileName;
+            this.serializedScript = scriptInfo.serializedScript;
+            this.truncatedScript = scriptInfo.truncatedScript;
+        }
+    }
     static class LogicalPlanFeatureVisitor extends LogicalRelationalNodesVisitor {
 
         private BitSet feature;
@@ -712,4 +740,10 @@ public abstract class ScriptState {
             }
         }
     }
+
+    private static class ScriptInfo {
+        String serializedScript;
+        String truncatedScript;
+        String fileName;
+    }
 }

Modified: pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java (original)
+++ pig/trunk/test/org/apache/pig/pigunit/pig/PigServer.java Mon Dec  4 20:28:18 2017
@@ -20,9 +20,11 @@ import java.io.InputStreamReader;
 import java.util.Map;
 import java.util.Properties;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.pig.ExecType;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.impl.util.Utils;
+import org.apache.pig.tools.pigstats.ScriptState;
 
 /**
  * Slightly modified PigServer that accepts a list of Pig aliases to override.
@@ -49,6 +51,8 @@ public class PigServer extends org.apach
   public void registerScript(String fileName, Map<String, String> aliasOverride)
       throws IOException {
     try {
+      ScriptState.get().setFileName(fileName);
+      ScriptState.get().setScript(IOUtils.toString(new FileInputStream(fileName)));
       InputStream compositeStream = Utils.getCompositeStream(new FileInputStream(fileName),
pigContext.getProperties());
       GruntParser grunt = new GruntParser(new InputStreamReader(compositeStream), this, aliasOverride);
       grunt.setInteractive(false);

Modified: pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java?rev=1817125&r1=1817124&r2=1817125&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServerLocal.java Mon Dec  4 20:28:18 2017
@@ -20,6 +20,7 @@ package org.apache.pig.test;
 import static org.apache.pig.builtin.mock.Storage.resetData;
 import static org.apache.pig.builtin.mock.Storage.tuple;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -54,6 +55,7 @@ import org.apache.pig.scripting.ScriptEn
 import org.apache.pig.tools.grunt.Grunt;
 import org.apache.pig.tools.grunt.GruntParser;
 import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.ScriptState;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -167,6 +169,7 @@ public class TestPigServerLocal {
         params.put("input", "test/org/apache/pig/test/data/passwd");
         String script="a = load '$input' using PigStorage(':');";
         pig.registerScript(new ByteArrayInputStream(script.getBytes("UTF-8")),params);
+        assertEquals("ScriptState contains different script", script, ScriptState.get().getScript());
         Iterator<Tuple> iter=pig.openIterator("a");
         int index=0;
         List<Tuple> expectedTuples=Util.readFile2TupleList("test/org/apache/pig/test/data/passwd",
":");



Mime
View raw message