pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject svn commit: r743908 - in /hadoop/pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/tools/grunt/ src/org/apache/pig/tools/pigscript/parser/ test/org/apache/pig/test/ test/org/apache/pig/test/data/
Date Thu, 12 Feb 2009 22:27:47 GMT
Author: olga
Date: Thu Feb 12 22:27:46 2009
New Revision: 743908

URL: http://svn.apache.org/viewvc?rev=743908&view=rev
Log:
PIG-574: allowing to run scripts from within grunt shell (hagleitn via olgan)

Added:
    hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf
    hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf
    hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig
    hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig
Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/PigServer.java
    hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
    hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
    hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Feb 12 22:27:46 2009
@@ -413,3 +413,6 @@
     PIG-553: EvalFunc.finish() not getting called (shravanmn via sms)
 
     PIG-654: Optimize build.xml (daijy)
+
+    PIG-574: allowing to run scripts from within grunt shell (hagleitn via
+    olgan)

Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Thu Feb 12 22:27:46 2009
@@ -123,19 +123,21 @@
     }
 
     public PigServer(ExecType execType, Properties properties) throws ExecException {
-        this.pigContext = new PigContext(execType, properties);
-        if (this.pigContext.getProperties().getProperty(PigContext.JOB_NAME) == null) {
-            setJobName("DefaultJobName") ;
-        }
-        pigContext.connect();
+        this(new PigContext(execType, properties), true);
     }
-    
+  
     public PigServer(PigContext context) throws ExecException {
+        this(context, true);
+    }
+    
+    public PigServer(PigContext context, boolean connect) throws ExecException {
         this.pigContext = context;
         if (this.pigContext.getProperties().getProperty(PigContext.JOB_NAME) == null) {
             setJobName("DefaultJobName") ;
         }
-        pigContext.connect();
+        if (connect) {
+            pigContext.connect();
+        }
     }
 
     public PigContext getPigContext(){

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Thu Feb 12 22:27:46 2009
@@ -21,13 +21,26 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Reader;
+import java.io.FileReader;
+import java.io.FileInputStream;
+import java.io.OutputStreamWriter;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+import java.io.FileNotFoundException;
+import java.io.StringReader;
+import java.io.StringWriter;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.ArrayList;
 import java.util.Properties;
 import java.io.ByteArrayOutputStream;
+import java.io.ByteArrayInputStream;
 import java.io.PrintStream;
 
 import jline.ConsoleReader;
+import jline.ConsoleReaderInputStream;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -47,6 +60,7 @@
 import org.apache.pig.tools.pigscript.parser.ParseException;
 import org.apache.pig.tools.pigscript.parser.PigScriptParser;
 import org.apache.pig.tools.pigscript.parser.PigScriptParserTokenManager;
+import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
 
 public class GruntParser extends PigScriptParser {
 
@@ -80,15 +94,16 @@
     {
         prompt();
         mDone = false;
-        while(!mDone)
+        while(!mDone) {
             parse();
+        }
     }
 
     public void parseContOnError()
     {
         prompt();
         mDone = false;
-        while(!mDone)
+        while(!mDone) {
             try
             {
                 parse();
@@ -109,6 +124,7 @@
             } catch (Error e) {
                 log.error(e);
             }
+        }
     }
 
     public void setParams(PigServer pigServer)
@@ -161,6 +177,69 @@
         mPigServer.registerJar(jar);
     }
 
+    private String runPreprocessor(String script, ArrayList<String> params, 
+                                   ArrayList<String> files) 
+        throws IOException, ParseException {
+
+        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
+        StringWriter writer = new StringWriter();
+
+        try{
+            psp.genSubstitutedFile(new BufferedReader(new FileReader(script)), 
+                                   writer,  
+                                   params.size() > 0 ? params.toArray(new String[0]) :
null, 
+                                   files.size() > 0 ? files.toArray(new String[0]) : null);
+        } catch (org.apache.pig.tools.parameters.ParseException pex) {
+            throw new ParseException(pex.getMessage());
+        }
+
+        return writer.toString();
+    }
+
+    protected void processScript(String script, boolean batch, 
+                                 ArrayList<String> params, ArrayList<String>
files) 
+        throws IOException, ParseException {
+        
+        Reader inputReader;
+        ConsoleReader reader;
+        boolean interactive;
+         
+        try {
+            String cmds = runPreprocessor(script, params, files);
+
+            if (mInteractive && !batch) { // Write prompt and echo commands
+                reader = new ConsoleReader(new ByteArrayInputStream(cmds.getBytes()),
+                                           new OutputStreamWriter(System.out));
+                reader.setHistory(mConsoleReader.getHistory());
+                InputStream in = new ConsoleReaderInputStream(reader);
+                inputReader = new BufferedReader(new InputStreamReader(in));
+                interactive = true;
+            } else { // Quietly parse the statements
+                inputReader = new StringReader(cmds);
+                reader = null;
+                interactive = false;
+            }
+        } catch (FileNotFoundException fnfe) {
+            throw new ParseException("File not found: " + script);
+        } catch (SecurityException se) {
+            throw new ParseException("Cannot access file: " + script);
+        }
+
+        // In batch mode: Use a new server to avoid side-effects (handles, etc)
+        PigServer pigServer = batch ? 
+            new PigServer(mPigServer.getPigContext(), false) : mPigServer;
+            
+        GruntParser parser = new GruntParser(inputReader);
+        parser.setParams(pigServer);
+        parser.setConsoleReader(reader);
+        parser.setInteractive(interactive);
+        
+        parser.parseStopOnError();
+        if (interactive) {
+            System.out.println("");
+        }
+    }
+
     protected void processSet(String key, String value) throws IOException, ParseException
{
         if (key.equals("debug"))
         {

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Thu Feb
12 22:27:46 2009
@@ -31,6 +31,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Stack;
+import java.util.ArrayList;
 
 import jline.ConsoleReader;
 
@@ -62,7 +63,7 @@
 	
 	abstract protected void processDescribe(String alias) throws IOException;
 
-    abstract protected void processExplain(String alias) throws IOException;
+	abstract protected void processExplain(String alias) throws IOException;
 	
 	abstract protected void processRegister(String jar) throws IOException;
 
@@ -98,7 +99,9 @@
 	
 	abstract protected void processIllustrate(String alias) throws IOException;
 
-	 static String unquote(String s)
+	abstract protected void processScript(String script, boolean batch, ArrayList<String>
params, ArrayList<String> files) throws IOException, ParseException;
+
+	static String unquote(String s)
 	{
 		if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
 			return s.substring(1, s.length()-1);
@@ -142,6 +145,10 @@
 TOKEN: {<REMOVEFORCE: "rmf">}
 TOKEN: {<SET: "set">}
 TOKEN: {<ILLUSTRATE: "illustrate">}
+TOKEN: {<RUN: "run">}
+TOKEN: {<EXEC: "exec">}
+TOKEN: {<PARAM: "-param">}
+TOKEN: {<PARAM_FILE: "-param_file">}
 
 // internal use commands
 TOKEN: {<SCRIPT_DONE: "scriptDone">}
@@ -320,8 +327,8 @@
 |	<#DIGIT : ["0"-"9"] >
 |	<#SPECIALCHAR : ["_"] >
 |	<#FSSPECIALCHAR: ["/"]>
-|       <#FLOAT: <INTEGER> ( "." <INTEGER> )? | "." <INTEGER> >
-|       <#INTEGER: ( <DIGIT> )+ >
+|	<#FLOAT: <INTEGER> ( "." <INTEGER> )? | "." <INTEGER> >
+|	<#INTEGER: ( <DIGIT> )+ >
 |	<#NUMBER: <INTEGER> | <FLOAT> | <FLOAT> ( ["e","E"] ([ "-","+"])?
<FLOAT> )?>
 }
 
@@ -433,6 +440,8 @@
 	t1 = GetPath()
 	{processRegister(t1.image);}
 	|
+	Script()
+	|
 	<REMOVE>
 	(
 		t1 = GetPath()
@@ -467,6 +476,40 @@
 	)
 }
 
+void Script() throws IOException:
+{
+    Token t;
+    String script;
+    boolean batch = false;
+    ArrayList<String> params;
+    ArrayList<String> files;
+}
+{
+	(
+ 		<RUN>
+		{batch = false;}
+	|
+		<EXEC>
+		{batch = true;}
+	)
+    	{
+		params = new ArrayList<String>(); 
+		files = new ArrayList<String>();
+	}
+	(
+		<PARAM>
+		t = GetPath()
+		{params.add(t.image);}
+	|
+		<PARAM_FILE>
+		t = GetPath()
+		{files.add(t.image);}
+	)*
+	t = GetPath()
+	{script = t.image;}
+	{processScript(script, batch, params, files);}
+}
+
 Token GetPath() :
 {
 	Token t;
@@ -475,7 +518,7 @@
 	(
 	t = <IDENTIFIER>
 	|
-        t = <PATH>
+	t = <PATH>
 	|
 	t = GetReserved()
 	)
@@ -548,6 +591,14 @@
 	t = <SET>
 	|
 	t = <SCRIPT_DONE>
+	|
+	t = <RUN>
+	|
+	t = <EXEC>
+	|
+	t = <PARAM>
+	|
+	t = <PARAM_FILE>
 	)
 
 	{return t;}

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Thu Feb 12 22:27:46 2009
@@ -36,8 +36,14 @@
 
 public class TestGrunt extends TestCase {
     MiniCluster cluster = MiniCluster.buildCluster();
+    private String basedir;
 
     private final Log log = LogFactory.getLog(getClass());
+
+    public TestGrunt(String name) {
+        super(name);
+        basedir = "test/org/apache/pig/test/data";
+    }
     
 /*    @Test 
     public void testCopyFromLocal() throws Throwable {
@@ -305,4 +311,84 @@
     
         grunt.exec();
     }
+
+    @Test
+    public void testRunStatment() throws Throwable {
+        PigServer server = new PigServer("MAPREDUCE");
+        PigContext context = server.getPigContext();
+        
+        String strCmd = "a = load 'foo' as (foo, fast, regenerate);" +
+                        " run -param LIMIT=5 -param_file " + basedir +
+                        "/test_broken.ppf " + basedir + "/testsub.pig; explain bar";
+        
+        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+        InputStreamReader reader = new InputStreamReader(cmd);
+        
+        Grunt grunt = new Grunt(new BufferedReader(reader), context);
+    
+        grunt.exec();
+    }
+
+    @Test
+    public void testExecStatment() throws Throwable {
+        PigServer server = new PigServer("MAPREDUCE");
+        PigContext context = server.getPigContext();
+        boolean caught = false;
+        
+        String strCmd = "a = load 'foo' as (foo, fast, regenerate);" +
+                        " exec -param LIMIT=5 -param FUNCTION=COUNT " +
+                        "-param FILE=foo " + basedir + "/testsub.pig; explain bar";
+        
+        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+        InputStreamReader reader = new InputStreamReader(cmd);
+        
+        Grunt grunt = new Grunt(new BufferedReader(reader), context);
+        
+        try {
+            grunt.exec();
+        } catch (Exception e) {
+            caught = true;
+            assertTrue(e.getMessage().contains("alias bar"));
+        }
+        assertTrue(caught);
+    }
+
+    @Test
+    public void testRunStatmentNested() throws Throwable {
+        PigServer server = new PigServer("MAPREDUCE");
+        PigContext context = server.getPigContext();
+        
+        String strCmd = "a = load 'foo' as (foo, fast, regenerate); run "
+            +basedir+"/testsubnested_run.pig; explain bar";
+        
+        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+        InputStreamReader reader = new InputStreamReader(cmd);
+        
+        Grunt grunt = new Grunt(new BufferedReader(reader), context);
+    
+        grunt.exec();
+    }
+
+    @Test
+    public void testExecStatmentNested() throws Throwable {
+        PigServer server = new PigServer("MAPREDUCE");
+        PigContext context = server.getPigContext();
+        boolean caught = false;
+        
+        String strCmd = "a = load 'foo' as (foo, fast, regenerate); exec "
+            +basedir+"/testsubnested_exec.pig; explain bar";
+        
+        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+        InputStreamReader reader = new InputStreamReader(cmd);
+        
+        Grunt grunt = new Grunt(new BufferedReader(reader), context);
+        
+        try {
+            grunt.exec();
+        } catch (Exception e) {
+            caught = true;
+            assertTrue(e.getMessage().contains("alias bar"));
+        }
+        assertTrue(caught);
+    }
 }

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf Thu Feb 12 22:27:46 2009
@@ -0,0 +1,3 @@
+LIMIT = 5
+FILE = foo
+FUNCTION = COUNT

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf Thu Feb 12 22:27:46 2009
@@ -0,0 +1,3 @@
+LIMIT = WONTWORK
+FILE = foo
+FUNCTION = COUNT

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig Thu Feb 12 22:27:46 2009
@@ -0,0 +1,4 @@
+foo = load '$FILE' as (foo, fast, regenerate);
+bar = limit foo $LIMIT;
+baz = foreach bar generate $FUNCTION($0);
+explain baz;

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig Thu Feb 12 22:27:46
2009
@@ -0,0 +1 @@
+exec -param_file test/org/apache/pig/test/data/test.ppf test/org/apache/pig/test/data/testsub.pig;

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig Thu Feb 12 22:27:46
2009
@@ -0,0 +1 @@
+run -param LIMIT=5 -param FILE=foo -param FUNCTION=COUNT test/org/apache/pig/test/data/testsub.pig;



Mime
View raw message