pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pradeep...@apache.org
Subject svn commit: r940995 - in /hadoop/pig/trunk: CHANGES.txt src/org/apache/pig/tools/grunt/GruntParser.java test/org/apache/pig/test/TestGrunt.java test/org/apache/pig/test/data/explainScript.pig
Date Tue, 04 May 2010 18:42:04 GMT
Author: pradeepkth
Date: Tue May  4 18:42:04 2010
New Revision: 940995

URL: http://svn.apache.org/viewvc?rev=940995&view=rev
Log:
"explain -script <script file>" executes grunt commands like run/dump/copy etc - explain
-script should not execute any grunt command and only explain the query plans (pradeepkth)

Added:
    hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig
Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=940995&r1=940994&r2=940995&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Tue May  4 18:42:04 2010
@@ -53,6 +53,10 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-1401: "explain -script <script file>" executes grunt commands like
+run/dump/copy etc - explain -script should not execute any grunt command and
+only explain the query plans (pradeepkth)
+
 PIG-1303: Inconsistent instantiation of parametrized UDFs (jrussek and dvryaboy)
 
 740 : Incorrect line number is generated when a string with double quotes is

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=940995&r1=940994&r2=940995&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Tue May  4 18:42:04 2010
@@ -224,10 +224,14 @@ public class GruntParser extends PigScri
     
     @Override
     protected void processDescribe(String alias) throws IOException {
-        if(alias==null) {
-            alias = mPigServer.getPigContext().getLastAlias();
+        if(mExplain == null) { // process only if not in "explain" mode
+            if(alias==null) {
+                alias = mPigServer.getPigContext().getLastAlias();
+            }
+            mPigServer.dumpSchema(alias);
+        } else {
+            log.warn("'describe' statement is ignored while processing explain");
         }
-        mPigServer.dumpSchema(alias);
     }
 
     @Override
@@ -317,7 +321,11 @@ public class GruntParser extends PigScri
 
     @Override
     protected void printAliases() throws IOException {
-        mPigServer.printAliases();
+        if(mExplain == null) { // process only if not in "explain" mode
+            mPigServer.printAliases();
+        } else {
+            log.warn("'aliases' statement is ignored while processing explain");
+        }
     }
     
     @Override
@@ -349,22 +357,26 @@ public class GruntParser extends PigScri
                                  List<String> params, List<String> files) 
         throws IOException, ParseException {
         
-        if (script == null) {
-            executeBatch();
-            return;
-        }
-        
-        if (batch) {
-            setBatchOn();
-            mPigServer.setJobName(script);
-            try {
-                loadScript(script, true, mLoadOnly, params, files);
+        if(mExplain == null) { // process only if not in "explain" mode
+            if (script == null) {
                 executeBatch();
-            } finally {
-                discardBatch();
+                return;
+            }
+            
+            if (batch) {
+                setBatchOn();
+                mPigServer.setJobName(script);
+                try {
+                    loadScript(script, true, mLoadOnly, params, files);
+                    executeBatch();
+                } finally {
+                    discardBatch();
+                }
+            } else {
+                loadScript(script, false, mLoadOnly, params, files);
             }
         } else {
-            loadScript(script, false, mLoadOnly, params, files);
+            log.warn("'run/exec' statement is ignored while processing explain");
         }
     }
 
@@ -465,44 +477,49 @@ public class GruntParser extends PigScri
     @Override
     protected void processCat(String path) throws IOException
     {
-        executeBatch();
-
-        try {
-            byte buffer[] = new byte[65536];
-            ElementDescriptor dfsPath = mDfs.asElement(path);
-            int rc;
+        if(mExplain == null) { // process only if not in "explain" mode
             
-            if (!dfsPath.exists())
-                throw new IOException("Directory " + path + " does not exist.");
-    
-            if (mDfs.isContainer(path)) {
-                ContainerDescriptor dfsDir = (ContainerDescriptor) dfsPath;
-                Iterator<ElementDescriptor> paths = dfsDir.iterator();
+            executeBatch();
+
+            try {
+                byte buffer[] = new byte[65536];
+                ElementDescriptor dfsPath = mDfs.asElement(path);
+                int rc;
                 
-                while (paths.hasNext()) {
-                    ElementDescriptor curElem = paths.next();
+                if (!dfsPath.exists())
+                    throw new IOException("Directory " + path + " does not exist.");
+        
+                if (mDfs.isContainer(path)) {
+                    ContainerDescriptor dfsDir = (ContainerDescriptor) dfsPath;
+                    Iterator<ElementDescriptor> paths = dfsDir.iterator();
                     
-                    if (mDfs.isContainer(curElem.toString())) {
-                        continue;
+                    while (paths.hasNext()) {
+                        ElementDescriptor curElem = paths.next();
+                        
+                        if (mDfs.isContainer(curElem.toString())) {
+                            continue;
+                        }
+                        
+                        InputStream is = curElem.open();
+                        while ((rc = is.read(buffer)) > 0) {
+                            System.out.write(buffer, 0, rc);
+                        }
+                        is.close();                
                     }
-                    
-                    InputStream is = curElem.open();
+                }
+                else {
+                    InputStream is = dfsPath.open();
                     while ((rc = is.read(buffer)) > 0) {
                         System.out.write(buffer, 0, rc);
                     }
-                    is.close();                
+                    is.close();            
                 }
             }
-            else {
-                InputStream is = dfsPath.open();
-                while ((rc = is.read(buffer)) > 0) {
-                    System.out.write(buffer, 0, rc);
-                }
-                is.close();            
+            catch (DataStorageException e) {
+                throw new IOException("Failed to Cat: " + path, e);
             }
-        }
-        catch (DataStorageException e) {
-            throw new IOException("Failed to Cat: " + path, e);
+        } else {
+            log.warn("'cat' statement is ignored while processing explain");
         }
     }
 
@@ -510,49 +527,60 @@ public class GruntParser extends PigScri
     protected void processCD(String path) throws IOException
     {    
         ContainerDescriptor container;
-
-        try {
-            if (path == null) {
-                container = mDfs.asContainer("/user/" + System.getProperty("user.name"));
-                mDfs.setActiveContainer(container);
-            }
-            else
-            {
-                container = mDfs.asContainer(path);
-    
-                if (!container.exists()) {
-                    throw new IOException("Directory " + path + " does not exist.");
+        if(mExplain == null) { // process only if not in "explain" mode
+            try {
+                if (path == null) {
+                    container = mDfs.asContainer("/user/" + System.getProperty("user.name"));
+                    mDfs.setActiveContainer(container);
                 }
-                
-                if (!mDfs.isContainer(path)) {
-                    throw new IOException(path + " is not a directory.");
+                else
+                {
+                    container = mDfs.asContainer(path);
+        
+                    if (!container.exists()) {
+                        throw new IOException("Directory " + path + " does not exist.");
+                    }
+                    
+                    if (!mDfs.isContainer(path)) {
+                        throw new IOException(path + " is not a directory.");
+                    }
+                    
+                    mDfs.setActiveContainer(container);
                 }
-                
-                mDfs.setActiveContainer(container);
             }
-        }
-        catch (DataStorageException e) {
-            throw new IOException("Failed to change working directory to " + 
-                                  ((path == null) ? ("/user/" + System.getProperty("user.name"))

-                                                     : (path)), e);
+            catch (DataStorageException e) {
+                throw new IOException("Failed to change working directory to " + 
+                                      ((path == null) ? ("/user/" + System.getProperty("user.name"))

+                                                         : (path)), e);
+            }
+        } else {
+            log.warn("'cd' statement is ignored while processing explain");
         }
     }
 
     @Override
     protected void processDump(String alias) throws IOException
     {
-        Iterator<Tuple> result = mPigServer.openIterator(alias);
-        while (result.hasNext())
-        {
-            Tuple t = result.next();
-            System.out.println(TupleFormat.format(t));
+        if(mExplain == null) { // process only if not in "explain" mode
+            Iterator<Tuple> result = mPigServer.openIterator(alias);
+            while (result.hasNext())
+            {
+                Tuple t = result.next();
+                System.out.println(TupleFormat.format(t));
+            }
+        } else {
+            log.warn("'dump' statement is ignored while processing explain");
         }
     }
     
     @Override
     protected void processIllustrate(String alias) throws IOException
     {
-	mPigServer.getExamples(alias);
+        if(mExplain == null) { // process only if not in "explain" mode
+            mPigServer.getExamples(alias);
+        } else {
+            log.warn("'illustrate' statement is ignored while processing explain");
+        }
     }
 
     @Override
@@ -575,39 +603,43 @@ public class GruntParser extends PigScri
     @Override
     protected void processLS(String path) throws IOException
     {
-        try {
-            ElementDescriptor pathDescriptor;
-            
-            if (path == null) {
-                pathDescriptor = mDfs.getActiveContainer();
-            }
-            else {
-                pathDescriptor = mDfs.asElement(path);
-            }
-
-            if (!pathDescriptor.exists()) {
-                throw new IOException("File or directory " + path + " does not exist.");
               
-            }
-            
-            if (mDfs.isContainer(pathDescriptor.toString())) {
-                ContainerDescriptor container = (ContainerDescriptor) pathDescriptor;
-                Iterator<ElementDescriptor> elems = container.iterator();
+        if(mExplain == null) { // process only if not in "explain" mode
+            try {
+                ElementDescriptor pathDescriptor;
+                
+                if (path == null) {
+                    pathDescriptor = mDfs.getActiveContainer();
+                }
+                else {
+                    pathDescriptor = mDfs.asElement(path);
+                }
+    
+                if (!pathDescriptor.exists()) {
+                    throw new IOException("File or directory " + path + " does not exist.");
               
+                }
                 
-                while (elems.hasNext()) {
-                    ElementDescriptor curElem = elems.next();
+                if (mDfs.isContainer(pathDescriptor.toString())) {
+                    ContainerDescriptor container = (ContainerDescriptor) pathDescriptor;
+                    Iterator<ElementDescriptor> elems = container.iterator();
                     
-                    if (mDfs.isContainer(curElem.toString())) {
-                           System.out.println(curElem.toString() + "\t<dir>");
-                    } else {
-                        printLengthAndReplication(curElem);
+                    while (elems.hasNext()) {
+                        ElementDescriptor curElem = elems.next();
+                        
+                        if (mDfs.isContainer(curElem.toString())) {
+                               System.out.println(curElem.toString() + "\t<dir>");
+                        } else {
+                            printLengthAndReplication(curElem);
+                        }
                     }
+                } else {
+                    printLengthAndReplication(pathDescriptor);
                 }
-            } else {
-                printLengthAndReplication(pathDescriptor);
             }
-        }
-        catch (DataStorageException e) {
-            throw new IOException("Failed to LS on " + path, e);
+            catch (DataStorageException e) {
+                throw new IOException("Failed to LS on " + path, e);
+            }
+        } else {
+            log.warn("'ls' statement is ignored while processing explain");
         }
     }
 
@@ -625,7 +657,11 @@ public class GruntParser extends PigScri
     @Override
     protected void processPWD() throws IOException 
     {
-        System.out.println(mDfs.getActiveContainer().toString());
+        if(mExplain == null) { // process only if not in "explain" mode
+            System.out.println(mDfs.getActiveContainer().toString());
+        } else {
+            log.warn("'pwd' statement is ignored while processing explain");
+        }
     }
 
     @Override
@@ -650,76 +686,100 @@ public class GruntParser extends PigScri
     @Override
     protected void processMove(String src, String dst) throws IOException
     {
-        executeBatch();
+        if(mExplain == null) { // process only if not in "explain" mode
 
-        try {
-            ElementDescriptor srcPath = mDfs.asElement(src);
-            ElementDescriptor dstPath = mDfs.asElement(dst);
-            
-            if (!srcPath.exists()) {
-                throw new IOException("File or directory " + src + " does not exist."); 
              
+            executeBatch();
+        
+            try {
+                ElementDescriptor srcPath = mDfs.asElement(src);
+                ElementDescriptor dstPath = mDfs.asElement(dst);
+                
+                if (!srcPath.exists()) {
+                    throw new IOException("File or directory " + src + " does not exist.");
               
+                }
+                
+                srcPath.rename(dstPath);
             }
-            
-            srcPath.rename(dstPath);
-        }
-        catch (DataStorageException e) {
-            throw new IOException("Failed to move " + src + " to " + dst, e);
+            catch (DataStorageException e) {
+                throw new IOException("Failed to move " + src + " to " + dst, e);
+            }
+        } else {
+            log.warn("'mv' statement is ignored while processing explain");
         }
     }
     
     @Override
     protected void processCopy(String src, String dst) throws IOException
     {
-        executeBatch();
+        if(mExplain == null) { // process only if not in "explain" mode
 
-        try {
-            ElementDescriptor srcPath = mDfs.asElement(src);
-            ElementDescriptor dstPath = mDfs.asElement(dst);
-            
-            srcPath.copy(dstPath, mConf, false);
-        }
-        catch (DataStorageException e) {
-            throw new IOException("Failed to copy " + src + " to " + dst, e);
+            executeBatch();
+        
+            try {
+                ElementDescriptor srcPath = mDfs.asElement(src);
+                ElementDescriptor dstPath = mDfs.asElement(dst);
+                
+                srcPath.copy(dstPath, mConf, false);
+            }
+            catch (DataStorageException e) {
+                throw new IOException("Failed to copy " + src + " to " + dst, e);
+            }
+        } else {
+            log.warn("'cp' statement is ignored while processing explain");
         }
     }
     
     @Override
     protected void processCopyToLocal(String src, String dst) throws IOException
     {
-        executeBatch();
-
-        try {
-            ElementDescriptor srcPath = mDfs.asElement(src);
-            ElementDescriptor dstPath = mLfs.asElement(dst);
+        if(mExplain == null) { // process only if not in "explain" mode
             
-            srcPath.copy(dstPath, false);
-        }
-        catch (DataStorageException e) {
-            throw new IOException("Failed to copy " + src + "to (locally) " + dst, e);
+            executeBatch();
+        
+            try {
+                ElementDescriptor srcPath = mDfs.asElement(src);
+                ElementDescriptor dstPath = mLfs.asElement(dst);
+                
+                srcPath.copy(dstPath, false);
+            }
+            catch (DataStorageException e) {
+                throw new IOException("Failed to copy " + src + "to (locally) " + dst, e);
+            }
+        } else {
+            log.warn("'copyToLocal' statement is ignored while processing explain");
         }
     }
 
     @Override
     protected void processCopyFromLocal(String src, String dst) throws IOException
     {
-        executeBatch();
-
-        try {
-            ElementDescriptor srcPath = mLfs.asElement(src);
-            ElementDescriptor dstPath = mDfs.asElement(dst);
+        if(mExplain == null) { // process only if not in "explain" mode
             
-            srcPath.copy(dstPath, false);
-        }
-        catch (DataStorageException e) {
-            throw new IOException("Failed to copy (loally) " + src + "to " + dst, e);
+            executeBatch();
+        
+            try {
+                ElementDescriptor srcPath = mLfs.asElement(src);
+                ElementDescriptor dstPath = mDfs.asElement(dst);
+                
+                srcPath.copy(dstPath, false);
+            }
+            catch (DataStorageException e) {
+                throw new IOException("Failed to copy (loally) " + src + "to " + dst, e);
+            }
+        } else {
+            log.warn("'copyFromLocal' statement is ignored while processing explain");
         }
     }
     
     @Override
     protected void processMkdir(String dir) throws IOException
     {
-        ContainerDescriptor dirDescriptor = mDfs.asContainer(dir);
-        dirDescriptor.create();
+        if(mExplain == null) { // process only if not in "explain" mode
+            ContainerDescriptor dirDescriptor = mDfs.asContainer(dir);
+            dirDescriptor.create();
+        } else {
+            log.warn("'mkdir' statement is ignored while processing explain");
+        }
     }
     
     @Override
@@ -741,27 +801,38 @@ public class GruntParser extends PigScri
     @Override
     protected void processRemove(String path, String options ) throws IOException
     {
-        ElementDescriptor dfsPath = mDfs.asElement(path);
+        if(mExplain == null) { // process only if not in "explain" mode
 
-        executeBatch();
+            ElementDescriptor dfsPath = mDfs.asElement(path);    
+            executeBatch();
         
-        if (!dfsPath.exists()) {
-            if (options == null || !options.equalsIgnoreCase("force")) {
-                throw new IOException("File or directory " + path + " does not exist.");

+            if (!dfsPath.exists()) {
+                if (options == null || !options.equalsIgnoreCase("force")) {
+                    throw new IOException("File or directory " + path + " does not exist.");

+                }
             }
-        }
-        else {
-            
-            dfsPath.delete();
+            else {
+                
+                dfsPath.delete();
+            }
+        } else {
+            log.warn("'rm/rmf' statement is ignored while processing explain");
         }
     }
 
     @Override
     protected void processFsCommand(String[] cmdTokens) throws IOException{
-        try {
-            shell.run(cmdTokens);
-        } catch (Exception e) {
-            throw new IOException(e);
+        if(mExplain == null) { // process only if not in "explain" mode
+            
+            executeBatch();
+            
+            try {
+                shell.run(cmdTokens);
+            } catch (Exception e) {
+                throw new IOException(e);
+            }
+        } else {
+            log.warn("'fs' statement is ignored while processing explain");
         }
     }
     

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=940995&r1=940994&r2=940995&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Tue May  4 18:42:04 2010
@@ -21,6 +21,9 @@ import org.junit.Test;
 import junit.framework.TestCase;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Appender;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.PatternLayout;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigException;
 import org.apache.pig.PigServer;
@@ -32,6 +35,8 @@ import org.apache.pig.tools.pigscript.pa
 import org.apache.pig.impl.util.LogUtils;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileReader;
 import java.io.InputStreamReader;
 import java.io.BufferedReader;
 import java.io.StringReader;
@@ -65,6 +70,7 @@ public class TestGrunt extends TestCase 
         grunt.exec();
     }
 */
+    
     @Test 
     public void testDefine() throws Throwable {
         PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
@@ -450,6 +456,54 @@ public class TestGrunt extends TestCase 
         grunt.exec();
     }
 
+    /**
+     * verify that grunt commands are ignored in explain -script mode
+     */
+    @Test
+    public void testExplainScript2() throws Throwable {
+        
+        PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+        PigContext context = server.getPigContext();
+        
+        String strCmd = "explain -script "
+                + basedir + "/explainScript.pig;";
+        
+        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+        InputStreamReader reader = new InputStreamReader(cmd);
+        
+        String logMessagesFile = "TestGrunt-testExplainScript2-stderr.txt";
+        // add a file based appender to the root logger so we can parse the 
+        // messages logged by grunt and verify that grunt commands are ignored
+        // in explain -script mode
+        Appender fileAppender = new FileAppender(new PatternLayout(), logMessagesFile);
+        
+        try {
+            org.apache.log4j.LogManager.getRootLogger().addAppender(fileAppender);
+            Grunt grunt = new Grunt(new BufferedReader(reader), context);
+            grunt.exec();
+            BufferedReader in = new BufferedReader(new FileReader(logMessagesFile));
+            String gruntLoggingContents = "";
+            //read file into a string
+            String line;
+            while ( (line = in.readLine()) != null) {
+                 gruntLoggingContents += line + "\n";
+            }
+            in.close();
+            String[] cmds = new String[] { "'rm/rmf'", "'cp'", "'cat'", "'cd'", "'pwd'",

+                    "'copyFromLocal'", "'copyToLocal'", "'describe'", "'ls'", 
+                    "'mkdir'", "'illustrate'", "'run/exec'", "'fs'", "'aliases'",
+                    "'mv'", "'dump'" };
+            for (String c : cmds) {
+                String expected = c + " statement is ignored while processing explain";
+                assertTrue("Checking if " + gruntLoggingContents + " contains " + 
+                        expected, gruntLoggingContents.contains(expected));
+            }
+        } finally {
+            org.apache.log4j.LogManager.getRootLogger().removeAppender(fileAppender);
+            new File(logMessagesFile).delete();
+        }
+    }
+
     @Test
     public void testExplainBrief() throws Throwable {
         PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());

Added: hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig?rev=940995&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/explainScript.pig Tue May  4 18:42:04 2010
@@ -0,0 +1 @@
+rmf input-copy.txt; cat 'foo'; a = load '1.txt' ; aliases;illustrate a; copyFromLocal foo
bar; copyToLocal foo bar; describe a; mkdir foo; run bar.pig; exec bar.pig; cp foo bar; explain
a;cd 'bar'; pwd; ls ; fs -ls ; fs -rmr foo; mv foo bar; dump a;store a into 'input-copy.txt'
; a = load '2.txt' as (b);explain a; rm foo; store a into 'bar';



Mime
View raw message