pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1790504 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/backend/hadoop/executionengine/tez/ test/org/apache/pig/test/
Date Fri, 07 Apr 2017 05:12:12 GMT
Author: daijy
Date: Fri Apr  7 05:12:12 2017
New Revision: 1790504

URL: http://svn.apache.org/viewvc?rev=1790504&view=rev
Log:
PIG-5210: Option to print MR/Tez plan before launching

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/PigConfiguration.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
    pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Apr  7 05:12:12 2017
@@ -36,6 +36,8 @@ PIG-5067: Revisit union on numeric type
  
 IMPROVEMENTS
 
+PIG-5210: Option to print MR/Tez plan before launching (ly16 via daijy)
+
 PIG-5175: Upgrade jruby to 1.7.26 (daijy)
 
 PIG-5110: Removing schema alias and :: coming from parent relation (szita via rohini)

Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Fri Apr  7 05:12:12 2017
@@ -508,6 +508,8 @@ public class PigConfiguration {
 
     public static final String PIG_STORE_SCHEMA_DISAMBIGUATE_DEFAULT = "true";
 
+    public static final String PIG_PRINT_EXEC_PLAN = "pig.print.exec.plan";
+
     // Deprecated settings of Pig 0.13
 
     /**

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Fri Apr  7 05:12:12 2017
@@ -321,6 +321,12 @@ public class JobControlCompiler{
                 if(mro instanceof NativeMapReduceOper) {
                     return null;
                 }
+
+                //Print MR plan before launching if needed
+                if (conf.getBoolean(PigConfiguration.PIG_PRINT_EXEC_PLAN, false)) {
+                    log.info(mro.toString());
+                }
+
                 Job job = getJob(plan, mro, conf, pigContext);
                 jobMroMap.put(job, mro);
                 jobCtrl.addJob(job);

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java Fri
Apr  7 05:12:12 2017
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.pig.PigConfiguration;
 import org.apache.pig.PigException;
 import org.apache.pig.backend.hadoop.PigATSClient;
 import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
@@ -110,6 +111,12 @@ public class TezJobCompiler {
             for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
{
                 log.info("Local resource: " + entry.getKey());
             }
+
+            // Print Tez plan before launching if needed
+            if (conf.getBoolean(PigConfiguration.PIG_PRINT_EXEC_PLAN, false)) {
+                log.info(tezPlanNode.getTezOperPlan());
+            }
+
             DAG tezDag = buildDAG(tezPlanNode, localResources);
             tezDag.setDAGInfo(createDagInfo(TezScriptState.get().getScript()));
             // set Tez caller context

Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java Fri Apr  7 05:12:12 2017
@@ -17,6 +17,8 @@
  */
 package org.apache.pig.test;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -25,6 +27,7 @@ import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -37,6 +40,12 @@ import java.util.StringTokenizer;
 import junit.framework.Assert;
 
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.log4j.Appender;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.SimpleLayout;
+import org.apache.log4j.WriterAppender;
 import org.apache.pig.ComparisonFunc;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.FuncSpec;
@@ -58,6 +67,7 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.impl.util.Pair;
 import org.apache.pig.impl.util.UDFContext;
 import org.apache.pig.impl.util.Utils;
+import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor;
 import org.apache.pig.test.utils.Identity;
 import org.junit.Assume;
 import org.junit.Before;
@@ -1251,7 +1261,17 @@ public class TestEvalPipelineLocal {
     @Test
     public void testBytesRawComparatorDesc() throws Exception{
         File f1 = createFile(new String[]{"2", "1", "4", "3"});
-        
+
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        Logger logger = Logger.getRootLogger();
+
+        logger.setLevel(Level.INFO);
+        SimpleLayout layout = new SimpleLayout();
+        Appender appender = new WriterAppender(layout, new PrintStream(bos));
+        logger.addAppender(appender);
+
+        // Also test PIG-5210 here in the same test
+        pigServer.getPigContext().getProperties().setProperty("pig.print.exec.plan", "true");
         pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
                 + "' as (value:long);");
         pigServer.registerQuery("b = foreach a generate " + TOTUPLENOINNERSCHEMA.class.getName()
+ "(value);");
@@ -1264,5 +1284,9 @@ public class TestEvalPipelineLocal {
         Assert.assertEquals(iter.next().toString(), "(2)");
         Assert.assertEquals(iter.next().toString(), "(1)");
         Assert.assertFalse(iter.hasNext());
+
+        logger.removeAppender(appender);
+
+        Assert.assertTrue(bos.toString().contains("New For Each(false,false)[tuple]"));
     }
 }



Mime
View raw message