pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roh...@apache.org
Subject svn commit: r1730045 - in /pig/trunk: src/org/apache/pig/backend/hadoop/executionengine/tez/plan/ src/org/apache/pig/tools/pigstats/tez/ test/org/apache/pig/test/data/GoldenFiles/tez/ test/org/apache/pig/tez/
Date Fri, 12 Feb 2016 15:38:36 GMT
Author: rohini
Date: Fri Feb 12 15:38:35 2016
New Revision: 1730045

URL: http://svn.apache.org/viewvc?rev=1730045&view=rev
Log:
Fix TestMultiQueryLocal.testStoreOrder test failure for PIG-4759

Added:
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld
Modified:
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java
    pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld
    pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java
Fri Feb 12 15:38:35 2016
@@ -31,6 +31,7 @@ import java.util.Set;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager;
 import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
 import org.apache.pig.impl.plan.OperatorKey;
 import org.apache.pig.impl.plan.OperatorPlan;
 import org.apache.pig.impl.plan.PlanException;
@@ -152,38 +153,109 @@ public class TezPlanContainer extends Op
     }
 
     public void split(TezPlanContainerNode planNode) throws PlanException {
+
         TezOperPlan tezOperPlan = planNode.getTezOperPlan();
+        if (tezOperPlan.size() == 1) {
+            // Nothing to split. Only one operator in the plan
+            return;
+        }
+
         TezOperator operToSegment = null;
         List<TezOperator> succs = new ArrayList<TezOperator>();
-        for (TezOperator tezOper : tezOperPlan) {
-            if (tezOper.needSegmentBelow() && tezOperPlan.getSuccessors(tezOper)!=null)
{
-                operToSegment = tezOper;
-                succs.addAll(tezOperPlan.getSuccessors(tezOper));
-                break;
-            }
+        try {
+            // Split top down from root to leaves
+            SegmentOperatorFinder finder = new SegmentOperatorFinder(tezOperPlan);
+            finder.visit();
+            operToSegment = finder.getOperatorToSegment();
+        } catch (VisitorException e) {
+            throw new PlanException(e);
         }
-        if (operToSegment != null) {
+
+        if (operToSegment != null && tezOperPlan.getSuccessors(operToSegment) !=
null) {
+            succs.addAll(tezOperPlan.getSuccessors(operToSegment));
             for (TezOperator succ : succs) {
                 tezOperPlan.disconnect(operToSegment, succ);
-                TezOperPlan newOperPlan = new TezOperPlan();
-                List<TezPlanContainerNode> containerSuccs = new ArrayList<TezPlanContainerNode>();
-                if (getSuccessors(planNode)!=null) {
-                    containerSuccs.addAll(getSuccessors(planNode));
-                }
-                tezOperPlan.moveTree(succ, newOperPlan);
-                TezPlanContainerNode newPlanNode = new TezPlanContainerNode(generateNodeOperatorKey(),
newOperPlan);
-                add(newPlanNode);
-                for (TezPlanContainerNode containerNodeSucc : containerSuccs) {
-                    disconnect(planNode, containerNodeSucc);
-                    connect(newPlanNode, containerNodeSucc);
+            }
+            for (TezOperator succ : succs) {
+                try {
+                    if (tezOperPlan.getOperator(succ.getOperatorKey()) == null) {
+                        // Has already been moved to a new plan by previous successor
+                        // as part of dependency. It could have been further split.
+                        // So walk the full plan to find the new plan and connect
+                        TezOperatorFinder finder = new TezOperatorFinder(this, succ);
+                        finder.visit();
+                        connect(planNode, finder.getPlanContainerNode());
+                        continue;
+                    }
+                    TezOperPlan newOperPlan = new TezOperPlan();
+                    tezOperPlan.moveTree(succ, newOperPlan);
+                    TezPlanContainerNode newPlanNode = new TezPlanContainerNode(
+                            generateNodeOperatorKey(), newOperPlan);
+                    add(newPlanNode);
+                    connect(planNode, newPlanNode);
+                    split(newPlanNode);
+                    if (newPlanNode.getTezOperPlan().getOperator(succ.getOperatorKey()) ==
null) {
+                        // On further split, the successor moved to a new plan container.
+                        // Connect to that
+                        TezOperatorFinder finder = new TezOperatorFinder(this, succ);
+                        finder.visit();
+                        disconnect(planNode, newPlanNode);
+                        connect(planNode, finder.getPlanContainerNode());
+                    }
+                } catch (VisitorException e) {
+                    throw new PlanException(e);
                 }
-                connect(planNode, newPlanNode);
-                split(newPlanNode);
             }
             split(planNode);
         }
     }
 
+    private static class SegmentOperatorFinder extends TezOpPlanVisitor {
+
+        private TezOperator operToSegment;
+
+        public SegmentOperatorFinder(TezOperPlan plan) {
+            super(plan, new DependencyOrderWalker<TezOperator, TezOperPlan>(plan));
+        }
+
+        public TezOperator getOperatorToSegment() {
+            return operToSegment;
+        }
+
+        @Override
+        public void visitTezOp(TezOperator tezOperator) throws VisitorException {
+            if (tezOperator.needSegmentBelow() && operToSegment == null) {
+                operToSegment = tezOperator;
+            }
+        }
+
+    }
+
+    private static class TezOperatorFinder extends TezPlanContainerVisitor {
+
+        private TezPlanContainerNode planContainerNode;
+        private TezOperator operatorToFind;
+
+        public TezOperatorFinder(TezPlanContainer plan, TezOperator operatorToFind) {
+            super(plan, new DependencyOrderWalker<TezPlanContainerNode, TezPlanContainer>(plan));
+            this.operatorToFind = operatorToFind;
+        }
+
+        public TezPlanContainerNode getPlanContainerNode() {
+            return planContainerNode;
+        }
+
+        @Override
+        public void visitTezPlanContainerNode(
+                TezPlanContainerNode tezPlanContainerNode)
+                throws VisitorException {
+            if (tezPlanContainerNode.getTezOperPlan().getOperatorKey(operatorToFind) != null)
{
+                planContainerNode = tezPlanContainerNode;
+            }
+        }
+
+    }
+
     private synchronized OperatorKey generateNodeOperatorKey() {
         OperatorKey opKey = new OperatorKey(jobName + "-" + dagId + "_scope", scopeId);
         scopeId++;

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java
Fri Feb 12 15:38:35 2016
@@ -19,7 +19,8 @@ package org.apache.pig.backend.hadoop.ex
 
 import java.io.PrintStream;
 
-import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPrinter.TezGraphPrinter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPrinter.TezDAGGraphPrinter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPrinter.TezVertexGraphPrinter;
 import org.apache.pig.impl.plan.DependencyOrderWalker;
 import org.apache.pig.impl.plan.VisitorException;
 
@@ -37,6 +38,19 @@ public class TezPlanContainerPrinter ext
         mStream.println("#--------------------------------------------------");
         mStream.println("# There are " + planContainer.size() + " DAGs in the session");
         mStream.println("#--------------------------------------------------");
+        printContainerPlan(planContainer);
+    }
+
+    private void printContainerPlan(TezPlanContainer planContainer) {
+        try {
+            if (planContainer.size() > 1) {
+                TezDAGGraphPrinter graphPrinter = new TezDAGGraphPrinter(planContainer);
+                graphPrinter.visit();
+                mStream.print(graphPrinter.toString());
+            }
+        } catch (VisitorException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     public void setVerbose(boolean verbose) {
@@ -48,7 +62,7 @@ public class TezPlanContainerPrinter ext
         mStream.println("#--------------------------------------------------");
         mStream.println("# TEZ DAG plan: " + tezPlanContainerNode.getOperatorKey());
         mStream.println("#--------------------------------------------------");
-        TezGraphPrinter graphPrinter = new TezGraphPrinter(tezPlanContainerNode.getTezOperPlan());
+        TezVertexGraphPrinter graphPrinter = new TezVertexGraphPrinter(tezPlanContainerNode.getTezOperPlan());
         graphPrinter.visit();
         mStream.print(graphPrinter.toString());
         TezPrinter printer = new TezPrinter(mStream, tezPlanContainerNode.getTezOperPlan());

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java Fri
Feb 12 15:38:35 2016
@@ -99,11 +99,11 @@ public class TezPrinter extends TezOpPla
     /**
      * This class prints the Tez Vertex Graph
      */
-    public static class TezGraphPrinter extends TezOpPlanVisitor {
+    public static class TezVertexGraphPrinter extends TezOpPlanVisitor {
 
         StringBuilder buf;
 
-        public TezGraphPrinter(TezOperPlan plan) {
+        public TezVertexGraphPrinter(TezOperPlan plan) {
             super(plan, new DependencyOrderWalker<TezOperator, TezOperPlan>(plan, true));
             buf = new StringBuilder();
         }
@@ -132,6 +132,43 @@ public class TezPrinter extends TezOpPla
                 }
             }
             buf.append("\n");
+        }
+
+        @Override
+        public String toString() {
+            buf.append("\n");
+            return buf.toString();
+        }
+    }
+
+    /**
+     * This class prints the Tez DAG Graph
+     */
+    public static class TezDAGGraphPrinter extends TezPlanContainerVisitor {
+
+        StringBuilder buf;
+
+        public TezDAGGraphPrinter(TezPlanContainer plan) {
+            super(plan, new DependencyOrderWalker<TezPlanContainerNode, TezPlanContainer>(plan,
true));
+            buf = new StringBuilder();
+        }
+
+        @Override
+        public void visitTezPlanContainerNode(TezPlanContainerNode tezPlanContainerNode)
throws VisitorException {
+            writePlan(mPlan, tezPlanContainerNode, buf);
+        }
+
+        public static void writePlan(TezPlanContainer mPlan, TezPlanContainerNode tezPlanContainerNode,
StringBuilder buf) {
+            buf.append("Tez DAG " + tezPlanContainerNode.getOperatorKey().toString());
+            List<TezPlanContainerNode> succs = mPlan.getSuccessors(tezPlanContainerNode);
+            if (succs != null) {
+                Collections.sort(succs);
+                buf.append("\t->\t");
+                for (TezPlanContainerNode op : succs) {
+                    buf.append("Tez DAG " + op.getOperatorKey().toString()).append(",");
+                }
+            }
+            buf.append("\n");
         }
 
         @Override

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java Fri Feb 12 15:38:35 2016
@@ -139,7 +139,7 @@ public class TezDAGStats extends JobStat
 
         @Override
         public void visitTezOp(TezOperator tezOp) throws VisitorException {
-            TezPrinter.TezGraphPrinter.writePlan(mPlan, tezOp, tezDAGPlan);
+            TezPrinter.TezVertexGraphPrinter.writePlan(mPlan, tezOp, tezDAGPlan);
             TezVertexStats currStats =
                     new TezVertexStats(tezOp.getOperatorKey().toString(), jobPlan, tezOp.isUseMRMapSettings());
             jobPlan.add(currStats);

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld Fri Feb 12
15:38:35 2016
@@ -1,6 +1,9 @@
 #--------------------------------------------------
 # There are 2 DAGs in the session
 #--------------------------------------------------
+Tez DAG pig-0_scope-0	->	Tez DAG pig-1_scope-1,
+Tez DAG pig-1_scope-1
+
 #--------------------------------------------------
 # TEZ DAG plan: pig-0_scope-0
 #--------------------------------------------------

Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld?rev=1730045&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld Fri
Feb 12 15:38:35 2016
@@ -0,0 +1,83 @@
+#--------------------------------------------------
+# There are 5 DAGs in the session
+#--------------------------------------------------
+Tez DAG pig-0_scope-0	->	Tez DAG pig-1_scope-1,Tez DAG pig-2_scope-2,Tez DAG pig-3_scope-3,
+Tez DAG pig-2_scope-2	->	Tez DAG pig-3_scope-3,Tez DAG pig-4_scope-4,
+Tez DAG pig-1_scope-1	->	Tez DAG pig-3_scope-3,
+Tez DAG pig-3_scope-3
+Tez DAG pig-4_scope-4
+
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-20
+
+Tez vertex scope-20
+# Plan on vertex
+a: Store(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-1
+|
+|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+#--------------------------------------------------
+# TEZ DAG plan: pig-2_scope-2
+#--------------------------------------------------
+Tez vertex scope-21
+
+Tez vertex scope-21
+# Plan on vertex
+a: Store(file:///tmp/output/Dir2:BinStorage) - scope-3
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-2
+#--------------------------------------------------
+# TEZ DAG plan: pig-1_scope-1
+#--------------------------------------------------
+Tez vertex scope-23
+
+Tez vertex scope-23
+# Plan on vertex
+a: Store(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-7
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-6
+#--------------------------------------------------
+# TEZ DAG plan: pig-3_scope-3
+#--------------------------------------------------
+Tez vertex scope-24	->	Tez vertex scope-27,
+Tez vertex scope-26	->	Tez vertex scope-27,
+Tez vertex scope-25	->	Tez vertex scope-27,
+Tez vertex scope-27
+
+Tez vertex scope-24
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-13	->	 scope-27
+|   |
+|   Project[bytearray][0] - scope-14
+|
+|---a: Load(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-26
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-17	->	 scope-27
+|   |
+|   Project[bytearray][0] - scope-18
+|
+|---c: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-10
+Tez vertex scope-25
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-15	->	 scope-27
+|   |
+|   Project[bytearray][0] - scope-16
+|
+|---b: Load(file:///tmp/output/Dir2:BinStorage) - scope-9
+Tez vertex scope-27
+# Plan on vertex
+d: Store(file:///tmp/output/Dir5:org.apache.pig.builtin.PigStorage) - scope-19
+|
+|---d: Package(Packager)[tuple]{bytearray} - scope-12
+#--------------------------------------------------
+# TEZ DAG plan: pig-4_scope-4
+#--------------------------------------------------
+Tez vertex scope-22
+
+Tez vertex scope-22
+# Plan on vertex
+a: Store(file:///tmp/output/Dir4:org.apache.pig.builtin.PigStorage) - scope-5
+|
+|---a: Load(file:///tmp/output/Dir2:BinStorage) - scope-4

Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld?rev=1730045&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld Fri Feb 12
15:38:35 2016
@@ -0,0 +1,83 @@
+#--------------------------------------------------
+# There are 5 DAGs in the session
+#--------------------------------------------------
+Tez DAG pig-0_scope-0	->	Tez DAG pig-1_scope-1,Tez DAG pig-3_scope-3,Tez DAG pig-4_scope-4,
+Tez DAG pig-1_scope-1	->	Tez DAG pig-2_scope-2,Tez DAG pig-4_scope-4,
+Tez DAG pig-2_scope-2
+Tez DAG pig-3_scope-3	->	Tez DAG pig-4_scope-4,
+Tez DAG pig-4_scope-4
+
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-20
+
+Tez vertex scope-20
+# Plan on vertex
+a: Store(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-1
+|
+|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+#--------------------------------------------------
+# TEZ DAG plan: pig-1_scope-1
+#--------------------------------------------------
+Tez vertex scope-21
+
+Tez vertex scope-21
+# Plan on vertex
+a: Store(file:///tmp/output/Dir2:BinStorage) - scope-3
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-2
+#--------------------------------------------------
+# TEZ DAG plan: pig-2_scope-2
+#--------------------------------------------------
+Tez vertex scope-22
+
+Tez vertex scope-22
+# Plan on vertex
+a: Store(file:///tmp/output/Dir4:org.apache.pig.builtin.PigStorage) - scope-5
+|
+|---a: Load(file:///tmp/output/Dir2:BinStorage) - scope-4
+#--------------------------------------------------
+# TEZ DAG plan: pig-3_scope-3
+#--------------------------------------------------
+Tez vertex scope-23
+
+Tez vertex scope-23
+# Plan on vertex
+a: Store(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-7
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-6
+#--------------------------------------------------
+# TEZ DAG plan: pig-4_scope-4
+#--------------------------------------------------
+Tez vertex scope-26	->	Tez vertex scope-27,
+Tez vertex scope-25	->	Tez vertex scope-27,
+Tez vertex scope-24	->	Tez vertex scope-27,
+Tez vertex scope-27
+
+Tez vertex scope-26
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-17	->	 scope-27
+|   |
+|   Project[bytearray][0] - scope-18
+|
+|---c: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-10
+Tez vertex scope-25
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-15	->	 scope-27
+|   |
+|   Project[bytearray][0] - scope-16
+|
+|---b: Load(file:///tmp/output/Dir2:BinStorage) - scope-9
+Tez vertex scope-24
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-13	->	 scope-27
+|   |
+|   Project[bytearray][0] - scope-14
+|
+|---a: Load(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-27
+# Plan on vertex
+d: Store(file:///tmp/output/Dir5:org.apache.pig.builtin.PigStorage) - scope-19
+|
+|---d: Package(Packager)[tuple]{bytearray} - scope-12

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld Fri Feb 12 15:38:35
2016
@@ -1,6 +1,10 @@
 #--------------------------------------------------
 # There are 3 DAGs in the session
 #--------------------------------------------------
+Tez DAG pig-0_scope-0	->	Tez DAG pig-1_scope-1,
+Tez DAG pig-1_scope-1	->	Tez DAG pig-2_scope-2,
+Tez DAG pig-2_scope-2
+
 #--------------------------------------------------
 # TEZ DAG plan: pig-0_scope-0
 #--------------------------------------------------

Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Fri Feb 12 15:38:35 2016
@@ -100,6 +100,31 @@ public class TestTezCompiler {
     }
 
     @Test
+    public void testStoreLoadMultiple() throws Exception {
+        String query =
+                "a = load 'file:///tmp/input';" +
+                "store a into 'file:///tmp/output/Dir1';" +
+                "a = load 'file:///tmp/output/Dir1';" +
+                "store a into 'file:///tmp/output/Dir2' using BinStorage();" +
+                "a = load 'file:///tmp/output/Dir1';" +
+                "store a into 'file:///tmp/output/Dir3';" +
+                "a = load 'file:///tmp/output/Dir2' using BinStorage();" +
+                "store a into 'file:///tmp/output/Dir4';" +
+                "a = load 'file:///tmp/output/Dir3';" +
+                "b = load 'file:///tmp/output/Dir2' using BinStorage();" +
+                "c = load 'file:///tmp/output/Dir1';" +
+                "d = cogroup a by $0, b by $0, c by $0;" +
+                "store d into 'file:///tmp/output/Dir5';";
+
+        // To get around difference in ordering of operators in plan due to JDK7 and JDK8
+        if (System.getProperties().getProperty("java.version").startsWith("1.8")) {
+            run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld");
+        } else {
+            run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld");
+        }
+    }
+
+    @Test
     public void testNative() throws Exception {
         String query =
                 "a = load 'file:///tmp/input' as (x:int, y:int);" +



Mime
View raw message