pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From y...@apache.org
Subject svn commit: r1050503 [2/2] - in /pig/trunk: src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/ ...
Date Fri, 17 Dec 2010 22:08:09 GMT
Modified: pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java (original)
+++ pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java Fri Dec 17 22:08:08 2010
@@ -41,14 +41,14 @@ import org.apache.pig.impl.PigContext;
 import org.apache.pig.PigException;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
 import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.LOSort;
-import org.apache.pig.impl.logicalLayer.LOLimit;
-import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.logical.relational.LOSort;
+import org.apache.pig.newplan.logical.relational.LOLimit;
+import org.apache.pig.newplan.Operator;
 import org.apache.pig.pen.util.DisplayExamples;
 import org.apache.pig.pen.util.LineageTracer;
 
@@ -60,6 +60,7 @@ import org.apache.pig.pen.util.LineageTr
 public class ExampleGenerator {
 
     LogicalPlan plan;
+    org.apache.pig.newplan.logical.relational.LogicalPlan newPlan;
     Map<LOLoad, DataBag> baseData = null;
     PigContext pigContext;
 
@@ -72,16 +73,16 @@ public class ExampleGenerator {
 
     private int MAX_RECORDS = 10000;
     
-    private Map<LogicalOperator, PhysicalOperator> logToPhyMap;
-    private Map<PhysicalOperator, LogicalOperator> poLoadToLogMap;
-    private Map<PhysicalOperator, LogicalOperator> poToLogMap;
+    private Map<Operator, PhysicalOperator> logToPhyMap;
+    private Map<PhysicalOperator, Operator> poLoadToLogMap;
+    private Map<PhysicalOperator, Operator> poToLogMap;
     private HashMap<PhysicalOperator, Collection<IdentityHashSet<Tuple>>> poToEqclassesMap;
     private LineageTracer lineage;
-    private Map<LogicalOperator, DataBag> logToDataMap = null;
-    private Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap;
-    Map<LOForEach, Map<LogicalOperator, PhysicalOperator>> forEachInnerLogToPhyMap;
+    private Map<Operator, DataBag> logToDataMap = null;
+    private Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap;
+    Map<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> forEachInnerLogToPhyMap;
     Map<LOLimit, Long> oriLimitMap = null;
-    Map<POLoad, Schema> poLoadToSchemaMap;
+    Map<POLoad, LogicalSchema> poLoadToSchemaMap;
 
     public ExampleGenerator(LogicalPlan plan, PigContext hadoopPigContext) {
         this.plan = plan;
@@ -99,14 +100,14 @@ public class ExampleGenerator {
         }
         execEngine = new HExecutionEngine(pigContext);
         localMRRunner = new LocalMapReduceSimulator();
-        poLoadToSchemaMap = new HashMap<POLoad, Schema>();
+        poLoadToSchemaMap = new HashMap<POLoad, LogicalSchema>();
     }
 
     public LineageTracer getLineage() {
       return lineage;
     }
     
-    public Map<LogicalOperator, PhysicalOperator> getLogToPhyMap() {
+    public Map<Operator, PhysicalOperator> getLogToPhyMap() {
         return logToPhyMap;
     }
     
@@ -114,36 +115,36 @@ public class ExampleGenerator {
         MAX_RECORDS = max;
     }
 
-    public Map<LogicalOperator, DataBag> getExamples() throws IOException, InterruptedException {
+    public Map<Operator, DataBag> getExamples() throws IOException, InterruptedException {
         if (pigContext.getProperties().getProperty("pig.usenewlogicalplan", "true").equals("false"))
             throw new ExecException("ILLUSTRATE must use the new logical plan!");
         pigContext.inIllustrator = true;
         physPlan = compilePlan(plan);
         physPlanReseter = new PhysicalPlanResetter(physPlan);
-        List<LogicalOperator> loads = plan.getRoots();
+        List<Operator> loads = newPlan.getSources();
         List<PhysicalOperator> pRoots = physPlan.getRoots();
         if (loads.size() != pRoots.size())
             throw new ExecException("Logical and Physical plans have different number of roots");
         logToPhyMap = execEngine.getLogToPhyMap();
         forEachInnerLogToPhyMap = execEngine.getForEachInnerLogToPhyMap();
-        poLoadToLogMap = new HashMap<PhysicalOperator, LogicalOperator>();
-        logToDataMap = new HashMap<LogicalOperator, DataBag>();
-        poToLogMap = new HashMap<PhysicalOperator, LogicalOperator>();
+        poLoadToLogMap = new HashMap<PhysicalOperator, Operator>();
+        logToDataMap = new HashMap<Operator, DataBag>();
+        poToLogMap = new HashMap<PhysicalOperator, Operator>();
         
         // set up foreach inner data map
-        forEachInnerLogToDataMap = new HashMap<LOForEach, Map<LogicalOperator, DataBag>>();
-        for (Map.Entry<LOForEach, Map<LogicalOperator, PhysicalOperator>> entry : forEachInnerLogToPhyMap.entrySet()) {
-            Map<LogicalOperator, DataBag> innerMap = new HashMap<LogicalOperator, DataBag>();
+        forEachInnerLogToDataMap = new HashMap<LOForEach, Map<LogicalRelationalOperator, DataBag>>();
+        for (Map.Entry<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> entry : forEachInnerLogToPhyMap.entrySet()) {
+            Map<LogicalRelationalOperator, DataBag> innerMap = new HashMap<LogicalRelationalOperator, DataBag>();
             forEachInnerLogToDataMap.put(entry.getKey(), innerMap);
         }
 
-        for (LogicalOperator load : loads)
+        for (Operator load : loads)
         {
             poLoadToLogMap.put(logToPhyMap.get(load), load);
         }
 
         boolean hasLimit = false;
-        for (LogicalOperator lo : logToPhyMap.keySet()) {
+        for (Operator lo : logToPhyMap.keySet()) {
             poToLogMap.put(logToPhyMap.get(lo), lo);
             if (!hasLimit && lo instanceof LOLimit)
                 hasLimit = true;
@@ -159,64 +160,58 @@ public class ExampleGenerator {
             throw new RuntimeException(e.getMessage());
         }
 
-        Map<LogicalOperator, DataBag> derivedData = null;
-        try {
+        Map<Operator, DataBag> derivedData = null;
 
-            // create derived data and trim base data
-            LineageTrimmingVisitor trimmer = new LineageTrimmingVisitor(plan,
-                    baseData, this, logToPhyMap, physPlan, pigContext);
-            trimmer.visit();
-            // System.out.println(
-            // "Obtained the first level derived and trimmed data");
-            // create new derived data from trimmed basedata
-            derivedData = getData(physPlan);
-
-            // System.out.println(
-            // "Got new derived data from the trimmed base data");
-            // augment base data
-            AugmentBaseDataVisitor augment = new AugmentBaseDataVisitor(plan,
-                    logToPhyMap, baseData, derivedData);
+        // create derived data and trim base data
+        LineageTrimmingVisitor trimmer = new LineageTrimmingVisitor(newPlan,
+                baseData, this, logToPhyMap, physPlan, pigContext);
+        trimmer.visit();
+        baseData = trimmer.getBaseData();
+        // System.out.println(
+        // "Obtained the first level derived and trimmed data");
+        // create new derived data from trimmed basedata
+        derivedData = getData(physPlan);
+
+        // System.out.println(
+        // "Got new derived data from the trimmed base data");
+        // augment base data
+        AugmentBaseDataVisitor augment = new AugmentBaseDataVisitor(newPlan,
+                logToPhyMap, baseData, derivedData);
+        augment.visit();
+        this.baseData = augment.getNewBaseData();
+        // System.out.println("Obtained augmented base data");
+        // create new derived data and trim the base data after augmenting
+        // base data with synthetic tuples
+        trimmer = new LineageTrimmingVisitor(newPlan, baseData, this,
+                logToPhyMap, physPlan, pigContext);
+        trimmer.visit();
+        baseData = trimmer.getBaseData();
+        // System.out.println("Final trimming");
+        // create the final version of derivedData to give to the output
+        derivedData = getData(physPlan);
+        // System.out.println("Obtaining final derived data for output");
+        
+        if (hasLimit)
+        {
+            augment.setLimit();
             augment.visit();
             this.baseData = augment.getNewBaseData();
-            // System.out.println("Obtained augmented base data");
-            // create new derived data and trim the base data after augmenting
-            // base data with synthetic tuples
-            trimmer = new LineageTrimmingVisitor(plan, baseData, this,
-                    logToPhyMap, physPlan, pigContext);
-            trimmer.visit();
-            // System.out.println("Final trimming");
-            // create the final version of derivedData to give to the output
-            derivedData = getData(physPlan);
-            // System.out.println("Obtaining final derived data for output");
-            
-            if (hasLimit)
-            {
-                augment.setLimit();
-                augment.visit();
-                this.baseData = augment.getNewBaseData();
-                oriLimitMap = augment.getOriLimitMap();
-                derivedData = getData();
-            }
-
-        } catch (VisitorException e) {
-            e.printStackTrace(System.out);
-            log.error("Visitor exception while creating example data "
-                    + e.getMessage());
-            throw new RuntimeException(e.getMessage());
+            oriLimitMap = augment.getOriLimitMap();
+            derivedData = getData();
         }
 
         // DisplayExamples.printSimple(plan.getLeaves().get(0),
         // derivedData.derivedData);
-        System.out.println(DisplayExamples.printTabular(plan,
+        System.out.println(DisplayExamples.printTabular(newPlan,
                 derivedData, forEachInnerLogToDataMap));
         pigContext.inIllustrator = false;
         return derivedData;
     }
 
-    private void readBaseData(List<LogicalOperator> loads) throws IOException, InterruptedException, FrontendException, ExecException {
+    private void readBaseData(List<Operator> loads) throws IOException, InterruptedException, FrontendException, ExecException {
         PhysicalPlan thisPhyPlan = new PhysicalPlan();
-        for (LogicalOperator op : loads) {
-            Schema schema = op.getSchema();
+        for (Operator op : loads) {
+            LogicalSchema schema = ((LOLoad) op).getSchema();
             if(schema == null) {
                 throw new ExecException("Example Generator requires a schema. Please provide a schema while loading data.");
             }
@@ -224,9 +219,9 @@ public class ExampleGenerator {
             thisPhyPlan.add(logToPhyMap.get(op));
         }
         baseData = null;
-        Map<LogicalOperator, DataBag> result = getData(thisPhyPlan);
+        Map<Operator, DataBag> result = getData(thisPhyPlan);
         baseData = new HashMap<LOLoad, DataBag>();
-        for (LogicalOperator lo : result.keySet()) {
+        for (Operator lo : result.keySet()) {
             if (lo instanceof LOLoad) {
                 baseData.put((LOLoad) lo, result.get(lo));
             }
@@ -234,14 +229,16 @@ public class ExampleGenerator {
     }
 
     PhysicalPlan compilePlan(LogicalPlan plan) throws ExecException, FrontendException {
-        return execEngine.compile(plan, null);
+        PhysicalPlan result = execEngine.compile(plan, null);
+        newPlan = execEngine.getNewPlan();
+        return result;
     }
     
-    public Map<LogicalOperator, DataBag> getData() throws IOException, InterruptedException {
+    public Map<Operator, DataBag> getData() throws IOException, InterruptedException {
       return getData(physPlan);
     }
     
-    private Map<LogicalOperator, DataBag> getData(PhysicalPlan plan) throws PigException, IOException, InterruptedException
+    private Map<Operator, DataBag> getData(PhysicalPlan plan) throws PigException, IOException, InterruptedException
     {
         // get data on a physical plan possibly trimmed of one branch 
         lineage = new LineageTracer();
@@ -257,7 +254,7 @@ public class ExampleGenerator {
             setLoadDataMap();
             physPlanReseter.visit();
         }
-        localMRRunner.launchPig(plan, baseData, poLoadToLogMap, lineage, attacher, this, pigContext);
+        localMRRunner.launchPig(plan, baseData, lineage, attacher, this, pigContext);
         if (baseData == null)
             poToEqclassesMap = attacher.poToEqclassesMap;
         else {
@@ -272,7 +269,7 @@ public class ExampleGenerator {
         return logToDataMap;
     }
     
-    public Map<LogicalOperator, DataBag> getData(Map<LOLoad, DataBag> newBaseData) throws Exception 
+    public Map<Operator, DataBag> getData(Map<LOLoad, DataBag> newBaseData) throws Exception 
     {
         baseData = newBaseData;
         return getData(physPlan);
@@ -281,7 +278,7 @@ public class ExampleGenerator {
     private void phyToMRTransform(PhysicalPlan plan, Map<PhysicalOperator, DataBag> phyToDataMap) {
         // remap the LO to PO as result of the MR compilation may have changed PO in the MR plans
         Map<PhysicalOperator, PhysicalOperator> phyToMRMap = localMRRunner.getPhyToMRMap();
-        for (Map.Entry<PhysicalOperator, LogicalOperator> entry : poToLogMap.entrySet()) {
+        for (Map.Entry<PhysicalOperator, Operator> entry : poToLogMap.entrySet()) {
             if (phyToMRMap.get(entry.getKey()) != null) {
                 PhysicalOperator poInMR = phyToMRMap.get(entry.getKey());
                 logToDataMap.put(entry.getValue(), phyToDataMap.get(poInMR));
@@ -292,15 +289,15 @@ public class ExampleGenerator {
     
     private void getLogToDataMap(Map<PhysicalOperator, DataBag> phyToDataMap) {
         logToDataMap.clear();
-        for (LogicalOperator lo : logToPhyMap.keySet()) {
+        for (Operator lo : logToPhyMap.keySet()) {
             if (logToPhyMap.get(lo) != null)
                 logToDataMap.put(lo, phyToDataMap.get(logToPhyMap.get(lo)));
         }
         
         // set the LO-to-Data mapping for the ForEach inner plans
-        for (Map.Entry<LOForEach, Map<LogicalOperator, DataBag>> entry : forEachInnerLogToDataMap.entrySet()) {
+        for (Map.Entry<LOForEach, Map<LogicalRelationalOperator, DataBag>> entry : forEachInnerLogToDataMap.entrySet()) {
             entry.getValue().clear();
-            for (Map.Entry<LogicalOperator, PhysicalOperator>  innerEntry : forEachInnerLogToPhyMap.get(entry.getKey()).entrySet()) {
+            for (Map.Entry<LogicalRelationalOperator, PhysicalOperator>  innerEntry : forEachInnerLogToPhyMap.get(entry.getKey()).entrySet()) {
                 entry.getValue().put(innerEntry.getKey(), phyToDataMap.get(innerEntry.getValue()));
             }
         }
@@ -328,10 +325,10 @@ public class ExampleGenerator {
         }
     }
     
-    public Collection<IdentityHashSet<Tuple>> getEqClasses() throws VisitorException {
-        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> logToEqclassesMap = getLoToEqClassMap();
+    public Collection<IdentityHashSet<Tuple>> getEqClasses() {
+        Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> logToEqclassesMap = getLoToEqClassMap();
         LinkedList<IdentityHashSet<Tuple>> ret = new LinkedList<IdentityHashSet<Tuple>>();
-        for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> entry :
+        for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> entry :
             logToEqclassesMap.entrySet()) {
             if (entry.getValue() != null)
                 ret.addAll(entry.getValue());
@@ -339,12 +336,12 @@ public class ExampleGenerator {
         return ret;
     }
 
-    public Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> getLoToEqClassMap() throws VisitorException {
-        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> ret =
-          EquivalenceClasses.getLoToEqClassMap(physPlan, plan, logToPhyMap, logToDataMap, forEachInnerLogToPhyMap, poToEqclassesMap);
+    public Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> getLoToEqClassMap() {
+        Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> ret =
+          EquivalenceClasses.getLoToEqClassMap(physPlan, newPlan, logToPhyMap, logToDataMap, forEachInnerLogToPhyMap, poToEqclassesMap);
         // eq classes adjustments based upon logical operators
         
-        for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> entry :ret.entrySet())
+        for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> entry :ret.entrySet())
         {
             if (entry.getKey() instanceof LOSort) {
                 Collection<IdentityHashSet<Tuple>> eqClasses = entry.getValue();

Modified: pig/trunk/src/org/apache/pig/pen/Illustrator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/Illustrator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/Illustrator.java (original)
+++ pig/trunk/src/org/apache/pig/pen/Illustrator.java Fri Dec 17 22:08:08 2010
@@ -29,7 +29,7 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.pen.util.LineageTracer;
 import org.apache.pig.impl.util.IdentityHashSet;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
 
 /**
  * Class used by physical operators to generate example tuples for the ILLUSTRATE
@@ -51,7 +51,7 @@ public class Illustrator {
     private Boolean[] subExpResult;
     private boolean eqClassesShared;
     private long oriLimit = -1;
-    private Schema schema;
+    private LogicalSchema schema;
 
     public Illustrator(LineageTracer lineage, LinkedList<IdentityHashSet<Tuple>> equivalenceClasses, IllustratorAttacher attacher, PigContext hadoopPigContext) {
         this.lineage = lineage;
@@ -64,7 +64,7 @@ public class Illustrator {
     }
     
     public Illustrator(LineageTracer lineage, LinkedList<IdentityHashSet<Tuple>> equivalenceClasses, int maxRecords, IllustratorAttacher attacher,
-        Schema schema, PigContext hadoopPigContext) {
+        LogicalSchema schema, PigContext hadoopPigContext) {
         this(lineage, equivalenceClasses, attacher, hadoopPigContext);
         this.maxRecords = maxRecords;
         this.schema = schema;
@@ -136,7 +136,7 @@ public class Illustrator {
         return eqClassesShared;
     }
     
-    public Schema getSchema() {
+    public LogicalSchema getSchema() {
         return schema;
     }
 }

Modified: pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java (original)
+++ pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java Fri Dec 17 22:08:08 2010
@@ -79,7 +79,7 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
 import org.apache.pig.pen.util.LineageTracer;
 import org.apache.pig.impl.plan.DepthFirstWalker;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
 
 /**
  * The class used to (re)attach illustrators to physical operators
@@ -98,10 +98,10 @@ public class IllustratorAttacher extends
     private int maxRecords;
     private boolean revisit = false;
     private ArrayList<Boolean[]> subExpResults = null;
-    private final Map<POLoad, Schema> poloadToSchemaMap;
+    private final Map<POLoad, LogicalSchema> poloadToSchemaMap;
     
     public IllustratorAttacher(PhysicalPlan plan, LineageTracer lineage, int maxRecords,
-        Map<POLoad, Schema> poLoadToSchemaMap, PigContext hadoopPigContext) throws VisitorException {
+        Map<POLoad, LogicalSchema> poLoadToSchemaMap, PigContext hadoopPigContext) throws VisitorException {
         super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
         pigContext = hadoopPigContext;
         this.lineage = lineage;

Modified: pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java Fri Dec 17 22:08:08 2010
@@ -39,22 +39,23 @@ import org.apache.pig.data.BagFactory;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.LOCogroup;
-import org.apache.pig.impl.logicalLayer.LOJoin;
-import org.apache.pig.impl.logicalLayer.LOCross;
-import org.apache.pig.impl.logicalLayer.LODistinct;
-import org.apache.pig.impl.logicalLayer.LOFilter;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LOLimit;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LOSort;
-import org.apache.pig.impl.logicalLayer.LOSplit;
-import org.apache.pig.impl.logicalLayer.LOUnion;
-import org.apache.pig.impl.logicalLayer.LOStore;
-import org.apache.pig.impl.logicalLayer.LOVisitor;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.logical.relational.LOCogroup;
+import org.apache.pig.newplan.logical.relational.LOJoin;
+import org.apache.pig.newplan.logical.relational.LOCross;
+import org.apache.pig.newplan.logical.relational.LODistinct;
+import org.apache.pig.newplan.logical.relational.LOFilter;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOLimit;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LOSort;
+import org.apache.pig.newplan.logical.relational.LOSplit;
+import org.apache.pig.newplan.logical.relational.LOUnion;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.IdentityHashSet;
 import org.apache.pig.impl.util.Pair;
 import org.apache.pig.impl.io.FileSpec;
@@ -63,18 +64,18 @@ import org.apache.pig.pen.util.MetricEva
 import org.apache.pig.pen.util.PreOrderDepthFirstWalker;
 import org.apache.pig.pen.util.ExampleTuple;
 
-public class LineageTrimmingVisitor extends LOVisitor {
+public class LineageTrimmingVisitor extends LogicalRelationalNodesVisitor {
 
     LogicalPlan plan = null;
     Map<LOLoad, DataBag> baseData;
     Map<FileSpec, DataBag> inputToDataMap;
-    Map<LogicalOperator, PhysicalOperator> LogToPhyMap = null;
+    Map<Operator, PhysicalOperator> LogToPhyMap = null;
     PhysicalPlan physPlan = null;
     double completeness = 100.0;
     Log log = LogFactory.getLog(getClass());
 
-    Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> AffinityGroups = new HashMap<LogicalOperator, Collection<IdentityHashSet<Tuple>>>();
-    Map<LogicalOperator, LineageTracer> Lineage = new HashMap<LogicalOperator, LineageTracer>();
+    Map<Operator, Collection<IdentityHashSet<Tuple>>> AffinityGroups = new HashMap<Operator, Collection<IdentityHashSet<Tuple>>>();
+    Map<Operator, LineageTracer> Lineage = new HashMap<Operator, LineageTracer>();
 
     boolean continueTrimming;
     PigContext pc;
@@ -83,10 +84,9 @@ public class LineageTrimmingVisitor exte
     public LineageTrimmingVisitor(LogicalPlan plan,
             Map<LOLoad, DataBag> baseData,
             ExampleGenerator eg,
-            Map<LogicalOperator, PhysicalOperator> LogToPhyMap,
+            Map<Operator, PhysicalOperator> LogToPhyMap,
             PhysicalPlan physPlan, PigContext pc) throws IOException, InterruptedException {
-        super(plan, new PreOrderDepthFirstWalker<LogicalOperator, LogicalPlan>(
-                plan));
+        super(plan, new PreOrderDepthFirstWalker(plan));
         // this.baseData.putAll(baseData);
         this.baseData = baseData;
         this.plan = plan;
@@ -100,11 +100,11 @@ public class LineageTrimmingVisitor exte
 
     public void init() throws IOException, InterruptedException {
 
-        Map<LogicalOperator, DataBag> data = eg.getData();
+        Map<Operator, DataBag> data = eg.getData();
 
         LineageTracer lineage = eg.getLineage();
-        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = eg.getLoToEqClassMap();
-        for (LogicalOperator leaf : plan.getLeaves()) {
+        Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = eg.getLoToEqClassMap();
+        for (Operator leaf : plan.getSinks()) {
             Lineage.put(leaf, lineage);
             AffinityGroups.put(leaf, eg.getEqClasses());
         }
@@ -116,7 +116,7 @@ public class LineageTrimmingVisitor exte
     }
 
     @Override
-    protected void visit(LOCogroup cg) throws VisitorException {
+    public void visit(LOCogroup cg) throws FrontendException {
         // can't separate CoGroup from succeeding ForEach
         if (plan.getSuccessors(cg) != null && plan.getSuccessors(cg).get(0) instanceof LOForEach)
             return;
@@ -128,91 +128,91 @@ public class LineageTrimmingVisitor exte
                 
                 LineageTracer lineage = null;
                 // create affinity groups
-                if (cg.getInputs().size() == 1) {
+                if (cg.getInputs(plan).size() == 1) {
                     lineage = eg.getLineage();
-                    AffinityGroups.put(cg.getInputs().get(0), eg.getEqClasses());
-                    Lineage.put(cg.getInputs().get(0), lineage);
+                    AffinityGroups.put(cg.getInputs(plan).get(0), eg.getEqClasses());
+                    Lineage.put(cg.getInputs(plan).get(0), lineage);
 
                 } else {
-                    for (LogicalOperator input : cg.getInputs()) {
+                    for (Operator input : cg.getInputs(plan)) {
                         Lineage.put(input, eg.getLineage());
                         AffinityGroups.put(input, eg.getEqClasses());
                     }
                 }
             } catch (Exception e) {
-                throw new VisitorException("Exception : "+e.getMessage());
+                throw new FrontendException("Exception : "+e.getMessage());
             }
         }
     }
 
     @Override
-    protected void visit(LOJoin join) throws VisitorException {
+    public void visit(LOJoin join) throws FrontendException {
         if (continueTrimming) {
           processOperator(join);
         }
     }
 
     @Override
-    protected void visit(LOCross cs) throws VisitorException {
+    public void visit(LOCross cs) throws FrontendException {
         if(continueTrimming)
             processOperator(cs);
 
     }
 
     @Override
-    protected void visit(LODistinct dt) throws VisitorException {
+    public void visit(LODistinct dt) throws FrontendException {
         if(continueTrimming)
             processOperator(dt);
 
     }
 
     @Override
-    protected void visit(LOFilter filter) throws VisitorException {
+    public void visit(LOFilter filter) throws FrontendException {
         if (continueTrimming)
             processOperator(filter);
     }
     
     @Override
-    protected void visit(LOStore store) throws VisitorException {
+    public void visit(LOStore store) throws FrontendException {
         if (continueTrimming)
             processOperator(store);
     }
 
     @Override
-    protected void visit(LOForEach forEach) throws VisitorException {
+    public void visit(LOForEach forEach) throws FrontendException {
         if (continueTrimming)
             processOperator(forEach);
     }
 
     @Override
-    protected void visit(LOLimit limOp) throws VisitorException {
+    public void visit(LOLimit limOp) throws FrontendException {
         if(continueTrimming)
             processOperator(limOp);
 
     }
 
     @Override
-    protected void visit(LOLoad load) throws VisitorException {
+    public void visit(LOLoad load) throws FrontendException {
         if (continueTrimming)
             processOperator(load);
     }
 
     @Override
-    protected void visit(LOSort s) throws VisitorException {
+    public void visit(LOSort s) throws FrontendException {
         if(continueTrimming)
             processOperator(s);
 
     }
 
     @Override
-    protected void visit(LOSplit split) throws VisitorException {
+    public void visit(LOSplit split) throws FrontendException {
         if(continueTrimming)
             processOperator(split);
 
     }
 
     @Override
-    protected void visit(LOUnion u) throws VisitorException {
+    public void visit(LOUnion u) throws FrontendException {
         if(continueTrimming)
             processOperator(u);
 
@@ -322,6 +322,7 @@ public class LineageTrimmingVisitor exte
             for (Tuple t : members)
                 tuplesToRetain.add(t);
         }
+
         Map<LOLoad, DataBag> newBaseData = new HashMap<LOLoad, DataBag>();
         for (LOLoad loadOp : baseData.keySet()) {
             DataBag data = baseData.get(loadOp);
@@ -338,10 +339,10 @@ public class LineageTrimmingVisitor exte
         return newBaseData;
     }
 
-    private void processLoad(LOLoad ld) throws VisitorException {
+    private void processLoad(LOLoad ld) throws FrontendException {
         // prune base records
-        if (inputToDataMap.get(ld.getInputFile()) != null) {
-            baseData.put(ld, inputToDataMap.get(ld.getInputFile()));
+        if (inputToDataMap.get(ld.getFileSpec()) != null) {
+            baseData.put(ld, inputToDataMap.get(ld.getFileSpec()));
             return;
         }
         
@@ -363,7 +364,7 @@ public class LineageTrimmingVisitor exte
         newBaseData.put(ld, newData);
         for (Map.Entry<LOLoad, DataBag> entry : baseData.entrySet()) {
             if (entry.getKey() != ld) {
-                if (!entry.getKey().getInputFile().equals(ld.getInputFile()))
+                if (!entry.getKey().getFileSpec().equals(ld.getFileSpec()))
                     newBaseData.put(entry.getKey(), entry.getValue());
                 else
                     newBaseData.put(entry.getKey(), newData);
@@ -373,10 +374,10 @@ public class LineageTrimmingVisitor exte
         if (checkNewBaseData(newData, newBaseData, realData))
             checkNewBaseData(newData, newBaseData, syntheticData);
         
-        inputToDataMap.put(ld.getInputFile(), baseData.get(ld));
+        inputToDataMap.put(ld.getFileSpec(), baseData.get(ld));
     }
     
-    private boolean checkNewBaseData(DataBag data, Map<LOLoad, DataBag> newBaseData, Set<Tuple> loadData) throws VisitorException {
+    private boolean checkNewBaseData(DataBag data, Map<LOLoad, DataBag> newBaseData, Set<Tuple> loadData) throws FrontendException {
         List<Pair<Tuple, Double>> sortedBase = new LinkedList<Pair<Tuple, Double>>();
         DataBag oldData = BagFactory.getInstance().newDefaultBag();
         oldData.addAll(data);
@@ -384,11 +385,11 @@ public class LineageTrimmingVisitor exte
         for (Tuple t : loadData) {
             data.add(t);
             // obtain the derived data 
-            Map<LogicalOperator, DataBag> derivedData;
+            Map<Operator, DataBag> derivedData;
             try {
                 derivedData = eg.getData(newBaseData);
             } catch (Exception e) {
-                throw new VisitorException("Exception: "+e.getMessage());
+                throw new FrontendException("Exception: "+e.getMessage());
             }
             double newCompleteness = MetricEvaluation.getCompleteness(null,
                     derivedData, eg.getLoToEqClassMap(), true);
@@ -412,11 +413,11 @@ public class LineageTrimmingVisitor exte
         for (Pair<Tuple, Double> p : sortedBase) {
             data.add(p.first);
             // obtain the derived data 
-            Map<LogicalOperator, DataBag> derivedData;
+            Map<Operator, DataBag> derivedData;
             try {
                 derivedData = eg.getData(newBaseData);
             } catch (Exception e) {
-                throw new VisitorException("Exception: "+e.getMessage());
+                throw new FrontendException("Exception: "+e.getMessage());
             }
             double newCompleteness = MetricEvaluation.getCompleteness(null,
                     derivedData, eg.getLoToEqClassMap(), true);
@@ -430,7 +431,7 @@ public class LineageTrimmingVisitor exte
         return true;
     }
     
-    private void processOperator(LogicalOperator op) throws VisitorException {
+    private void processOperator(LogicalRelationalOperator op) throws FrontendException {
         
         try {
             if (op instanceof LOLoad) {
@@ -446,28 +447,28 @@ public class LineageTrimmingVisitor exte
             if (continueTrimming == false)
                 return;
 
-            LogicalOperator childOp = plan.getPredecessors(op).get(0);
+            Operator childOp = plan.getPredecessors(op).get(0);
             if (op instanceof LOForEach && childOp instanceof LOCogroup)
             {
                 LOCogroup cg = (LOCogroup) childOp;
-                for (LogicalOperator input : cg.getInputs()) {
+                for (Operator input : cg.getInputs(plan)) {
                     AffinityGroups.put(input, eg.getEqClasses());
                     Lineage.put(input, eg.getLineage());
                 }
             } else {
-                List<LogicalOperator> childOps = plan.getPredecessors(op);
-                for (LogicalOperator lo : childOps) {
+                List<Operator> childOps = plan.getPredecessors(op);
+                for (Operator lo : childOps) {
                     AffinityGroups.put(lo, eg.getEqClasses());
                     Lineage.put(lo, eg.getLineage());
                 }
             }
         } catch (Exception e) {
           e.printStackTrace(System.out);
-          throw new VisitorException("Exception: "+e.getMessage());
+          throw new FrontendException("Exception: "+e.getMessage());
         }
     }
 
-    private boolean checkCompleteness(LogicalOperator op) throws Exception, VisitorException {
+    private boolean checkCompleteness(LogicalRelationalOperator op) throws Exception {
         LineageTracer lineage = Lineage.get(op);
         Lineage.remove(op);
 
@@ -479,7 +480,7 @@ public class LineageTrimmingVisitor exte
                 baseData, lineage, affinityGroups);
 
         // obtain the derived data
-        Map<LogicalOperator, DataBag> derivedData = eg.getData(newBaseData);
+        Map<Operator, DataBag> derivedData = eg.getData(newBaseData);
         double newCompleteness = MetricEvaluation.getCompleteness(null,
                 derivedData, eg.getLoToEqClassMap(), true);
 
@@ -492,4 +493,8 @@ public class LineageTrimmingVisitor exte
 
         return continueTrimming;
     }
+    
+    Map<LOLoad, DataBag> getBaseData() {
+        return baseData;
+    }
 }

Modified: pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java (original)
+++ pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java Fri Dec 17 22:08:08 2010
@@ -56,8 +56,8 @@ import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.io.NullableTuple;
 import org.apache.pig.impl.io.PigNullableWritable;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
 import org.apache.pig.impl.plan.DepthFirstWalker;
 import org.apache.pig.impl.plan.OperatorKey;
 import org.apache.pig.impl.plan.VisitorException;
@@ -82,7 +82,6 @@ public class LocalMapReduceSimulator {
 
     @SuppressWarnings("unchecked")
     public void launchPig(PhysicalPlan php, Map<LOLoad, DataBag> baseData,
-                              Map<PhysicalOperator, LogicalOperator> poLoadToLogMap,
                               LineageTracer lineage,
                               IllustratorAttacher attacher,
                               ExampleGenerator eg,
@@ -158,7 +157,7 @@ public class LocalMapReduceSimulator {
                 for (POLoad ld : lds) {
                     input = output.get(ld.getLFile().getFileName());
                     if (input == null && baseData != null) {
-                        for (LogicalOperator lo : baseData.keySet()) {
+                        for (LogicalRelationalOperator lo : baseData.keySet()) {
                             if (((LOLoad) lo).getSchemaFile().equals(ld.getLFile().getFileName()))
                             {
                                  input = baseData.get(lo);
@@ -174,7 +173,7 @@ public class LocalMapReduceSimulator {
                     input = output.get(ld.getLFile().getFileName());
                     if (input == null && baseData != null) {
                         if (input == null && baseData != null) {
-                            for (LogicalOperator lo : baseData.keySet()) {
+                            for (LogicalRelationalOperator lo : baseData.keySet()) {
                                 if (((LOLoad) lo).getSchemaFile().equals(ld.getLFile().getFileName()))
                                 {
                                      input = baseData.get(lo);
@@ -224,7 +223,9 @@ public class LocalMapReduceSimulator {
                         context = reduce.getIllustratorContext(job, intermediateData, (POPackage) pack);
                     reduce.run(context);
                 }
-                phyToMRMap.putAll(mro.phyToMRMap);
+                for (PhysicalOperator key : mro.phyToMRMap.keySet())
+                    for (PhysicalOperator value : mro.phyToMRMap.get(key))
+                        phyToMRMap.put(key, value);
             }
             
             

Modified: pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java (original)
+++ pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java Fri Dec 17 22:08:08 2010
@@ -29,12 +29,14 @@ import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataType;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
-import org.apache.pig.impl.logicalLayer.LOStore;
-import org.apache.pig.impl.logicalLayer.LOLimit;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LOLimit;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.Operator;
 import org.apache.pig.impl.util.IdentityHashSet;
 
 //Class containing some generic printing methods to print example data in a simple/tabular form
@@ -44,9 +46,9 @@ public class DisplayExamples {
     public static final int MAX_DATAATOM_LENGTH = 25;
 
     static void printMetrics(
-            LogicalOperator op,
-            Map<LogicalOperator, DataBag> derivedData,
-            Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses) {
+            Operator op,
+            Map<Operator, DataBag> derivedData,
+            Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses) {
         /*
          * System.out.println("Realness : " + Metrics.getRealness(op,
          * derivedData, true)); System.out.println("Completeness : " +
@@ -67,26 +69,34 @@ public class DisplayExamples {
     }
 
     public static String printTabular(LogicalPlan lp,
-            Map<LogicalOperator, DataBag> exampleData,
-            Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap) {
+            Map<Operator, DataBag> exampleData,
+            Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap) throws FrontendException {
         StringBuffer output = new StringBuffer();
-        Set<LogicalOperator> seen = new HashSet<LogicalOperator>();
-        for (LogicalOperator currentOp : lp.getLeaves())
-            printTabular(currentOp, exampleData, forEachInnerLogToDataMap, seen, output);
+        Set<Operator> seen = new HashSet<Operator>();
+        for (Operator currentOp : lp.getSinks()) {
+            if (currentOp instanceof LOStore && ((LOStore) currentOp).isTmpStore())
+            {
+                // display the branches of the temporary store first
+                printTabular(currentOp, lp, exampleData, forEachInnerLogToDataMap, seen, output);
+            }
+        }
+        for (Operator currentOp : lp.getSinks())
+            printTabular(currentOp, lp, exampleData, forEachInnerLogToDataMap, seen, output);
         return output.toString();
     }
 
-    static void printTabular(LogicalOperator op,
-            Map<LogicalOperator, DataBag> exampleData,
-            Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap,
-            Set<LogicalOperator> seen,
+    static void printTabular(Operator op,
+            LogicalPlan lp,
+            Map<Operator, DataBag> exampleData,
+            Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap,
+            Set<Operator> seen,
             StringBuffer output) {
 
-        List<LogicalOperator> inputs = op.getPlan().getPredecessors(op);
+        List<Operator> inputs = lp.getPredecessors(op);
         if (inputs != null) { // to avoid an exception when op == LOLoad
-            for (LogicalOperator Op : inputs) {
+            for (Operator Op : inputs) {
                 if (!seen.contains(Op))
-                  printTabular(Op, exampleData, forEachInnerLogToDataMap, seen, output);
+                  printTabular(Op, lp, exampleData, forEachInnerLogToDataMap, seen, output);
             }
         }
         seen.add(op);
@@ -95,8 +105,13 @@ public class DisplayExamples {
             printNestedTabular((LOForEach)op, forEachInnerLogToDataMap, exampleData.get(op), output);
         }
         
-        if (op.getAlias() != null) {
+        if (((LogicalRelationalOperator)op).getAlias() != null) {
             DataBag bag = exampleData.get(op);
+            if (op instanceof LOLoad && ((LOLoad)op).isCastInserted())
+            {
+                op = op.getPlan().getSuccessors(op).get(0);
+                bag = exampleData.get(op);
+            }
             try {
                 DisplayTable(MakeArray(op, bag), op, bag, output);
             } catch (FrontendException e) {
@@ -112,31 +127,29 @@ public class DisplayExamples {
 
     // print out nested gen block in ForEach
     static void printNestedTabular(LOForEach foreach,
-            Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap,
+            Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap,
             DataBag foreachData,
             StringBuffer output) {
-        List<LogicalPlan> plans = foreach.getForEachPlans();
-        if (plans != null) {
-            for (LogicalPlan plan : plans) {
-                printNestedTabular(plan.getLeaves().get(0), foreach.getAlias(), foreachData, forEachInnerLogToDataMap.get(foreach), output);
-            }
+        LogicalPlan plan = foreach.getInnerPlan();
+        if (plan != null) {
+            printNestedTabular(plan.getSinks().get(0), plan, foreach.getAlias(), foreachData, forEachInnerLogToDataMap.get(foreach), output);
         }
     }
 
-    static void printNestedTabular(LogicalOperator lo, String foreachAlias, DataBag foreachData, 
-            Map<LogicalOperator, DataBag> logToDataMap, StringBuffer output) {
+    static void printNestedTabular(Operator lo, LogicalPlan lp, String foreachAlias, DataBag foreachData, 
+            Map<LogicalRelationalOperator, DataBag> logToDataMap, StringBuffer output) {
         
-        List<LogicalOperator> inputs = lo.getPlan().getPredecessors(lo);
+        List<Operator> inputs = lp.getPredecessors(lo);
         if (inputs != null) {
-            for (LogicalOperator op : inputs)
-                printNestedTabular(op, foreachAlias, foreachData, logToDataMap, output);
+            for (Operator op : inputs)
+                printNestedTabular(op, lp, foreachAlias, foreachData, logToDataMap, output);
         }
         
         DataBag bag = logToDataMap.get(lo);
         if (bag == null)
           return;
         
-        if (lo.getAlias() != null) {
+        if (((LogicalRelationalOperator)lo).getAlias() != null) {
             try {
               DisplayNestedTable(MakeArray(lo, bag), lo, foreachAlias, foreachData, bag, output);
             } catch (FrontendException e) {
@@ -149,20 +162,20 @@ public class DisplayExamples {
         }
     }
     
-    public static void printSimple(LogicalOperator op,
-            Map<LogicalOperator, DataBag> exampleData) {
+    public static void printSimple(Operator op, LogicalPlan lp,
+            Map<Operator, DataBag> exampleData) {
         DataBag bag = exampleData.get(op);
 
-        List<LogicalOperator> inputs = op.getPlan().getPredecessors(op);
+        List<Operator> inputs = lp.getPredecessors(op);
         if (inputs != null) {
-            for (LogicalOperator lOp : inputs) {
-                printSimple(lOp, exampleData);
+            for (Operator lOp : inputs) {
+                printSimple(lOp, lp, exampleData);
             }
         }
-        if (op.getAlias() != null) {
+        if (((LogicalRelationalOperator)op).getAlias() != null) {
             // printTable(op, bag, output);
             // DisplayTable(MakeArray(op, bag), op, bag, output);
-            System.out.println(op.getAlias() + " : " + bag);
+            System.out.println(((LogicalRelationalOperator)op).getAlias() + " : " + bag);
         }
         // System.out.println(op.getAlias() + " : " + bag);
     }
@@ -178,13 +191,13 @@ public class DisplayExamples {
         return str.toString();
     }
 
-    static void DisplayTable(String[][] table, LogicalOperator op, DataBag bag,
+    static void DisplayTable(String[][] table, Operator op, DataBag bag,
             StringBuffer output) throws FrontendException {
         if (op instanceof LOStore && ((LOStore) op).isTmpStore())
             return;
         
-        int cols = op.getSchema().getFields().size();
-        List<FieldSchema> fields = op.getSchema().getFields();
+        int cols = ((LogicalRelationalOperator)op).getSchema().getFields().size();
+        List<LogicalSchema.LogicalFieldSchema> fields = ((LogicalRelationalOperator)op).getSchema().getFields();
         int rows = (int) bag.size();
         int[] maxColSizes = new int[cols];
         for (int i = 0; i < cols; ++i) {
@@ -193,7 +206,8 @@ public class DisplayExamples {
                 maxColSizes[i] = 5;
         }
         int total = 0;
-        int aliasLength = (op instanceof LOStore ? op.getAlias().length() + 12 : op.getAlias().length() + 4);
+        final String alias = ((LogicalRelationalOperator)op).getAlias();
+        int aliasLength = (op instanceof LOStore ? alias.length() + 12 : alias.length() + 4);
         for (int j = 0; j < cols; ++j) {
             for (int i = 0; i < rows; ++i) {
                 int length = table[i][j].length();
@@ -214,11 +228,11 @@ public class DisplayExamples {
                         false)
                         + "\n");
         if (op instanceof LOStore)
-            output.append("| Store : " + op.getAlias() + AddSpaces(4, true) + " | ");
+            output.append("| Store : " + alias + AddSpaces(4, true) + " | ");
         else
-            output.append("| " + op.getAlias() + AddSpaces(4, true) + " | ");
+            output.append("| " + alias + AddSpaces(4, true) + " | ");
         for (int i = 0; i < cols; ++i) {
-            String field = fields.get(i).toString();
+            String field = fields.get(i).toString(false);
             output.append(field
                     + AddSpaces(maxColSizes[i] - field.length(), true) + " | ");
         }
@@ -243,10 +257,11 @@ public class DisplayExamples {
                         + "\n");
     }
 
-    static void DisplayNestedTable(String[][] table, LogicalOperator op, String foreachAlias, DataBag bag,
+    static void DisplayNestedTable(String[][] table, Operator op, String foreachAlias, DataBag bag,
             DataBag foreachData, StringBuffer output) throws FrontendException {
-        int cols = op.getSchema().getFields().size();
-        List<FieldSchema> fields = op.getSchema().getFields();
+        LogicalRelationalOperator lop = (LogicalRelationalOperator) op;
+        int cols = lop.getSchema().getFields().size();
+        List<LogicalSchema.LogicalFieldSchema> fields = lop.getSchema().getFields();
         int rows = (int) bag.size();
         int[] maxColSizes = new int[cols];
         for (int i = 0; i < cols; ++i) {
@@ -255,7 +270,8 @@ public class DisplayExamples {
                 maxColSizes[i] = 5;
         }
         int total = 0;
-        int aliasLength = op.getAlias().length() + +foreachAlias.length() + 5;
+        final String alias = ((LogicalRelationalOperator)op).getAlias();
+        int aliasLength = alias.length() + +foreachAlias.length() + 5;
         for (int j = 0; j < cols; ++j) {
             for (int i = 0; i < rows; ++i) {
                 int length = table[i][j].length();
@@ -270,10 +286,10 @@ public class DisplayExamples {
                 .append(AddSpaces(total + 3 * (cols + 1) + aliasLength + 1,
                         false)
                         + "\n");
-        output.append("| " + foreachAlias + "." + op.getAlias() + AddSpaces(4, true) + " | ");
+        output.append("| " + foreachAlias + "." + alias + AddSpaces(4, true) + " | ");
         for (int i = 0; i < cols; ++i) {
             String field;
-            field = fields.get(i).toString();
+            field = fields.get(i).toString(false);
             output.append(field
                     + AddSpaces(maxColSizes[i] - field.length(), true) + " | ");
         }
@@ -298,10 +314,10 @@ public class DisplayExamples {
                         + "\n");
     }
 
-    static String[][] MakeArray(LogicalOperator op, DataBag bag)
+    static String[][] MakeArray(Operator op, DataBag bag)
             throws Exception {
         int rows = (int) bag.size();
-        int cols = op.getSchema().getFields().size();
+        int cols = ((LogicalRelationalOperator)op).getSchema().getFields().size();
         String[][] table = new String[rows][cols];
         Iterator<Tuple> it = bag.iterator();
         for (int i = 0; i < rows; ++i) {

Modified: pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java (original)
+++ pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java Fri Dec 17 22:08:08 2010
@@ -24,19 +24,20 @@ import java.util.Map;
 
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
 import org.apache.pig.impl.util.IdentityHashSet;
 
 //Evaluates various metrics
 public class MetricEvaluation {
-    public static float getRealness(LogicalOperator op,
-            Map<LogicalOperator, DataBag> exampleData, boolean overallRealness) {
+    public static float getRealness(Operator op,
+            Map<Operator, DataBag> exampleData, boolean overallRealness) {
         // StringBuffer str = new StringBuffer();
         int noTuples = 0;
         int noSynthetic = 0;
-        for (Map.Entry<LogicalOperator, DataBag> e : exampleData.entrySet()) {
+        for (Map.Entry<Operator, DataBag> e : exampleData.entrySet()) {
             // if(e.getKey() instanceof LORead) continue;
-            if (e.getKey().getAlias() == null)
+            if (((LogicalRelationalOperator)e.getKey()).getAlias() == null)
                 continue;
             DataBag bag;
             if (overallRealness) {
@@ -65,9 +66,9 @@ public class MetricEvaluation {
     }
 
     public static float getConciseness(
-            LogicalOperator op,
-            Map<LogicalOperator, DataBag> exampleData,
-            Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
+            Operator op,
+            Map<Operator, DataBag> exampleData,
+            Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
             boolean overallConciseness) {
         DataBag bag = exampleData.get(op);
 
@@ -85,7 +86,7 @@ public class MetricEvaluation {
             conciseness = 0;
             int noOperators = 0;
 
-            for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
+            for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
                     .entrySet()) {
                 if (e.getKey().getAlias() == null)
                     continue;
@@ -108,9 +109,9 @@ public class MetricEvaluation {
     }
 
     public static float getCompleteness(
-            LogicalOperator op,
-            Map<LogicalOperator, DataBag> exampleData,
-            Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
+            Operator op,
+            Map<Operator, DataBag> exampleData,
+            Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
             boolean overallCompleteness) {
 
         int noClasses = 0;
@@ -126,7 +127,7 @@ public class MetricEvaluation {
 
             return 100 * ((float) noCoveredClasses) / (float) noClasses;
         } else {
-            for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
+            for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
                     .entrySet()) {
                 noCoveredClasses = 0;
                 noClasses = 0;

Modified: pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java (original)
+++ pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java Fri Dec 17 22:08:08 2010
@@ -23,16 +23,15 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.pig.impl.plan.DepthFirstWalker;
-import org.apache.pig.impl.plan.Operator;
-import org.apache.pig.impl.plan.OperatorPlan;
-import org.apache.pig.impl.plan.PlanVisitor;
-import org.apache.pig.impl.plan.PlanWalker;
-import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.DepthFirstWalker;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.PlanVisitor;
+import org.apache.pig.newplan.PlanWalker;
+import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.Utils;
 
-public class PreOrderDepthFirstWalker<O extends Operator, P extends OperatorPlan<O>>
-        extends PlanWalker<O, P> {
+public class PreOrderDepthFirstWalker extends PlanWalker {
   
     private boolean branchFlag = false;
     
@@ -40,7 +39,7 @@ public class PreOrderDepthFirstWalker<O 
      * @param plan
      *            Plan for this walker to traverse.
      */
-    public PreOrderDepthFirstWalker(P plan) {
+    public PreOrderDepthFirstWalker(OperatorPlan plan) {
         super(plan);
     }
 
@@ -60,29 +59,28 @@ public class PreOrderDepthFirstWalker<O 
      * @throws VisitorException
      *             if an error is encountered while walking.
      */
-    public void walk(PlanVisitor<O, P> visitor) throws VisitorException {
-        List<O> leaves = mPlan.getLeaves();
-        Set<O> seen = new HashSet<O>();
+    public void walk(PlanVisitor visitor) throws FrontendException {
+        List<Operator> leaves = plan.getSinks();
+        Set<Operator> seen = new HashSet<Operator>();
 
         depthFirst(null, leaves, seen, visitor);
     }
 
-    public PlanWalker<O, P> spawnChildWalker(P plan) {
-        return new DepthFirstWalker<O, P>(plan);
+    public PlanWalker spawnChildWalker(OperatorPlan plan) {
+        return new DepthFirstWalker(plan);
     }
 
-    @SuppressWarnings("unchecked")
-    private void depthFirst(O node, Collection<O> predecessors, Set<O> seen,
-            PlanVisitor<O, P> visitor) throws VisitorException {
+    private void depthFirst(Operator node, Collection<Operator> predecessors, Set<Operator> seen,
+            PlanVisitor visitor) throws FrontendException {
         if (predecessors == null)
             return;
 
         boolean thisBranchFlag = branchFlag;
-        for (O pred : predecessors) {
+        for (Operator pred : predecessors) {
             if (seen.add(pred)) {
                 branchFlag = thisBranchFlag;
-                pred.visit(visitor);
-                Collection<O> newPredecessors = Utils.mergeCollection(mPlan.getPredecessors(pred), mPlan.getSoftLinkPredecessors(pred));
+                pred.accept(visitor);
+                Collection<Operator> newPredecessors = Utils.mergeCollection(plan.getPredecessors(pred), plan.getSoftLinkPredecessors(pred));
                 depthFirst(pred, newPredecessors, seen, visitor);
             }
         }

Modified: pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java Fri Dec 17 22:08:08 2010
@@ -32,7 +32,7 @@ import org.apache.pig.backend.executione
 import org.apache.pig.data.DataBag;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.Operator;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
@@ -105,7 +105,7 @@ public class TestExampleGenerator extend
         String query = "A = load " + A
                 + " using PigStorage() as (x : int, y : int);\n";
         pigserver.registerQuery(query);
-        Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("A");
+        Map<Operator, DataBag> derivedData = pigserver.getExamples("A");
 
         assertTrue(derivedData != null);
 
@@ -121,7 +121,7 @@ public class TestExampleGenerator extend
         pigserver.registerQuery(query);
         query = "B = filter A by x > 10;";
         pigserver.registerQuery(query);
-        Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("B");
+        Map<Operator, DataBag> derivedData = pigserver.getExamples("B");
 
         assertTrue(derivedData != null);
 
@@ -137,7 +137,7 @@ public class TestExampleGenerator extend
         pigserver.registerQuery(query);
         query = "B = filter A by x > 5 AND y < 6;";
         pigserver.registerQuery(query);
-        Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("B");
+        Map<Operator, DataBag> derivedData = pigserver.getExamples("B");
 
         assertTrue(derivedData != null);
     }
@@ -158,7 +158,7 @@ public class TestExampleGenerator extend
         pigserver.registerQuery(query);
         query = "E = DISTINCT D;";
         pigserver.registerQuery(query);
-        Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("E");
+        Map<Operator, DataBag> derivedData = pigserver.getExamples("E");
 
         assertTrue(derivedData != null);
 
@@ -172,7 +172,7 @@ public class TestExampleGenerator extend
                 + " using PigStorage() as (x : int, y : int);");
         pigServer.registerQuery("B = foreach A generate x + y as sum;");
 
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
 
         assertTrue(derivedData != null);
     }
@@ -185,7 +185,7 @@ public class TestExampleGenerator extend
 
         pigServer.registerQuery("E = join A1 by x, B1 by x;");
 
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("E");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("E");
 
         assertTrue(derivedData != null);
     }
@@ -198,7 +198,7 @@ public class TestExampleGenerator extend
 
         pigServer.registerQuery("E = join A1 by x, B1 by x;");
 
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("E");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("E");
 
         assertTrue(derivedData != null);
     }
@@ -210,7 +210,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("A = load " + A + " as (x, y);");
         pigServer.registerQuery("B = load " + B + " as (x, y);");
         pigServer.registerQuery("C = cogroup A by (x, y), B by (x, y);");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
 
         assertTrue(derivedData != null);
     }
@@ -221,7 +221,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("A = load " + A + " as (x, y);");
         pigServer.registerQuery("B = load " + B + " as (x, y);");
         pigServer.registerQuery("C = cogroup A by x, B by x;");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
 
         assertTrue(derivedData != null);
     }
@@ -231,7 +231,7 @@ public class TestExampleGenerator extend
         PigServer pigServer = new PigServer(pigContext);
         pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
         pigServer.registerQuery("B = group A by x;");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
 
         assertTrue(derivedData != null);
 
@@ -243,7 +243,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("A = load " + A.toString() + " as (x:int, y:int);");
         pigServer.registerQuery("B = group A by x;");
         pigServer.registerQuery("C = foreach B generate group, COUNT(A);};");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
 
         assertTrue(derivedData != null);
 
@@ -256,7 +256,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("B = FILTER A by x  > 3;");
         pigServer.registerQuery("C = group B by y;");
         pigServer.registerQuery("D = foreach C generate group, COUNT(B);};");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("D");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("D");
 
         assertTrue(derivedData != null);
 
@@ -269,7 +269,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("B = FILTER A by x  > 3;");
         pigServer.registerQuery("C = FILTER A by x < 3;");
         pigServer.registerQuery("D = UNION B, C;");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("D");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("D");
 
         assertTrue(derivedData != null);
 
@@ -281,7 +281,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("A = load " + A.toString() + " as (x:int, y:int);");
         pigServer.registerQuery("B = group A by x;");
         pigServer.registerQuery("C = foreach B { FA = filter A by y == 6; generate group, COUNT(FA);};");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
 
         assertTrue(derivedData != null);
 
@@ -293,7 +293,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("A = load " + A.toString() + " as (x:int, y:int);");
         pigServer.registerQuery("B = group A by x;");
         pigServer.registerQuery("C = foreach B { FA = filter A by y == 6; DA = DISTINCT FA; generate group, COUNT(DA);};");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
 
         assertTrue(derivedData != null);
 
@@ -305,7 +305,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
         pigServer.registerQuery("B = load " + B.toString() + " as (x, y);");
         pigServer.registerQuery("C = union A, B;");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
 
         assertTrue(derivedData != null);
     }
@@ -315,7 +315,7 @@ public class TestExampleGenerator extend
         PigServer pigServer = new PigServer(pigContext);
         pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
         pigServer.registerQuery("B = DISTINCT A;");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
 
         assertTrue(derivedData != null);
     }
@@ -326,7 +326,7 @@ public class TestExampleGenerator extend
         pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
         pigServer.registerQuery("B = load " + B.toString() + " as (x, y);");
         pigServer.registerQuery("C = CROSS A, B;");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
 
         assertTrue(derivedData != null);
     }
@@ -336,7 +336,7 @@ public class TestExampleGenerator extend
         PigServer pigServer = new PigServer(pigContext);
         pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
         pigServer.registerQuery("B = limit A 5;");
-        Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+        Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
 
         assertTrue(derivedData != null);
     }

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld Fri Dec 17 22:08:08 2010
@@ -18,4 +18,4 @@ Reduce Plan Empty
     |   |   |
     |   |   Project[tuple][*] - scope-126
     |   |
-    |   |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaabfkaaangfgogeepggebgmgmejgohahfhehihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgoeaondlnechegpmlgacaaaiemaaakgneghcgpgnefgeghgfhdheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlemaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaognfdgpggheeghcgpgnefgeghgfhdhbaahoaaaeemaaamgnfdgpgghefegpefgeghgfhdhbaahoaaaeemaaaignfegpefgeghgfhdhbaahoaaaehihahdhcaacbgphcghcogbhagb
 gdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbhggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahhheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaaffkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghfkaaaehdhegbhcemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhedlhihcaagcgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphac
 ogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhdgjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaamfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgbgmgjgbhdhbaahoaaaoemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhj
 gfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaacecaaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdhfgmheheaabcemgkgbhggbcpgmg
 bgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaabejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahi','','b','scope','true')) - scope-117
\ No newline at end of file
+    |   |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaacfkaaangfgogeepggebgmgmejgohahfheemaaafgphagngbhaheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlhihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgoeaondlnechegpmlgacaaaiemaaakgneghcgpgnefgeghgfhdhbaahoaaademaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaognfdgpggheeghcgpgnefgeghgfhdhbaahoaaademaaamgnfdgpgghefegpefgeghgfhdhbaahoaaademaaaignfegpefgeghgfhdhbaahoaaad
 hihahdhcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbhggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahhheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaaffkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghfkaaaehdhegbhcemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhedlhihcaagcgphcghcogbhagbgdgigfcohagjghcogcg
 bgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhdgjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaamfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgbgmgjgbhdhbaahoaaaoemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgo
 gfcphagihjhdgjgdgbgmemgbhjgfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaacecaaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdh
 fgmheheaabcemgkgbhggbcpgmgbgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaabejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahahi','','b','scope','true')) - scope-117



Mime
View raw message