pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r662844 [1/2] - in /incubator/pig/branches/types: ./ src/org/apache/pig/impl/logicalLayer/ src/org/apache/pig/impl/logicalLayer/schema/ src/org/apache/pig/impl/plan/ test/org/apache/pig/test/ test/org/apache/pig/test/data/DotFiles/ test/org...
Date Tue, 03 Jun 2008 17:21:59 GMT
Author: gates
Date: Tue Jun  3 10:21:59 2008
New Revision: 662844

URL: http://svn.apache.org/viewvc?rev=662844&view=rev
Log:
PIG-249 Pi's graph comparison utilities.


Added:
    incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java
    incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/
    incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot
    incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot
    incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot
    incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot
    incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot
    incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/planComparer/
    incubator/pig/branches/types/test/org/apache/pig/test/utils/planComparer/LogicalPlanComparer.java
    incubator/pig/branches/types/test/org/apache/pig/test/utils/planComparer/PlanStructuralComparer.java
Modified:
    incubator/pig/branches/types/build.xml
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
    incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java
    incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
    incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java

Modified: incubator/pig/branches/types/build.xml
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Tue Jun  3 10:21:59 2008
@@ -17,10 +17,6 @@
     <property name="src.lib.dir" value="${basedir}/lib-src/" />
     <property name="src.gen.dir" value="${basedir}/src-gen/" />
 
-    <!-- javacc properties -->
-    <property name="src.gen.query.parser.dir" value="${src.gen.dir}/org/apache/pig/impl/logicalLayer/parser" />
-    <property name="src.gen.script.parser.dir" value="${src.gen.dir}/org/apache/pig/tools/pigscript/parser" />
-
     <property name="javacc.home" value="${basedir}/lib" />
 
     <!-- javac properties -->
@@ -63,6 +59,11 @@
     <property name="test.log.dir" value="${basedir}/test/logs"/>
     <property name="junit.hadoop.conf" value="${user.home}/pigtest/conf/"/>
 
+    <!-- javacc properties -->
+    <property name="src.gen.query.parser.dir" value="${src.gen.dir}/org/apache/pig/impl/logicalLayer/parser" />
+    <property name="src.gen.script.parser.dir" value="${src.gen.dir}/org/apache/pig/tools/pigscript/parser" />
+    <property name="src.gen.dot.parser.dir" value="${test.src.dir}/org/apache/pig/test/utils/dotGraph/parser" />
+
     <!-- ====================================================== -->
     <!-- Stuff needed by all targets                            -->
     <!-- ====================================================== -->
@@ -88,6 +89,7 @@
         <mkdir dir="${dist.dir}" />
         <mkdir dir="${build.classes}" />
         <mkdir dir="${test.build.classes}" />
+        <mkdir dir="${src.gen.dot.parser.dir}" />
     </target>
 
     <!-- ================================================================== -->
@@ -96,6 +98,7 @@
     <target name="clean">
         <delete dir="${src.gen.dir}" />
         <delete dir="${build.dir}" />
+        <delete dir="${src.gen.dot.parser.dir}" />
     </target>
 
     <!-- ================================================================== -->
@@ -105,6 +108,8 @@
         <jjtree target="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt" outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
         <javacc target="${src.gen.query.parser.dir}/QueryParser.jj" outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
         <javacc target="${src.dir}/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj" outputdirectory="${src.gen.script.parser.dir}" javacchome="${javacc.home}" />
+        <jjtree target="${test.src.dir}/org/apache/pig/test/utils/dotGraph/Dot.jjt" outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
+        <javacc target="${src.gen.dot.parser.dir}/Dot.jj" outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
     </target>
 
     <!-- ================================================================== -->
@@ -238,6 +243,7 @@
                     <!-- Excluced because we don't want to run them -->
                     <exclude name="**/TypeCheckingTestUtil.java" />
                     <exclude name="**/TypeGraphPrinter.java" />
+                    <exclude name="**/LogicalPlanTester.java" />
                     <exclude name="**/TestHelper.java" />
                     <exclude name="**/TestLargeFile.java" />
                     <exclude name="**/TestOrderBy.java" />

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java Tue Jun  3 10:21:59 2008
@@ -238,4 +238,13 @@
     public LogicalPlan getPlan() {
         return mPlan ;
     }
+
+
+    /***
+     * IMPORTANT:
+     * This method is only used for unit testing purpose.
+     */
+    public void setSchemaComputed(boolean computed) {
+       mIsSchemaComputed = computed ;   
+    }
 }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java Tue Jun  3 10:21:59 2008
@@ -517,6 +517,13 @@
                                  Schema other,
                                  boolean relaxInner,
                                  boolean relaxAlias) {
+
+        // If both of them are null, they are equal
+        if ((schema == null) && (other == null)) {
+            return true ;
+        }
+
+        // otherwise
         if (schema == null) {
             return false ;
         }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java Tue Jun  3 10:21:59 2008
@@ -47,4 +47,8 @@
         
         return nextId;
     }
+
+    public static void reset(String scope) {
+        theGenerator.scopeToIdMap.put(scope, 0L) ;
+    }
 }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java Tue Jun  3 10:21:59 2008
@@ -107,21 +107,19 @@
      * Get the map of operator key and associated operators
      * @return map of operator key and operators.
      */
-/*
     public Map<OperatorKey, E> getKeys() {
         return mKeys;
     }
-*/
+
     /**
      * Get the map of operators and associated operator keys
      * @return map of operator and operator keys.
      */
-/*
+    /*
     public Map<E, OperatorKey> getOps() {
         return mOps;
     }
-*/
-    
+    */
     /**
      * Insert an operator into the plan.  This only inserts it as a node in
      * the graph, it does not connect it to any other operators.  That should
@@ -343,5 +341,9 @@
         return tmpList.size() == 1 ;
     }
 
+    public int size() {
+        return mKeys.size() ;
+    }
+
 
 }

Added: incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import org.junit.Test;
+import org.apache.pig.impl.logicalLayer.parser.QueryParser;
+import org.apache.pig.impl.logicalLayer.parser.ParseException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.data.DataType;
+
+import java.io.ByteArrayInputStream;
+
+import junit.framework.TestCase;
+
+
+public class TestSchemaParser extends TestCase {
+
+    @Test
+    public void test1() throws ParseException {
+        String test = "fieldA: int, fieldB:double,fieldC:bYteaRRay, fieldD:charARRAY";
+        Schema schema = parseSchema(test) ;
+        assertEquals(schema.size(), 4) ;
+        assertAliases(schema, "fieldA", "fieldB", "fieldC", "fieldD") ;
+        assertTypes(schema, DataType.INTEGER, DataType.DOUBLE,
+                            DataType.BYTEARRAY, DataType.CHARARRAY);
+    }
+
+    @Test
+    public void test2() throws ParseException {
+        String test = "fieldA: bag {tuple1:tuple(a:int,b:long,c:float,d:double)}";
+        Schema schema = parseSchema(test) ;
+
+        assertEquals(schema.size(), 1);
+        assertEquals(schema.getField(0).type, DataType.BAG) ;
+        assertEquals(schema.getField(0).alias, "fieldA") ;
+
+        Schema inner = schema.getField(0).schema ;
+        assertEquals(inner.size(), 1) ;
+        assertAliases(inner.getField(0).schema, "a", "b", "c", "d") ;
+        assertTypes(inner.getField(0).schema, DataType.INTEGER, DataType.LONG,
+                            DataType.FLOAT, DataType.DOUBLE);
+
+    }
+
+   @Test
+    public void test3() throws ParseException {
+        String test = "tuple1: tuple(a:chararray,b:long),"
+                    +" tuple2: tuple(c:int,d:float) ";
+        Schema schema = parseSchema(test) ;
+
+        assertEquals(schema.size(), 2);
+        assertAliases(schema, "tuple1", "tuple2") ;
+        assertTypes(schema, DataType.TUPLE, DataType.TUPLE);
+
+        Schema inner1 = schema.getField(0).schema ;
+        assertEquals(inner1.size(), 2);
+        assertAliases(inner1, "a", "b") ;
+        assertTypes(inner1, DataType.CHARARRAY, DataType.LONG);
+
+        Schema inner2 = schema.getField(1).schema ;
+        assertEquals(inner2.size(), 2);
+        assertAliases(inner2, "c", "d") ;
+        assertTypes(inner2, DataType.INTEGER, DataType.FLOAT) ;
+
+    }
+
+    @Test
+    public void test4() throws ParseException {
+        String test = "garage: bag{tuple1: tuple(num_tools: int)}, links: int";
+        Schema schema = parseSchema(test) ;
+
+        // the schema
+        assertEquals(schema.size(), 2) ;
+        assertAliases(schema, "garage", "links") ;
+        assertTypes(schema, DataType.BAG, DataType.INTEGER);
+        assertEquals(schema.getField(0).type, DataType.BAG) ;
+
+        // inner schema
+        Schema inner = schema.getField(0).schema ;
+        assertEquals(inner.size(), 1) ;
+        assertEquals(inner.getField(0).type, DataType.TUPLE) ;
+        assertEquals(inner.getField(0).alias, "tuple1") ;
+
+        // inner of inner
+        Schema innerInner = inner.getField(0).schema ;
+        assertEquals(innerInner.size(), 1) ;
+        assertEquals(innerInner.getField(0).type, DataType.INTEGER) ;
+    }
+
+    private void assertAliases(Schema schema, String... aliases)
+                                            throws ParseException {
+        for(int i=0; i < aliases.length;i++) {
+            assertEquals(schema.getField(i).alias, aliases[i]);
+        }
+    }
+
+    private void assertTypes(Schema schema, byte... types)
+                                            throws ParseException {
+        for(int i=0; i < types.length;i++) {
+            assertEquals(schema.getField(i).type, types[i]);
+        }
+    }
+
+    private Schema parseSchema(String schemaString)
+                                            throws ParseException {
+        ByteArrayInputStream stream = new ByteArrayInputStream(schemaString.getBytes()) ;
+        QueryParser parser = new QueryParser(stream) ;
+        return parser.TupleSchema() ;
+    }
+}

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java Tue Jun  3 10:21:59 2008
@@ -20,30 +20,92 @@
 
 import junit.framework.TestCase;
 import org.apache.pig.impl.logicalLayer.*;
-import org.apache.pig.impl.logicalLayer.validators.TypeCheckingValidator;
-import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.plan.CompilationMessageCollector;
-import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.impl.plan.PlanValidationException;
-import org.apache.pig.ExecType;
 import org.apache.pig.test.utils.TypeCheckingTestUtil;
+import org.apache.pig.test.utils.LogicalPlanTester;
 import org.junit.Test;
-
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.io.IOException;
+import org.junit.Before;
 
 public class TestTypeChecking extends TestCase {
 
+    final String FILE_BASE_LOCATION = "test/org/apache/pig/test/data/DotFiles/" ;
+
+    LogicalPlanTester planTester = new LogicalPlanTester() ;
+
+    @Before
+    public void setUp() {
+        planTester.reset();
+    }
+
+
+    @Test
+    public void testSimple1() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.buildPlan("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+        LogicalPlan plan = planTester.buildPlan("b = distinct a ;") ;
+        planTester.typeCheckAgainstDotFile(plan, FILE_BASE_LOCATION + "plan1.dot");
+    }
+
+
+    @Test
+    public void testByScript1() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript1.dot");
+    }
+
+    @Test
+    public void testByScript2() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript2.dot");
+    }
+
+    // Problem with schema parser in QueryParser
+    /*
+        @Test
+    public void testByScript4() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript4.dot");
+    }
+    */
+
+    /*
+
+    @Test
+    public void testByScript3() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript3.dot");
+    }
+
+    */
+
+    // TODO: Convert all of these to dot files
+
+    /*
     @Test
     public void testValidation1() throws Throwable {
         TypeCheckingTestUtil.printCurrentMethodName() ;
-        buildPlan("a = load 'a' as (field1: chararray, field2: tuple(inner1 : bytearray, inner2 : int));");
-        LogicalPlan plan = buildPlan("b = group a by field1;");
-        validatePlan(plan) ;
+        planTester.buildPlan("a = load 'a' as (" 
+                            + "group: tuple(field1:int, field2:double) ,"
+                            + "a: bag{tuple1:tuple(field1: int,field2: long)},"
+                            + "b: bag{tuple1:tuple(field1: bytearray,field2: double)} "
+                            + " ) ;") ;
+        LogicalPlan plan = planTester.buildPlan("b = group a by field1;");
+        planTester.typeCheckPlan(plan);
+
+
+    }
+    */
+
+    /*
+    @Test
+    public void testValidation1() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.buildPlan("a = load 'a' as (field1: chararray, field2: tuple(inner1 : bytearray, inner2 : int));");
+        LogicalPlan plan = planTester.buildPlan("b = group a by field1;");
+        planTester.typeCheckPlan(plan);
     }
+    */
 
+    /*
     @Test
     public void testValidation2() throws Throwable {
         TypeCheckingTestUtil.printCurrentMethodName() ;
@@ -119,7 +181,7 @@
 		LogicalPlan plan = buildPlan("e = foreach a generate name, details.(age, gpa), field3.(a,b) ;");
         validatePlan(plan) ;
     }
-    
+
     @Test
     public void testValidation11_2() throws Throwable {
         TypeCheckingTestUtil.printCurrentMethodName() ;
@@ -137,6 +199,7 @@
         LogicalPlan plan = buildPlan("d = foreach c generate a.(field1, field2), b.(field1, field2)  ;");
         validatePlan(plan) ;
     }
+    */
 
     // I suspect there is something wrong in Schema.reconcile()
     /*
@@ -164,6 +227,7 @@
     }
     */
 
+    /*
    @Test
     public void testQuery20() throws Throwable {
        TypeCheckingTestUtil.printCurrentMethodName() ;
@@ -179,62 +243,9 @@
         String query = "b = foreach a generate (field1+field2)*(field1-field2) ;";
         validatePlan(buildPlan(query));
     }
-    
-    /***************** Helpers ******************/
 
-    public LogicalPlan buildPlan(String query) {
-        return buildPlan(query, LogicalPlanBuilder.class.getClassLoader());
-    }
+    */
+
 
-    public LogicalPlan buildPlan(String query, ClassLoader cldr) {
 
-        LogicalPlanBuilder.classloader = TestTypeChecking.class.getClassLoader() ;
-        PigContext pigContext = new PigContext(ExecType.LOCAL);
-        LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
-
-        try {
-            LogicalPlan lp = builder.parse("Test-Plan-Builder",
-                                           query,
-                                           aliases,
-                                           logicalOpTable,
-                                           aliasOp,
-                                           defineAliases);
-
-            List<LogicalOperator> roots = lp.getRoots();
-
-            if(roots.size() > 0) {
-                if (logicalOpTable.get(roots.get(0)) instanceof LogicalOperator){
-                    System.out.println(query);
-                    System.out.println(logicalOpTable.get(roots.get(0)));
-                }
-                if ((roots.get(0)).getAlias()!=null){
-                    aliases.put(roots.get(0), lp);
-                }
-            }
-
-            assertTrue(lp != null);
-
-            return lp ;
-        }
-        catch (IOException e) {
-            fail("IOException: " + e.getMessage());
-        }
-        catch (Exception e) {
-            fail(e.getClass().getName() + ": " + e.getMessage() + " -- " + query);
-        }
-        return null;
-    }
-
-    public void validatePlan(LogicalPlan plan) throws PlanValidationException {
-        CompilationMessageCollector collector = new CompilationMessageCollector() ;
-        TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
-        typeValidator.validate(plan, collector) ;
-        TypeCheckingTestUtil.printMessageCollector(collector) ;
-        TypeCheckingTestUtil.printTypeGraph(plan) ;
-    }
-
-    Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
-    Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
-    Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
-    Map<String, ExpressionOperator> defineAliases = new HashMap<String, ExpressionOperator>();
 }

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java Tue Jun  3 10:21:59 2008
@@ -19,9 +19,12 @@
 
 import java.util.Iterator;
 import java.util.List;
+import java.util.ArrayList;
 
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.logicalLayer.parser.ParseException ;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.* ;
 
 /**
@@ -31,11 +34,18 @@
 public class TypeGraphPrinter extends LOVisitor {
     
     private StringBuilder sb = null ;
+    private int currentTabCount = 0 ;
     
     public TypeGraphPrinter(LogicalPlan plan) {
         super(plan, new DependencyOrderWalker<LogicalOperator, LogicalPlan>(plan));
         sb = new StringBuilder() ;
     }
+
+    private void printTabs() {
+        for (int i=0; i< currentTabCount; i++) {
+            sb.append("\t") ;
+        }
+    }
     
     protected void visit(LogicalOperator op) {
         appendOp(op) ;
@@ -53,12 +63,42 @@
         appendOp(op) ;
     }
     
-    protected void visit(LOCogroup op) {
+    protected void visit(LOCogroup op) throws VisitorException {
         appendOp(op) ;
+        List<LogicalOperator> inputs = op.getInputs() ;
+        if (inputs != null) {
+            for(LogicalOperator input: inputs) {
+                List<LogicalPlan> plans
+                    = new ArrayList<LogicalPlan>(op.getGroupByPlans().get(input)) ;
+                if (plans != null) {
+                    for(LogicalPlan plan: plans) {
+                        currentTabCount++ ;
+                        printTabs() ;
+                        sb.append("<COGroup Inner Plan>\n") ;
+                        pushWalker(mCurrentWalker.spawnChildWalker(plan)) ;
+                        visit();
+                        popWalker();
+                        currentTabCount-- ;
+                    }
+                }
+            }
+        }
     }
     
-    protected void visit(LOGenerate op) {
+    protected void visit(LOGenerate op) throws VisitorException {
         appendOp(op) ;
+        List<LogicalPlan> plans = op.getGeneratePlans() ;
+        if (plans != null) {
+            for (LogicalPlan plan: plans) {
+                currentTabCount++ ;
+                printTabs() ;
+                sb.append("<Generate Inner Plan>\n") ;
+                pushWalker(mCurrentWalker.spawnChildWalker(plan)) ;
+                visit();
+                popWalker();
+                currentTabCount-- ;
+            }
+        }
     }
     
     protected void visit(LOSort op) {
@@ -67,19 +107,32 @@
     
     protected void visit(LOFilter op) throws VisitorException {
         appendOp(op) ;
-        sb.append("Filter Inner Plan:\n") ;
-        pushWalker(mCurrentWalker.spawnChildWalker(op.getComparisonPlan())) ;
-        visit();
-        popWalker();
-        sb.append("\n") ;
+        if (op.getComparisonPlan() != null) {
+            currentTabCount++ ;
+            printTabs() ;
+            sb.append("<Filter Inner Plan>\n") ;
+            pushWalker(mCurrentWalker.spawnChildWalker(op.getComparisonPlan())) ;
+            visit();
+            popWalker();
+            currentTabCount-- ;
+        }
     }
     
     protected void visit(LOSplit op) {
         appendOp(op) ;
     }
     
-    protected void visit(LOForEach op) {
+    protected void visit(LOForEach op) throws VisitorException {
         appendOp(op) ;
+        if (op.getForEachPlan() != null) {
+            currentTabCount++ ;
+            printTabs() ;
+            sb.append("<ForEach Inner Plan>\n") ;
+            pushWalker(mCurrentWalker.spawnChildWalker(op.getForEachPlan())) ;
+            visit();
+            popWalker();
+            currentTabCount-- ;
+        }
     }
     
     protected void visit(LOUserFunc op) {
@@ -89,6 +142,14 @@
     protected void visit(LOBinCond op) {
         appendOp(op) ;
     }
+
+    protected void visit(LOCross op) {
+        appendOp(op) ;
+    }
+    
+    protected void visit(LOUnion op) {
+        appendOp(op) ;
+    }
     
     protected void visit(LOCast op) { 
         appendOp(op) ;
@@ -119,15 +180,42 @@
     }
 
 
-    private void appendOp(LogicalOperator op) {
+    private void appendOp(LogicalOperator op)  {
+        printTabs() ;
         sb.append("(") ;
         sb.append(op.getOperatorKey().getId()) ;
         sb.append(":") ;
         sb.append(op.getClass().getSimpleName()) ;
         sb.append("=") ;
-        sb.append(DataType.findTypeName(op.getType())) ;
-        sb.append("{") ;
-        List<LogicalOperator> list = mPlan.getSuccessors(op) ;
+        Schema schema = null ;
+
+        if (!DataType.isComplex(op.getType())) {
+            sb.append(DataType.findTypeName(op.getType())) ;
+        }
+        else {
+            try {
+                if (op instanceof ExpressionOperator) {
+                    ExpressionOperator eOp = (ExpressionOperator) op ;
+                    Schema.FieldSchema fs = eOp.getFieldSchema() ;
+                    Schema.stringifySchema(sb, fs.schema, DataType.TUPLE) ;
+                }
+                else {
+                    schema = op.getSchema() ;
+                    Schema.stringifySchema(sb, schema, op.getType()) ;
+                }
+            }
+            catch (FrontendException fe) {
+                throw new RuntimeException("PROBLEM PRINTING SCHEMA") ;
+            }
+            catch (ParseException pe) {
+                throw new RuntimeException("PROBLEM PRINTING SCHEMA") ;
+            }
+        }
+
+        sb.append("==>") ;
+        List<LogicalOperator> list
+                    = mCurrentWalker.getPlan().getSuccessors(op) ;
+
         if (list!=null) {
             boolean isFirst = true ;
             for(LogicalOperator tmp: list) {         
@@ -140,7 +228,9 @@
                 sb.append(tmp.getOperatorKey().getId()) ;                
             }
         }
-        sb.append("}") ;
+        else {
+            sb.append("TERMINAL") ;
+        }
         sb.append(")") ;
         sb.append("\n") ;
     }

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot Tue Jun  3 10:21:59 2008
@@ -0,0 +1,10 @@
+digraph graph1 {
+    graph [pigScript=" A = LOAD 'tmp' ; B = DISTINCT A ; "] ;
+
+    node [schema="field1: int, field2: float, field3: chararray"] ;
+
+    load [key="10", type="LOLoad"] ;
+    distinct [key="12", type="LODistinct"] ;
+
+    load -> distinct ;
+}
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot Tue Jun  3 10:21:59 2008
@@ -0,0 +1,10 @@
+digraph graph1 {
+    graph [pigScript=" A = LOAD 'tmp' as (field1: int, field2: float, field3: chararray ) ; B = DISTINCT A ; "] ;
+
+    node [schema="field1: int, field2: float, field3: chararray"] ;
+
+    load [key="10", type="LOLoad"] ;
+    distinct [key="20", type="LODistinct"] ;
+
+    load -> distinct ;
+}
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot Tue Jun  3 10:21:59 2008
@@ -0,0 +1,8 @@
+digraph graph1 {
+    graph [pigScript=" A = LOAD 'tmp' ; B = DISTINCT A ; "] ;
+
+    load [key="10", type="LOLoad"] ;
+    distinct [key="20", type="LODistinct"] ;
+
+    load -> distinct ;
+}
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot Tue Jun  3 10:21:59 2008
@@ -0,0 +1,9 @@
+digraph graph1 {
+    graph [pigScript=" a = load 'a' as (field1: long, field2: tuple(inner1 : bytearray, inner2 : float) ) ;
+                       b = group a by field2; "] ;
+
+    load [key="10", type="LOLoad", schema="field1: long, field2: tuple(inner1 : bytearray, inner2 : float)"] ;
+    group [key="20", type="LOCogroup" , schema="group: chararray,a: bag{ tuple1:tuple(field1: long,field2: tuple(inner1: bytearray,inner2: integer))}"] ;
+
+    load -> group ;
+}
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot Tue Jun  3 10:21:59 2008
@@ -0,0 +1,14 @@
+digraph graph1 {
+    graph [pigScript=" a = load 'a' as (field1: int, field2: long);
+                       b = load 'a' as (field1: bytearray, field2: double);
+                       c = group a by (field1,field2) , b by (field1,field2) ; "] ;
+
+    load1 [key="10", type="LOLoad", schema="field1: int, field2: long"] ;
+    load2 [key="15", type="LOLoad", schema="field1: bytearray, field2: double"] ;
+    group [key="20", type="LOCogroup" , schema="group: tuple(field1:int, field2:double) ,
+                                                a: bag{tuple1:tuple(field1: int,field2: long)},
+                                                b: bag{tuple1:tuple(field1: bytearray,field2: double)} "] ;
+
+    load1 -> group ;
+    load2 -> group ;
+}
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils;
+
+import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.logicalLayer.validators.TypeCheckingValidator;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.PlanValidationException;
+import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.NodeIdGenerator;
+import org.apache.pig.ExecType;
+import static org.apache.pig.test.utils.TypeCheckingTestUtil.* ;
+import org.apache.pig.test.utils.dotGraph.LogicalPlanLoader;
+import org.apache.pig.test.utils.planComparer.LogicalPlanComparer;
+import org.apache.pig.test.utils.dotGraph.DotGraphReader;
+import org.apache.pig.test.utils.dotGraph.DotGraph;
+
+import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.io.IOException;
+
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
+
+/***
+ * This class is used for logical plan testing
+ */
+public class LogicalPlanTester {
+
+    static final String SCOPE = "scope" ;
+
+    private Map<LogicalOperator, LogicalPlan> aliases  = null ;
+    private Map<OperatorKey, LogicalOperator> logicalOpTable = null ;
+    private Map<String, LogicalOperator> aliasOp = null ;
+    private Map<String, ExpressionOperator> defineAliases = null;
+
+    public LogicalPlanTester() {
+        reset() ;
+    }
+
+    /***
+     * Reset state
+     */
+    public void reset() {
+        aliases = new HashMap<LogicalOperator, LogicalPlan>();
+        logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
+        aliasOp = new HashMap<String, LogicalOperator>();
+        defineAliases = new HashMap<String, ExpressionOperator>();
+        NodeIdGenerator.reset(SCOPE);
+    }
+
+    /***
+     * Build plan by the given query string (Pig script)
+     * @param query
+     * @return
+     */
+    public LogicalPlan buildPlan(String query) {
+        return buildPlan(query, LogicalPlanBuilder.class.getClassLoader());
+    }
+
+
+    /***
+     * Type check the given plan
+     * @param plan
+     * @throws PlanValidationException
+     */
+    public void typeCheckPlan(LogicalPlan plan) throws PlanValidationException {
+        CompilationMessageCollector collector = new CompilationMessageCollector() ;
+        TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
+        typeValidator.validate(plan, collector) ;
+        printMessageCollector(collector) ;
+        System.out.println("Actual plan:") ;
+        printTypeGraph(plan) ;
+    }
+
+    /***
+     * Run type checking and compare the result with  plan structure
+     * stored in Dot file
+     * @param plan
+     * @param file
+     * @throws PlanValidationException
+     */
+    public void typeCheckAgainstDotFile(LogicalPlan plan, String file)
+                                            throws PlanValidationException {
+        // validate the given plan
+        typeCheckPlan(plan);
+
+        // load the expected plan from file
+        LogicalPlanLoader planLoader = new LogicalPlanLoader() ;
+        LogicalPlan expectedPlan = planLoader.loadFromFile(file, LogicalPlan.class) ;
+        System.out.println("Expected plan:") ;
+        printTypeGraph(expectedPlan) ;
+
+        // do the comparison
+        LogicalPlanComparer comparer = new LogicalPlanComparer() ;
+        StringBuilder errMsg = new StringBuilder() ;
+        boolean result = comparer.structurallyEquals(plan, expectedPlan, errMsg) ;
+
+        // check
+        System.out.println(errMsg.toString()) ;
+        assertTrue("The expected plan is different", result);
+        System.out.println("Checking DONE!") ;
+    }
+
+    public void typeCheckUsingDotFile(String file)
+                                            throws PlanValidationException {
+        DotGraphReader reader = new DotGraphReader() ;
+        DotGraph graph = reader.loadFromFile(file) ;
+        if (!graph.attributes.containsKey("pigScript")) {
+            throw new AssertionError("pigScript attribute doesn't exist"
+                                     + " in Dot file") ;
+        }
+
+        String script = graph.attributes.get("pigScript") ;
+        // TODO: Script splitting here is a quick hack.
+        String[] queries = script.split(";") ;
+        LogicalPlan plan = null ;
+        for(String query : queries) {
+            if (!query.trim().equals("")) {
+                plan = buildPlan(query + ";") ;
+            }
+        }
+        typeCheckAgainstDotFile(plan, file) ;
+
+    }
+
+    ////////////// Helpers ////////////////
+
+    // The actual plan builder
+    private LogicalPlan buildPlan(String query, ClassLoader cldr) {
+
+        LogicalPlanBuilder.classloader = LogicalPlanTester.class.getClassLoader() ;
+        PigContext pigContext = new PigContext(ExecType.LOCAL);
+        LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
+
+        try {
+            LogicalPlan lp = builder.parse(SCOPE,
+                                           query,
+                                           aliases,
+                                           logicalOpTable,
+                                           aliasOp,
+                                           defineAliases);
+
+            List<LogicalOperator> roots = lp.getRoots();
+
+            if(roots.size() > 0) {
+                if (logicalOpTable.get(roots.get(0)) instanceof LogicalOperator){
+                    System.out.println(query);
+                    System.out.println(logicalOpTable.get(roots.get(0)));
+                }
+                if ((roots.get(0)).getAlias()!=null){
+                    aliases.put(roots.get(0), lp);
+                }
+            }
+
+            assertTrue(lp != null);
+
+            return lp ;
+        }
+        catch (IOException e) {
+            fail("IOException: " + e.getMessage());
+        }
+        catch (Exception e) {
+            fail(e.getClass().getName() + ": " + e.getMessage() + " -- " + query);
+        }
+        return null;
+    }
+
+
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt Tue Jun  3 10:21:59 2008
@@ -0,0 +1,203 @@
+options {
+  // Generate non-static functions
+  STATIC = false;
+  // Case is ignored in keywords
+  IGNORE_CASE = true;
+}
+
+PARSER_BEGIN(DOTParser)
+
+package org.apache.pig.test.utils.dotGraph.parser ;
+
+import java.util.*;
+import java.io.*;
+import org.apache.pig.test.utils.dotGraph.* ;
+
+public class DOTParser {
+
+    static String unquote(String s) {
+        return s.substring(1, s.length()-1);
+    }
+    
+    static class DotState {
+        public Map<String,String> nodeAttributes = new HashMap<String,String>() ;
+        public Map<String,String> edgeAttributes = new HashMap<String,String>() ;
+    }
+
+}
+
+PARSER_END(DOTParser)
+
+// Skip all the new lines, tabs and spaces
+SKIP : { " " |	"\r" |	"\t" |	"\n" }
+
+MORE :
+{
+  "//" : SINGLE_COMMENT
+  |
+  "#" : SINGLE_COMMENT
+  |
+  "/*" : MULTI_COMMENT
+}
+
+<MULTI_COMMENT> SPECIAL_TOKEN  :
+{
+	<("\n" | "\r" | "\r\n")>
+	|
+	<"*/"> : DEFAULT
+}
+
+<SINGLE_COMMENT> SPECIAL_TOKEN :
+{
+  < ("\n" | "\r" | "\r\n") > : DEFAULT
+}
+
+<MULTI_COMMENT> MORE :
+{
+	< ~[] >
+}
+
+<SINGLE_COMMENT> MORE :
+{
+	< ~[] >
+}
+
+
+TOKEN:
+{
+
+      <LPAREN : "{">
+    | <RPAREN : "}">
+    | <LSQBRACKET: "[">
+    | <RSQBRACKET: "]">
+    | <EQUAL: "=">
+    | <COMMA: ",">
+    | <SEMICOLON: ";">
+    | <DIRECTED_EDGE: "->">
+    | <EDGE: "edge">
+    | <NODE: "node">
+    | <GRAPH: "graph">
+    | <DIGRAPH : "digraph">
+    | <#LETTER : ["a"-"z", "A"-"Z"] >
+    | <#DIGIT : ["0"-"9"] >
+    | <#SPECIAL_CHAR : "_" | "$" >
+    | <NAME :  <LETTER> ( <LETTER> | <DIGIT> | <SPECIAL_CHAR> )* >
+    | <QUOTEDSTRING : "\"" (~["\""])* "\"">
+}
+
+
+DotGraph Parse() :
+{
+    DotGraph dotGraph = null ;
+    DotState dotState = new DotState() ;
+	Token graphName ;
+}
+{
+	(
+		<DIGRAPH>
+		graphName = <NAME> { dotGraph = new DotGraph(graphName.image) ; }
+		<LPAREN>
+
+		(  LOOKAHEAD(2)
+		     EdgeStatement(dotGraph, dotState)
+		   | NodeStatement(dotGraph, dotState)
+		   | AttributeStatement(dotGraph, dotState)
+		)+
+
+		<RPAREN>
+	)
+	{ return dotGraph ; }
+}
+
+void AttributeStatement(DotGraph dotGraph, DotState dotState) :
+{
+    Map<String,String> attributes ;
+}
+{
+    (
+      ( <EDGE> attributes = AttributeList() { dotState.edgeAttributes = attributes ; } )
+    | ( <NODE> attributes = AttributeList() { dotState.nodeAttributes = attributes ; } )
+    | ( <GRAPH> attributes = AttributeList() { dotGraph.attributes = attributes ; } )
+    )
+    <SEMICOLON>
+}
+
+void NodeStatement(DotGraph dotGraph, DotState dotState) :
+{
+    Token nodeName ;
+    DotNode node = new DotNode() ;
+    Map<String,String> attributes ;
+}
+{
+    nodeName = <NAME> { node.name = nodeName.image ; }
+    ( attributes = AttributeList()  {
+                                        node.attributes = new HashMap<String,String>() ;
+                                        if (dotState != null) {
+                                            node.attributes.putAll(dotState.nodeAttributes) ;
+                                        }
+                                        node.attributes.putAll(attributes) ;
+                                    }
+    )?
+    <SEMICOLON>
+    { dotGraph.nodes.add(node) ; }
+}
+
+void EdgeStatement(DotGraph dotGraph, DotState dotState) :
+{
+    Token nodeName1 ;
+    Token nodeName2 ;
+    String startingNode ;
+    DotNode node = new DotNode() ;
+    Map<String,String> attributes ;
+}
+{
+    nodeName1 = <NAME> { startingNode = nodeName1.image ; }
+    (
+     <DIRECTED_EDGE>
+     nodeName2 = <NAME>
+     {
+        DotEdge edge = new DotEdge() ;
+        edge.fromNode = startingNode ;
+        edge.toNode = nodeName2.image ;
+
+        dotGraph.edges.add(edge) ;
+
+        startingNode = nodeName2.image ;
+     }
+    )+
+    <SEMICOLON>
+
+}
+
+Map<String,String> AttributeList() :
+{
+    Map<String,String> attributes = new HashMap<String,String>() ;
+    String[] keyValuePair ;
+}
+{
+    (
+    <LSQBRACKET>
+    (keyValuePair = Attribute() { attributes.put(keyValuePair[0], keyValuePair[1]) ; }  )
+        (
+            <COMMA>
+            (keyValuePair = Attribute() { attributes.put(keyValuePair[0], keyValuePair[1]) ; }  )
+        )*
+    <RSQBRACKET>
+    )
+    { return attributes ; }
+}
+
+String[] Attribute() :
+{
+    Token attName ;
+    Token value ;
+    String[] keyValuePair = new String[2] ;
+}
+{
+    (
+    attName = <NAME> { keyValuePair[0] = attName.image ; }
+    <EQUAL>
+    value = <QUOTEDSTRING> { keyValuePair[1] = unquote(value.image) ; }
+    )
+    { return keyValuePair ; }
+}
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+/**
+ * This represents an edge in DOT format.
+ * An edge in DOT can have attributes but we're not interested
+ */
+public class DotEdge {
+    public String fromNode ;
+    public String toNode ;
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+
+/***
+ * This represents graph structure in DOT format
+ */
+public class DotGraph {
+
+    public String name;
+    public List<DotEdge> edges = new ArrayList<DotEdge>() ;
+    public List<DotNode> nodes = new ArrayList<DotNode>() ;
+    public Map<String, String> attributes = new HashMap<String,String>() ;
+
+
+    public DotGraph(String name) {
+        this.name = name ;
+    }
+
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.test.utils.dotGraph.parser.DOTParser;
+import org.apache.pig.test.utils.dotGraph.parser.ParseException;
+
+import java.io.*;
+
+/***
+ * This class is responsible for loading textual Dot graph
+ * into object representation.
+ */
+public class DotGraphReader {
+
+    /***
+     * Load Dot graph from string
+     *
+     * @param dotContent the Dot content
+     * @return graph
+     */
+
+    public DotGraph load(String dotContent) {
+        ByteArrayInputStream stream
+                = new ByteArrayInputStream(dotContent.getBytes()) ;
+        DOTParser dotParser = new DOTParser(stream) ;
+        DotGraph graph = null ;
+        try {
+            graph = dotParser.Parse() ;
+        }
+        catch (ParseException pe) {
+            System.out.println(pe.getMessage()) ;
+            throw new RuntimeException("Bad Dot file") ;
+        }
+        return graph ;
+    }
+
+    /***
+     * Convenient method for loading Dot graph from text file
+     * @param file the file containing Dot content
+     * @return graph
+     */
+
+    public DotGraph loadFromFile(String file) {
+        StringBuilder sb = new StringBuilder() ;
+        BufferedReader br = null ;
+        try {
+            br = new BufferedReader(new FileReader(file)) ;
+            String str ;
+            while((str=br.readLine())!=null) {
+                sb.append(str) ;
+                sb.append("\n") ;
+            }
+        }
+        catch (FileNotFoundException fnfe) {
+            throw new RuntimeException("file:" + file + " not found!") ;
+        }
+        catch (IOException ioe) {
+            throw new RuntimeException("Error while reading from:" + file) ;
+        }
+
+        return load(sb.toString()) ;
+    }
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.plan.PlanWalker;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.plan.MultiMap;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
+
+import java.util.Iterator;
+
+/***
+ * Not implemented yet
+ */
+public class DotGraphVisitor extends LOVisitor {
+
+    public DotGraphVisitor(LogicalPlan plan,
+        PlanWalker<LogicalOperator, LogicalPlan> walker) {
+        super(plan, walker);
+    }
+
+    /**
+     * @param lOp
+     *            the logical operator that has to be visited
+     * @throws org.apache.pig.impl.plan.VisitorException
+     */
+    protected void visit(LogicalOperator lOp)
+            throws VisitorException {
+        //
+        // Do Nothing
+        //
+    }
+
+    /**
+     * @param eOp
+     *            the logical expression operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(ExpressionOperator eOp)
+            throws VisitorException {
+        //
+        // Do Nothing
+        //
+    }
+
+    /**
+     * @param binOp
+     *            the logical binary expression operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(BinaryExpressionOperator binOp)
+            throws VisitorException {
+        //
+        // Visit the left hand side operand followed by the right hand side
+        // operand
+        //
+
+        binOp.getLhsOperand().visit(this);
+        binOp.getRhsOperand().visit(this);
+    }
+
+    /**
+     *
+     * @param uniOp
+     *            the logical unary operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(UnaryExpressionOperator uniOp) throws VisitorException {
+        // Visit the operand
+
+        uniOp.getOperand().visit(this);
+    }
+
+    /**
+     *
+     * @param cg
+     *            the logical cogroup operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(LOCogroup cg) throws VisitorException {
+        // Visit each of the inputs of cogroup.
+        MultiMap<LogicalOperator, LogicalPlan> mapGByPlans = cg.getGroupByPlans();
+        for(LogicalOperator op: cg.getInputs()) {
+            for(LogicalPlan lp: mapGByPlans.get(op)) {
+                if (null != lp) {
+                    // TODO FIX - How do we know this should be a
+                    // DependencyOrderWalker?  We should be replicating the
+                    // walker the current visitor is using.
+                    PlanWalker w = new DependencyOrderWalker(lp);
+                    pushWalker(w);
+                    for(LogicalOperator logicalOp: lp.getRoots()) {
+                        logicalOp.visit(this);
+                    }
+                    popWalker();
+                }
+            }
+        }
+    }
+
+    /**
+     *
+     * @param g
+     *            the logical generate operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(LOGenerate g) throws VisitorException {
+        // Visit each of generates projection elements.
+        for(LogicalPlan lp: g.getGeneratePlans()) {
+            PlanWalker w = new DependencyOrderWalker(lp);
+            pushWalker(w);
+            for(LogicalOperator logicalOp: lp.getRoots()) {
+                logicalOp.visit(this);
+            }
+            popWalker();
+        }
+    }
+
+    /**
+     *
+     * @param s
+     *            the logical sort operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(LOSort s) throws VisitorException {
+        // Visit the sort function
+        for(LogicalPlan lp: s.getSortColPlans()) {
+            PlanWalker w = new DependencyOrderWalker(lp);
+            pushWalker(w);
+            for(LogicalOperator logicalOp: lp.getRoots()) {
+                logicalOp.visit(this);
+            }
+            popWalker();
+        }
+    }
+
+    /**
+     *
+     * @param filter
+     *            the logical filter operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(LOFilter filter) throws VisitorException {
+        // Visit the condition for the filter followed by the input
+        PlanWalker w = new DependencyOrderWalker(filter.getComparisonPlan());
+        pushWalker(w);
+        for(LogicalOperator logicalOp: filter.getComparisonPlan().getRoots()) {
+            logicalOp.visit(this);
+        }
+        popWalker();
+    }
+
+    /**
+     *
+     * @param split
+     *            the logical split operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(LOSplit split) throws VisitorException {
+        // Visit each of split's conditions
+        for(LogicalOperator logicalOp: split.getOutputs()) {
+            logicalOp.visit(this);
+        }
+    }
+
+    /**
+     *
+     * @param forEach
+     *            the logical foreach operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(LOForEach forEach) throws VisitorException {
+        // Visit the operators that are part of the foreach plan
+        LogicalPlan plan = forEach.getForEachPlan();
+        PlanWalker w = new DependencyOrderWalker(plan);
+        pushWalker(w);
+        for(LogicalOperator logicalOp: plan.getRoots()) {
+            logicalOp.visit(this);
+        }
+        popWalker();
+    }
+
+    /**
+     * Iterate over each expression that is part of the function argument list
+     *
+     * @param func
+     *            the user defined function
+     * @throws VisitorException
+     */
+    protected void visit(LOUserFunc func) throws VisitorException {
+        // Visit each of the arguments
+        Iterator<ExpressionOperator> i = func.getArguments().iterator();
+        while (i.hasNext()) {
+            i.next().visit(this);
+        }
+    }
+
+    /**
+     * @param binCond
+     *            the logical binCond operator that has to be visited
+     * @throws VisitorException
+     */
+    protected void visit(LOBinCond binCond) throws VisitorException {
+        /*
+         * Visit the conditional expression followed by the left hand operator
+         * and the right hand operator respectively
+         */
+
+        binCond.getCond().visit(this);
+        binCond.getLhsOp().visit(this);
+        binCond.getRhsOp().visit(this);
+    }
+
+    protected void visit(LOCast cast) throws VisitorException {
+        // Visit the expression to be cast
+
+        cast.getExpression().visit(this);
+    }
+
+    protected void visit(LORegexp regexp) throws VisitorException {
+        // Visit the operand of the regexp
+        regexp.getOperand().visit(this);
+    }
+
+    protected void visit(LOLoad load) throws VisitorException{
+
+
+    }
+
+    protected void visit(LOStore store) throws VisitorException{
+
+    }
+
+    protected void visit(LOConst store) throws VisitorException{
+
+    }
+
+    protected void visit(LOUnion u) throws VisitorException {
+
+    }
+
+    protected void visit(LOSplitOutput sop) throws VisitorException {
+        LogicalPlan lp = sop.getConditionPlan();
+        if (null != lp) {
+            PlanWalker w = new DependencyOrderWalker(lp);
+            pushWalker(w);
+            for(LogicalOperator logicalOp: lp.getRoots()) {
+                logicalOp.visit(this);
+            }
+            popWalker();
+        }
+    }
+
+    protected void visit(LODistinct dt) throws VisitorException {
+
+    }
+
+    protected void visit(LOCross cs) throws VisitorException {
+
+    }
+
+    protected void visit(LOProject project) throws VisitorException {
+        // Visit the operand of the project as long as the sentinel is false
+
+        if(!project.getSentinel()) {
+            project.getExpression().visit(this);
+        }
+    }
+
+    public void visit(LOGreaterThan op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOLesserThan op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOGreaterThanEqual op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOLesserThanEqual op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOEqual op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LONotEqual op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOAdd op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOSubtract op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOMultiply op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LODivide op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOMod op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+
+	public void visit(LONegative op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOMapLookup op) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOAnd binOp) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LOOr binOp) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+
+	public void visit(LONot uniOp) throws VisitorException {
+		// TODO Auto-generated method stub
+		return;
+	}
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import java.util.Map;
+import java.util.HashMap;
+
+/***
+ * This represents a node in DOT format
+ */
+public class DotNode {
+
+    public String name ;
+    public Map<String, String> attributes = new HashMap<String,String>() ;
+
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.Operator;
+
+import java.util.Map;
+import java.util.Iterator;
+import java.util.HashMap;
+
+/***
+ * This matcher only does exact key matching
+ */
+public class ExactKeyMatcher implements NodeMatcher<Operator,
+                                                    OperatorPlan<Operator>> {
+
+    public Map<OperatorKey, OperatorKey> match(OperatorPlan<Operator> plan1,
+                                               OperatorPlan<Operator> plan2,
+                                               StringBuilder messages) {
+        // Find plan1.OperatorSet - plan2.OperatorSet
+        int diff1 = diffKeys(plan1, plan2, messages, "plan2") ;
+
+        // Find plan2.OperatorSet - plan1.OperatorSet
+        int diff2 = diffKeys(plan2, plan1, messages, "plan1") ;
+
+        // If there is a problem, just finish here
+        if ( (diff1 != 0) || (diff2 != 0) ) {
+            return null ;
+        }
+
+        // if no problem, we just return exact matching
+        Iterator<Operator> iter = plan1.getKeys().values().iterator() ;
+        Map<OperatorKey, OperatorKey> outputMap
+                                = new HashMap<OperatorKey, OperatorKey>() ;
+        while(iter.hasNext()) {
+            Operator op = iter.next() ;
+            outputMap.put(op.getOperatorKey(), op.getOperatorKey()) ;
+        }
+
+        return outputMap;
+    }
+
+    /***
+     * Report plan1.OperatorSet - plan2.OperatorSet
+     *
+     * @param plan1
+     * @param plan2
+     * @param messages where the report messages go. null if no messages needed
+     * @param plan2Name the name that is used to refer to plan2 in messages
+     * @return
+     */
+
+    private int diffKeys(OperatorPlan<Operator> plan1,
+                         OperatorPlan<Operator> plan2,
+                         StringBuilder messages,
+                         String plan2Name) {
+        int count = 0 ;
+
+        // prepare
+        Map<OperatorKey, Operator> keyList = plan1.getKeys() ;
+        Iterator<OperatorKey> iter = keyList.keySet().iterator() ;
+
+        // go through the list of vertices of the first plan
+        while(iter.hasNext()) {
+
+            OperatorKey key = iter.next() ;
+
+            // if the same key doesn't exist in the second plan
+            // we've got a problem
+            if (plan2.getOperator(key) == null) {
+                Operator op1 = plan1.getOperator(key) ;
+                if (messages != null) {
+                    messages.append(op1.getClass().getSimpleName()) ;
+                    appendOpKey(op1.getOperatorKey(), messages) ;
+                    messages.append(" doesn't exist") ;
+                    if (plan2Name != null) {
+                        messages.append(" in ") ;
+                        messages.append(plan2Name) ;
+                        messages.append("\n") ;
+                    }
+                }
+                // increment diff counter
+                count++ ;
+            }
+        }
+
+        return count ;
+    }
+
+    ////////////// String Formatting Helpers //////////////
+
+    protected void appendOpKey(OperatorKey operatorKey, StringBuilder sb) {
+        sb.append("(") ;
+        sb.append(operatorKey.toString()) ;
+        sb.append(")") ;
+    }
+
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.Operator;
+
+import java.util.*;
+
+/***
+ * This matcher allows matching of different keys but both
+ * key sets have to be in the same increasing order
+ *
+ * Example:
+ * Plan1:  0-Load 1-Distinct 2-Split 3-SplitOutput
+ *         4-SplitOutput 5-Union
+ *
+ * Plan2: 10-Load 14-Distinct 15-Split 100-SplitOutput
+ *        110-SplitOutput 9999-Union
+ *
+ *
+ * Note: All the keys have to be in the same scope.
+ *
+ */
+public class IncreasingKeyMatcher implements
+                    NodeMatcher<Operator,OperatorPlan<Operator>> {
+
+    public Map<OperatorKey, OperatorKey> match(OperatorPlan plan1,
+                                               OperatorPlan plan2,
+                                               StringBuilder messages) {
+
+        List<OperatorKey> keyList1 = getSortedKeyList(plan1) ;
+        List<OperatorKey> keyList2 = getSortedKeyList(plan2) ;
+
+        // This matching logic only works when both plans have
+        // the same number of operators
+        if (keyList1.size() != keyList2.size()) {
+            messages.append("Two plans have different size") ;
+            return null ;
+        }
+
+        // Populating the output map
+        Map<OperatorKey, OperatorKey> outputMap
+                                = new HashMap<OperatorKey, OperatorKey>() ;
+        for(int i=0; i< keyList1.size() ; i++) {
+            outputMap.put(keyList1.get(i), keyList2.get(i)) ;
+        }
+
+        return outputMap ;
+    }
+
+
+    ////////// Helper ///////////
+    private List<OperatorKey> getSortedKeyList(OperatorPlan plan) {
+
+        List<OperatorKey> keyList
+                        = new ArrayList<OperatorKey>(plan.getKeys().keySet()) ;
+        Collections.sort(keyList); 
+        return keyList ;
+    }
+
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.parser.QueryParser ;
+import org.apache.pig.impl.logicalLayer.parser.ParseException ;
+import org.apache.pig.impl.io.FileSpec;
+import org.apache.pig.builtin.PigStorage;
+
+import java.io.IOException;
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+
+
+public class LogicalPlanLoader
+                extends OperatorPlanLoader<LogicalOperator, LogicalPlan> {
+
+    /***
+     * Create various Logical Operators
+     * @param node
+     * @param plan
+     * @return
+     */
+    protected LogicalOperator createOperator(DotNode node, LogicalPlan plan) {
+        String operatorType = node.attributes.get("type") ;
+
+        // Cannot work without the correct type
+        if (operatorType == null) {
+            throw new RuntimeException("Unspecified operator type from Dot file") ;
+        }
+
+        if (operatorType.equals("LOLoad")) {
+            return createLOLoad(node, plan) ;
+        }
+        else if (operatorType.equals("LOFilter")) {
+            return createLOFilter(node, plan) ;
+        }
+        else if (operatorType.equals("LODistinct")) {
+            return createLODistinct(node, plan) ;
+        }
+        else if (operatorType.equals("LOSort")) {
+            return createLOSort(node, plan) ;
+        }
+        else if (operatorType.equals("LOForEach")) {
+            return createLOForEach(node, plan) ;
+        }
+        else if (operatorType.equals("LOSplit")) {
+            return createLOSplit(node, plan) ;
+        }
+        else if (operatorType.equals("LOSplitOutput")) {
+            return createLOSplitOutput(node, plan) ;
+        }
+        else if (operatorType.equals("LOCogroup")) {
+            return createLOCogroup(node, plan) ;
+        }
+        else if (operatorType.equals("LOForEach")) {
+            return createLOForEach(node, plan) ;
+        }
+        else if (operatorType.equals("LOUnion")) {
+            return createLOUnion(node, plan) ;
+        }
+        else if (operatorType.equals("LOCross")) {
+            return createLOCross(node, plan) ;
+        }
+
+        // else
+        throw new AssertionError("Unknown operator type") ;
+    }
+
+    private LOLoad createLOLoad(DotNode node, LogicalPlan plan) {
+        LOLoad load = null ;
+        FileSpec fileSpec = new FileSpec("pi",
+                                         PigStorage.class.getName()) ;
+        try {
+            load = new LOLoad(plan, getKey(node.attributes), fileSpec, null) ;
+            fillSchema(load, node.attributes) ;
+        }
+        catch (IOException ioe) {
+            throw new AssertionError("Dummy data is not good") ;
+        }
+        return load ;
+    }
+
+    private LOFilter createLOFilter(DotNode node, LogicalPlan plan) {
+        LOFilter filter = new LOFilter(plan, getKey(node.attributes), null, null) ;
+        fillSchema(filter, node.attributes) ;
+        return filter ;
+    }
+
+    private LODistinct createLODistinct(DotNode node, LogicalPlan plan) {
+        LODistinct distinct = new LODistinct(plan, getKey(node.attributes), null) ;
+        fillSchema(distinct, node.attributes) ;
+        return distinct ;
+    }
+
+    private LOSort createLOSort(DotNode node, LogicalPlan plan) {
+        LOSort sort = new LOSort(plan, getKey(node.attributes),
+                                 null, null, null, "") ;
+        fillSchema(sort, node.attributes) ;
+        return sort ;
+    }
+
+    private LOForEach createLOForEach(DotNode node, LogicalPlan plan) {
+        LOForEach foreach = new LOForEach(plan, getKey(node.attributes), null) ;
+        fillSchema(foreach, node.attributes) ;
+        return foreach ;
+    }
+
+    private LOSplit createLOSplit(DotNode node, LogicalPlan plan) {
+        LOSplit split = new LOSplit(plan, getKey(node.attributes),  null) ;
+        fillSchema(split, node.attributes) ;
+        return split ;
+    }
+
+    private LOSplitOutput createLOSplitOutput(DotNode node, LogicalPlan plan) {
+        LOSplitOutput splitOut = new LOSplitOutput(plan,
+                                        getKey(node.attributes), 0,  null) ;
+        fillSchema(splitOut, node.attributes) ;
+        return splitOut ;
+    }
+
+    private LOCogroup createLOCogroup(DotNode node, LogicalPlan plan) {
+        LOCogroup cogroup = new LOCogroup(plan, getKey(node.attributes),
+                                          null, null, null) ;
+        fillSchema(cogroup, node.attributes) ;
+        return cogroup ;
+    }
+
+    private LOUnion createLOUnion(DotNode node, LogicalPlan plan) {
+        LOUnion union = new LOUnion(plan, getKey(node.attributes),  null) ;
+        fillSchema(union, node.attributes) ;
+        return union ;
+    }
+
+    private LOCross createLOCross(DotNode node, LogicalPlan plan) {
+        LOCross cross = new LOCross(plan, getKey(node.attributes),  null) ;
+        fillSchema(cross, node.attributes) ;
+        return cross ;
+    }
+
+    private void fillSchema(LogicalOperator op, Map<String,String> attributes) {
+        String schemaString = attributes.get("schema") ;
+        if (schemaString != null) {
+
+            ByteArrayInputStream stream
+                    = new ByteArrayInputStream(schemaString.getBytes()) ;
+            QueryParser queryParser = new QueryParser(stream) ;
+            Schema schema = null ;
+            try {
+                schema = queryParser.TupleSchema() ;
+                op.forceSchema(schema);
+                op.setSchemaComputed(true);
+            }
+            catch (ParseException pe) {
+                System.out.println(pe.getMessage()) ;
+                throw new RuntimeException("Error reading schema string") ;
+            }
+        }
+        else {
+            op.forceSchema(null);
+        }
+    }
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.Operator;
+
+import java.util.Map;
+
+/***
+ * This is a common interface for graph vertex mapping logic.
+ * Though I call it Matcher so that people don't get confused
+ * with mapper in MapReduce.
+ */
+
+public interface NodeMatcher<E extends Operator,
+                             P extends OperatorPlan<E>> {
+
+    /***
+     * This method does matching between vertices in two
+     * given plans.
+     *
+     * @param plan1
+     * @param plan2
+     * @param messages
+     * @return The output map: plan1Key -> plan2Key
+     */
+    Map<OperatorKey, OperatorKey> match(P plan1,
+                                        P plan2,
+                                        StringBuilder messages) ;
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java Tue Jun  3 10:21:59 2008
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.*;
+
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+import java.util.HashMap;
+
+public abstract class OperatorPlanLoader<E extends Operator,
+                                         P extends OperatorPlan<E>> {
+
+    /***
+     * This method is used for loading an operator plan encoded in Dot format
+     * @param dotContent the dot content
+     * @param clazz the plan type to be created
+     * @return
+     */
+    public P load(String dotContent, Class<P> clazz) {
+        DotGraphReader dotReader = new DotGraphReader() ;
+        DotGraph graph = dotReader.load(dotContent) ;
+        return constructPlan(graph, clazz) ;
+    }
+
+    /***
+     * Convenient method for loading directly from file
+     * @param file
+     * @param clazz
+     * @return
+     */
+    public P loadFromFile(String file, Class<P> clazz) {
+        DotGraphReader dotReader = new DotGraphReader() ;
+        DotGraph graph = dotReader.loadFromFile(file) ;
+        return constructPlan(graph, clazz) ;
+    }
+
+    public PlanAndGraphEntry loadFromFileWithGraph(String file,
+                                                        Class<P> clazz) {
+        DotGraphReader dotReader = new DotGraphReader() ;
+        DotGraph graph = dotReader.loadFromFile(file) ;
+        P plan = constructPlan(graph, clazz) ;
+        return new PlanAndGraphEntry(plan, graph) ;
+    }
+
+    public class PlanAndGraphEntry {
+        public PlanAndGraphEntry(P plan, DotGraph dotGraph) {
+            this.plan = plan ;
+            this.dotGraph = dotGraph ;
+        }
+        public P plan ;
+        public DotGraph dotGraph ;
+    }
+
+    /***
+     * This method has be overridden to instantiate the correct vertex type
+     *
+     * @param node
+     * @param plan
+     * @return
+     */
+    protected abstract E createOperator(DotNode node, P plan) ;
+
+    //////////////// Helpers //////////////////
+
+    /***
+     * Construct the plan based on the given Dot graph
+     * 
+     * @param graph
+     * @return
+     */
+
+    P constructPlan(DotGraph graph, Class<P> clazz) {
+
+        P plan ;
+        Map<String, E> nameMap = new HashMap<String, E>() ;
+
+        try {
+            plan = clazz.newInstance() ;
+        }
+        catch (IllegalAccessException iae) {
+            throw new AssertionError("Cannot instantiate a plan") ;
+        }
+        catch (InstantiationException ie) {
+            throw new AssertionError("Cannot instantiate a plan") ;
+        }
+
+        for(DotNode node: graph.nodes) {
+            E op = createOperator(node, plan) ;
+            nameMap.put(node.name, op) ;
+            plan.add(op);
+        }
+
+        for(DotEdge edge: graph.edges) {
+            E fromOp = nameMap.get(edge.fromNode)  ;
+            E toOp = nameMap.get(edge.toNode)  ;
+            try {
+                plan.connect(fromOp, toOp);
+            }
+            catch (PlanException pe) {
+                throw new RuntimeException("Invalid Dot file") ;
+            }
+        }
+
+        return plan ;
+    }
+
+
+    /***
+     * Helper for retrieving operator key from encoded attributes.
+     * By default, it will look for "key" in attributes.
+     * If no key is found, an arbitrary one will be generated.
+     * @param attributes
+     * @return
+     */
+    protected OperatorKey getKey(Map<String,String> attributes) {
+        String key = attributes.get("key") ;
+        if (key != null) {
+            return new OperatorKey("scope", Long.parseLong(key)) ;
+        }
+        else {
+            long newId = NodeIdGenerator.getGenerator().getNextNodeId("scope") ;
+            return new OperatorKey("scope", newId) ;
+        }
+    }
+
+}



Mime
View raw message