pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r668885 - in /incubator/pig/branches/types: ./ src/org/apache/pig/impl/logicalLayer/ src/org/apache/pig/impl/logicalLayer/parser/ src/org/apache/pig/impl/logicalLayer/validators/ test/org/apache/pig/test/
Date Wed, 18 Jun 2008 00:39:00 GMT
Author: gates
Date: Tue Jun 17 17:39:00 2008
New Revision: 668885

URL: http://svn.apache.org/viewvc?rev=668885&view=rev
Log:
PIG-158 Santhosh's cast_fix patch.


Modified:
    incubator/pig/branches/types/build.xml
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java

Modified: incubator/pig/branches/types/build.xml
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Tue Jun 17 17:39:00 2008
@@ -234,11 +234,7 @@
                     <exclude name="**/TestCompressedFiles.java" />
                     <exclude name="**/TestEvalPipeline.java" />
                     <exclude name="**/TestFilterOpNumeric.java" />
-                    <exclude name="**/TestFilterOpString.java" />
-                    <exclude name="**/TestInfixArithmetic.java" />
-                    <!-- <exclude name="**/TestMapReduce.java" /> -->
                     <exclude name="**/TestPigFile.java" />
-                    <exclude name="**/TestPigSplit.java" />
                     <exclude name="**/TestStoreOld.java" />
                     <!-- Excluced because we don't want to run them -->
                     <exclude name="**/TypeCheckingTestUtil.java" />

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java Tue
Jun 17 17:39:00 2008
@@ -19,6 +19,7 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.ArrayList;
 import java.io.IOException;
 
 import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -60,13 +61,18 @@
         if (!mIsSchemaComputed && (null == mSchema)) {
             // Get the schema of the parent
             Collection<LogicalOperator> s = mPlan.getPredecessors(this);
+            ArrayList<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>();
             try {
                 LogicalOperator op = s.iterator().next();
                 if (null == op) {
-                    log.debug("getSchema: Operator not in plan");
                     throw new FrontendException("Could not find operator in plan");
                 }
-                mSchema = s.iterator().next().getSchema();
+                if(op instanceof ExpressionOperator) {
+                    fss.add(((ExpressionOperator)op).getFieldSchema());
+                    mSchema = new Schema(fss);
+                } else {
+                    mSchema = op.getSchema();
+                }
                 mIsSchemaComputed = true;
             } catch (FrontendException ioe) {
                 mSchema = null;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java Tue Jun
17 17:39:00 2008
@@ -18,6 +18,7 @@
 package org.apache.pig.impl.logicalLayer;
 
 import java.util.List;
+import java.util.ArrayList;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.OperatorKey;
@@ -64,8 +65,14 @@
     @Override
     public Schema getSchema() throws FrontendException {
         if (!mIsSchemaComputed && (null == mSchema)) {
+            ArrayList<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>();
             try {
-                mSchema = mInput.getSchema();
+                if(mInput instanceof ExpressionOperator) {
+                    fss.add(((ExpressionOperator)mInput).getFieldSchema());
+                    mSchema = new Schema(fss);
+                } else {
+                    mSchema = mInput.getSchema();
+                }
                 mIsSchemaComputed = true;
             } catch (FrontendException ioe) {
                 mSchema = null;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOPrinter.java Tue Jun
17 17:39:00 2008
@@ -21,11 +21,17 @@
 import java.util.Iterator;
 import java.util.List;
 import java.io.PrintStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.Collection;
+import java.util.Collections;
 
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.DepthFirstWalker;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
 import org.apache.pig.impl.plan.MultiMap;
 import org.apache.pig.impl.plan.VisitorException;
 
@@ -35,299 +41,163 @@
 public class LOPrinter extends LOVisitor {
 
     private PrintStream mStream = null;
-    private int mIndent = 0;
+    private String TAB1 = "    ";
+    private String TABMore = "|   ";
+    private String LSep = "|\n|---";
+    private String USep = "|   |\n|   ";
+    private int levelCntr = -1;
+    private OutputStream printer;
 
     /**
      * @param ps PrintStream to output plan information to
      * @param plan Logical plan to print
      */
     public LOPrinter(PrintStream ps, LogicalPlan plan) {
+        //super(plan, new DependencyOrderWalker(plan));
         super(plan, new DepthFirstWalker(plan));
+        mStream = ps;
     }
 
-    public void visit(LOAdd a) throws VisitorException {
-        visitBinary(a, "+");
-    }
-
-    public void visit(LOAnd a) throws VisitorException {
-        visitBinary(a, "AND");
-    }
-    
-    public void visit(LOBinCond bc) throws VisitorException {
-        print(bc);
-        mStream.print(" COND: (");
-        bc.getCond().visit(this);
-        mStream.print(") TRUE: (");
-        bc.getLhsOp().visit(this);
-        mStream.print(") FALSE (");
-        bc.getRhsOp().visit(this);
-        mStream.print(")");
-    }
-
-    public void visit(LOCogroup g) throws VisitorException {
-        print(g);
-        mStream.print("GROUP BY PLANS:");
-        MultiMap<LogicalOperator, LogicalPlan> plans = g.getGroupByPlans();
-        for (LogicalOperator lo : plans.keySet()) {
-            // Visit the associated plans
-            for (LogicalPlan plan : plans.get(lo)) {
-                mIndent++;
-                pushWalker(new DepthFirstWalker(plan));
-                visit();
-                popWalker();
-                mIndent--;
-            }
-            mStream.println();
-        }
-        // Visit input operators
-        for (LogicalOperator lo : plans.keySet()) {
-            // Visit the operator
-            lo.visit(this);
+    @Override
+    public void visit() throws VisitorException {
+        try {
+            mStream.write(depthFirstLP().getBytes());
+        } catch (IOException e) {
+            throw new VisitorException(e.getMessage());
         }
     }
-        
-    public void visit(LOConst c) throws VisitorException {
-        print(c);
-        mStream.print(" VALUE (" + c.getValue() + ")");
-    }
 
-    public void visit(LOCross c) throws VisitorException {
-        print(c);
-        mStream.println();
-        super.visit(c);
+    public void print(OutputStream printer) throws VisitorException, IOException {
+        this.printer = printer;
+        printer.write(depthFirstLP().getBytes());
     }
 
-    public void visit(LODistinct d) throws VisitorException {
-        print(d);
-        mStream.println();
-        super.visit(d);
-    }
-
-    public void visit(LODivide d) throws VisitorException {
-        visitBinary(d, "/");
-    }
 
-    public void visit(LOEqual e) throws VisitorException {
-        visitBinary(e, "==");
-    }
-
-    public void visit(LOFilter f) throws VisitorException {
-        print(f);
-        mStream.print(" COMP: ");
-        mIndent++;
-        pushWalker(new DepthFirstWalker(f.getComparisonPlan()));
-        visit();
-        mIndent--;
-        mStream.println();
-        f.getInput().visit(this);
-    }
-
-     public void visit(LOForEach f) throws VisitorException {
-        print(f);
-        mStream.print(" PLAN: ");
-        mIndent++;
-        pushWalker(new DepthFirstWalker(f.getForEachPlan()));
-        visit();
-        mIndent--;
-        mStream.println();
-        // Visit our input
-        mPlan.getPredecessors((LogicalOperator)f).get(0).visit(this);
-    }
- 
-    public void visit(LOGreaterThan gt) throws VisitorException {
-        visitBinary(gt, ">");
-    }
-
-    public void visit(LOGreaterThanEqual gte) throws VisitorException {
-        visitBinary(gte, ">=");
-    }
-
-    public void visit(LOLesserThan lt) throws VisitorException {
-        visitBinary(lt, "<");
-    }
-
-    public void visit(LOLesserThanEqual lte) throws VisitorException {
-        visitBinary(lte, "<=");
-    }
-
-    public void visit(LOLoad load) throws VisitorException {
-        print(load);
-        mStream.print(" FILE: " + load.getInputFile().getFileName());
-        mStream.print(" FUNC: " + load.getLoadFunc().getClass().getName());
-        mStream.println();
-    }
-
-    public void visit(LOMapLookup mlu) throws VisitorException {
-        print(mlu);
-        mStream.print("(");
-        mlu.getMap().visit(this);
-        mStream.print(")# " + mlu.getOperatorKey());
-    }
-
-    public void visit(LOMod m) throws VisitorException {
-        visitBinary(m, "MOD");
-    }
-
-    public void visit(LOMultiply m) throws VisitorException {
-        visitBinary(m, "*");
-    }
-
-    public void visit(LONegative n) throws VisitorException {
-        visitUnary(n, "-");
-    }
-
-    public void visit(LONot n) throws VisitorException {
-        visitUnary(n, "NOT");
+    protected String depthFirstLP() throws VisitorException {
+        StringBuilder sb = new StringBuilder();
+        List<LogicalOperator> leaves = mPlan.getLeaves();
+        Collections.sort(leaves);
+        for (LogicalOperator leaf : leaves) {
+            sb.append(depthFirst(leaf));
+            sb.append("\n");
+        }
+        sb.delete(sb.length() - "\n".length(), sb.length());
+        sb.delete(sb.length() - "\n".length(), sb.length());
+        return sb.toString();
     }
-
-    public void visit(LONotEqual ne) throws VisitorException {
-        visitBinary(ne, "!=");
+    
+    private String planString(LogicalPlan lp){
+        StringBuilder sb = new StringBuilder();
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        if(lp!=null)
+            lp.explain(baos, mStream);
+        else
+            return "";
+        sb.append(USep);
+        sb.append(shiftStringByTabs(baos.toString(), 2));
+        return sb.toString();
     }
-
-    public void visit(LOOr or) throws VisitorException {
-        visitBinary(or, "OR");
+    
+    private String planString(List<LogicalPlan> logicalPlanList){
+        StringBuilder sb = new StringBuilder();
+        if(logicalPlanList!=null)
+            for (LogicalPlan lp : logicalPlanList) {
+                sb.append(planString(lp));
+            }
+        return sb.toString();
     }
 
-    public void visit(LOProject p) throws VisitorException {
-        print(p);
-        if (p.isStar()) {
-            mStream.print(" ALL ");
+    private String depthFirst(LogicalOperator node) throws VisitorException {
+        StringBuilder sb = new StringBuilder(node.name());
+        if(node instanceof ExpressionOperator) {
+            sb.append(" FieldSchema: ");
+            try {
+                sb.append(((ExpressionOperator)node).getFieldSchema());
+            } catch (Exception e) {
+                //sb.append("Caught Exception: " + e.getMessage());
+            }
         } else {
-            List<Integer> cols = p.getProjection();
-            mStream.print(" COL");
-            if (cols.size() > 1) mStream.print("S");
-            mStream.print(" (");
-            for (int i = 0; i < cols.size(); i++) {
-                if (i > 0) mStream.print(", ");
-                mStream.print(cols.get(i));
+            sb.append(" Schema: ");
+            try {
+                sb.append(node.getSchema());
+            } catch (Exception e) {
+                //sb.append("Caught exception: " + e.getMessage());
             }
-            mStream.print(")");
         }
-        mStream.print(" FROM ");
-        if (p.getSentinel()) {
-            // This project is connected to some other relation, don't follow
-            // that path or we'll cycle in the graph.
-            p.getExpression().name();
-        } else {
-            mIndent++;
-            p.getExpression().visit(this);
-            mIndent--;
+        sb.append(" Type: " + DataType.findTypeName(node.getType()));
+        sb.append("\n");
+        if(node instanceof LOFilter){
+            sb.append(planString(((LOFilter)node).getComparisonPlan()));
+        }
+        else if(node instanceof LOForEach){
+            sb.append(planString(((LOForEach)node).getForEachPlan()));        
+        }
+        else if(node instanceof LOGenerate){
+            sb.append(planString(((LOGenerate)node).getGeneratePlans())); 
+            
+        }
+        else if(node instanceof LOCogroup){
+            MultiMap<LogicalOperator, LogicalPlan> plans = ((LOCogroup)node).getGroupByPlans();
+            for (LogicalOperator lo : plans.keySet()) {
+                // Visit the associated plans
+                for (LogicalPlan plan : plans.get(lo)) {
+                    sb.append(planString(plan));
+                }
+            }
+        }
+        else if(node instanceof LOSort){
+            sb.append(planString(((LOSort)node).getSortColPlans())); 
+        }
+        else if(node instanceof LOSplitOutput){
+            sb.append(planString(((LOSplitOutput)node).getConditionPlan()));
         }
-    }
-
-    public void visit(LORegexp r) throws VisitorException {
-        print(r);
-        mStream.print(" REGEX (" + r.getRegexp() + ") LOOKING IN (");
-        r.getOperand().visit(this);
-        mStream.print(")");
-    }
-
-    /**
-     * Only LOUnion.visit() should ever call this method.
-     */
-    /*
-    @Override
-    public void visitUnion(LOUnion u) {
-        print(u, u.name());
-        super.visitUnion(u);
-    }
         
-    /**
-     * Only LOSort.visit() should ever call this method.
-     */
-    /*
-    @Override
-    public void visitSort(LOSort s) {
-        List<EvalSpec> ls = new ArrayList<EvalSpec>();
-        ls.add(s.getSpec());
-        print(s, s.name());
-        super.visitSort(s);
-    }
+        List<LogicalOperator> predecessors = mPlan.getPredecessors(node);
+        if(node instanceof LOProject) {
+            System.err.println("LOProject " + node + " predecessors: " + predecessors + "
in plan " + mPlan);
+            System.err.println("mPlan size: " + mPlan.size());
+        } else if(node instanceof LOSort) {
+            System.err.println("LOSort : " + node + " predecessors: " + predecessors + "
in plan " + mPlan);
+            System.err.println("mPlan size: " + mPlan.size());
+        }
         
-    /**
-     * Only LOSplit.visit() should ever call this method.
-     */
-    /*
-    @Override
-    public void visitSplit(LOSplit s) {
-        print(s, s.name());
-        super.visitSplit(s);
-    }
+        if (predecessors == null)
+            return sb.toString();
         
-    /**
-     * Only LOStore.visit() should ever call this method.
-     */
-    /*
-    @Override
-    public void visitStore(LOStore s) {
-        print(s, s.name());
-        super.visitStore(s);
-    }
-
-    private void print(LogicalOperator lo, String name) {
-        List<EvalSpec> empty = new ArrayList<EvalSpec>();
-        print(lo, name, empty);
-    }
-    */
-
-    private void visitBinary(
-            BinaryExpressionOperator b,
-            String op) throws VisitorException {
-        print(b);
-        mStream.print(" (");
-        b.getLhsOperand().visit(this);
-        mStream.print(") " + op + " (");
-        b.getRhsOperand().visit(this);
-        mStream.print(") ");
-    }
-
-    private void visitUnary(
-            UnaryExpressionOperator e,
-            String op) throws VisitorException {
-        print(e);
-        mStream.print(op + " (");
-        e.getOperand().visit(this);
-        mStream.print(") ");
+        Collections.sort(predecessors);
+        int i = 0;
+        for (LogicalOperator pred : predecessors) {
+            i++;
+            String DFStr = depthFirst(pred);
+            if (DFStr != null) {
+                sb.append(LSep);
+                if (i < predecessors.size())
+                    sb.append(shiftStringByTabs(DFStr, 2));
+                else
+                    sb.append(shiftStringByTabs(DFStr, 1));
+            }
+        }
+        return sb.toString();
     }
 
-    private void print(LogicalOperator lo) {
-        for (int i = 0; i < mIndent; i++) mStream.print("    ");
+    private String shiftStringByTabs(String DFStr, int TabType) {
+        StringBuilder sb = new StringBuilder();
+        String[] spl = DFStr.split("\n");
 
-        printName(lo);
+        String tab = (TabType == 1) ? TAB1 : TABMore;
 
-        if (!(lo instanceof ExpressionOperator)) {
-            mStream.print("Inputs: ");
-            for (LogicalOperator predecessor : mPlan.getPredecessors(lo)) {
-                printName(predecessor);
-            }
-            mStream.print("Schema: ");
-            try {
-                printSchema(lo.getSchema());
-            } catch (FrontendException fe) {
-                // ignore it, nothing we can do
-                mStream.print("()");
-            }
+        sb.append(spl[0] + "\n");
+        for (int i = 1; i < spl.length; i++) {
+            sb.append(tab);
+            sb.append(spl[i]);
+            sb.append("\n");
         }
-        mStream.print(" : ");
-    }
-
-    private void printName(LogicalOperator lo) {
-        mStream.println(lo.name() + " key(" + lo.getOperatorKey().scope + 
-            ", " + lo.getOperatorKey().id + ") ");
+        return sb.toString();
     }
 
-    private void printSchema(Schema schema) {
-        mStream.print("(");
-        for (Schema.FieldSchema fs : schema.getFields()) {
-            if (fs.alias != null) mStream.print(fs.alias + ": ");
-            mStream.print(DataType.findTypeName(fs.type));
-            if (fs.schema != null) {
-                if (fs.type == DataType.BAG) mStream.print("{");
-                printSchema(fs.schema);
-                if (fs.type == DataType.BAG) mStream.print("}");
-            }
-        }
-        mStream.print(")");
+    private void dispTabs() {
+        for (int i = 0; i < levelCntr; i++)
+            System.out.print(TAB1);
     }
 }
 

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java Tue Jun
17 17:39:00 2008
@@ -148,7 +148,7 @@
 
     @Override
     public String name() {
-        return "Project " + mKey.scope + "-" + mKey.id;
+        return "Project " + mKey.scope + "-" + mKey.id + " Projections: " + (mIsStar? " [*]
": mProjection);
     }
 
     @Override
@@ -184,10 +184,10 @@
                             log.debug("expression operator schema: " + expressionOperator.getSchema());
                             log.debug("expression operator type: " + expressionOperator.getType());
                             //TODO
-                            //the type of the operator will be unkown. when type checking
is in place
+                            //the type of the operator will be unknown. when type checking
is in place
                             //add the type of the operator as a parameter to the fieldschema
creation
-                            //mFieldSchema = new Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema(), expressionOperator.getType());
-                            mFieldSchema = new Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema());
+                            mFieldSchema = new Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema(), expressionOperator.getType());
+                            //mFieldSchema = new Schema.FieldSchema(expressionOperator.getAlias(),
expressionOperator.getSchema());
                         }
                     } else {
                         log.warn("The input for a projection operator cannot be null");
@@ -246,8 +246,13 @@
                                 Schema.FieldSchema expOpFs = ((ExpressionOperator)expressionOperator).getFieldSchema();
                                 if(null != expOpFs) {
                                     Schema s = expOpFs.schema;
+                                    log.debug("Schema s: " + s);
                                     if(null != s) {
-                                        fss.add(s.getField(colNum));
+                                        if(colNum < s.size()) {
+                                            fss.add(s.getField(colNum));
+                                        } else {
+                                            fss.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
+                                        }
                                     } else {
                                         fss.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
                                     }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java Tue Jun
17 17:39:00 2008
@@ -100,12 +100,18 @@
         if (!mIsSchemaComputed && (null == mSchema)) {
             // get our parent's schema
             Collection<LogicalOperator> s = mPlan.getPredecessors(this);
+            ArrayList<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>();
             try {
                 LogicalOperator op = s.iterator().next();
                 if (null == op) {
                     throw new FrontendException("Could not find operator in plan");
                 }
-                mSchema = op.getSchema();
+                if(op instanceof ExpressionOperator) {
+                    fss.add(((ExpressionOperator)op).getFieldSchema());
+                    mSchema = new Schema(fss);
+                } else {
+                    mSchema = op.getSchema();
+                }
                 mIsSchemaComputed = true;
             } catch (FrontendException ioe) {
                 mSchema = null;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Tue
Jun 17 17:39:00 2008
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintStream;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -27,6 +28,7 @@
 import java.util.Map;
 
 import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.VisitorException;
 
 public class LogicalPlan extends OperatorPlan<LogicalOperator> {
     private static final long serialVersionUID = 2L;
@@ -46,4 +48,17 @@
         return getSingleLeafPlanOutputOp().getType() ;
     }
 
+    public void explain(OutputStream out, PrintStream ps){
+        LOPrinter lpp = new LOPrinter(ps, this);
+
+        try {
+            lpp.print(out);
+        } catch (VisitorException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
 }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Tue Jun 17 17:39:00 2008
@@ -236,6 +236,10 @@
 			flattenedColumns.add(column);
 			(gis.get(i)).isInner = true;
 			projectPlan.add(column);
+            if(projectInput instanceof ExpressionOperator) {
+			    projectPlan.add(projectInput);
+			    projectPlan.connect(projectInput, column);
+            }
 			log.debug("parseCogroup: Added operator " + column.getClass().getName() + " " + column
+ " to logical plan " + projectPlan);
 			generatePlans.add(projectPlan);
 		}
@@ -647,6 +651,7 @@
 		log.debug("Added operator: " + distinct.getClass().getName() + " to the logical plan");

 		lp.connect(op, distinct);
 		log.debug("Connected alias: " + op.getAlias() + " operator " + op.getClass().getName()
+ " to operator " + distinct.getClass().getName());
+        op = distinct;
 	})
 |	(<CROSS> op = CrossClause(lp))
 |   (<JOIN> op = JoinClause(lp))
@@ -1521,6 +1526,10 @@
 			project.setStar(true); 
 			item = project;
 			lp.add(project);
+            if(input instanceof ExpressionOperator) {
+			    lp.add(input);
+			    lp.connect(input, project);
+            }
 			log.debug("FGItem: Added operator " + project.getClass().getName() + " " + project + "
to logical plan " + lp);
 		}
 	)
@@ -1728,7 +1737,6 @@
 			"." projection = BracketedSimpleProj(subSchema,lp,item) 
 			{
 				assertAtomic(item,false); 
-				lp.remove(item);
 				item = projection;
 			}
 		)
@@ -1892,6 +1900,10 @@
 		project.setStar(true); 
 		item = project;
 		lp.add(project);
+        if(input instanceof ExpressionOperator) {
+		    lp.add(input);
+		    lp.connect(input, project);
+        }
 		log.debug("EvalArgsItem: Added operator " + project.getClass().getName() + " " + project
+ " to logical plan " + lp);
 	}
 	)
@@ -2215,6 +2227,10 @@
 		ExpressionOperator project = new LOProject(lp, new OperatorKey(scope, getNextId()), eOp,
colList);
 		lp.add(project);
 		log.debug("SimpleProj: Added operator " + project.getClass().getName() + " " + project
+ " to logical plan " + lp);
+        if(eOp instanceof ExpressionOperator) {
+		    lp.add(eOp);
+		    lp.connect(eOp, project);
+        }
 		log.trace("Exiting SimpleProj");
 		return project;
 	}
@@ -2389,6 +2405,10 @@
 			log.debug("eOp: " + eOp.getClass().getName() + " " + eOp);
 			lp.add(project);
 			log.debug("DollarVar: Added operator " + project.getClass().getName() + " " + project
+ " to logical plan " + lp);
+            if(eOp instanceof ExpressionOperator) {
+			    lp.add(eOp);
+			    lp.connect(eOp, project);
+            }
 		} catch (Exception planException) {
 			ParseException parseException = new ParseException(planException.getMessage());
 			throw parseException;
@@ -2429,6 +2449,10 @@
 				try {
 					lp.add(item);
 					log.debug("AliasFieldOrSpec: Added operator " + item.getClass().getName() + " " + item
+ " to logical plan " + lp);
+                    if(op instanceof ExpressionOperator) {
+					    lp.add(op);
+					    lp.connect(op, item);
+                    }
 				} catch (Exception planException) {
 					ParseException parseException = new ParseException(planException.getMessage());
 					throw parseException;
@@ -2457,6 +2481,10 @@
 			try {
 				lp.add(item);
 				log.debug("AliasFieldOrSpec: Added operator " + item.getClass().getName() + " " + item
+ " to logical plan " + lp);
+                if(eOp instanceof ExpressionOperator) {
+				    lp.add(eOp);
+				    lp.connect(eOp, item);
+                }
 			} catch (Exception planException) {
 				ParseException parseException = new ParseException(planException.getMessage());
 				throw parseException;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
Tue Jun 17 17:39:00 2008
@@ -93,6 +93,9 @@
         else if (eOp instanceof LONot) {
             visit((LONot) eOp) ;
         }
+        else if (eOp instanceof LOMapLookup) {
+            visit((LOMapLookup) eOp) ;
+        }
         // TODO: Check that all operators are included here
     }
 
@@ -275,6 +278,19 @@
 
     }
 
+    @Override
+    public void visit(LOMapLookup map)
+                        throws VisitorException {
+        if(!DataType.isAtomic(DataType.findType(map.getLookUpKey()))) {
+            String msg = "Map key should be a basic type" ;
+            msgCollector.collect(msg, MessageType.Error);
+            throw new VisitorException(msg) ;
+        }
+
+        map.setType(map.getValueType());
+
+    }
+
     /**
      * LORegexp expects CharArray as input
      * Itself always returns Boolean
@@ -957,6 +973,11 @@
         }
     }
 
+    @Override
+    public void visit(LOIsNull uniOp) throws VisitorException {
+        uniOp.setType(DataType.BOOLEAN) ;
+    }
+
     private void insertLeftCastForBinaryOp(BinaryExpressionOperator binOp,
                                            byte toType ) {
         LogicalPlan currentPlan =  (LogicalPlan) mCurrentWalker.getPlan() ;
@@ -1983,8 +2004,8 @@
                                     outputSchemaIdx++ ;
                                 }
                             } else {
-                                schema.getField(outputSchemaIdx).type = leaf.getType() ;
-                                schema.getField(outputSchemaIdx).schema = leaf.getSchema()
;
+                                schema.getField(outputSchemaIdx).type = DataType.BYTEARRAY;
+                                schema.getField(outputSchemaIdx).schema = null;
                                 outputSchemaIdx++ ;
                             }
                         }
@@ -1996,8 +2017,8 @@
                         }
                     }
                     else {
-                        throw new AssertionError("Unsupported inner plan type"
-                                                 + " in LOGenerate" ) ;
+                        throw new AssertionError("Unsupported inner plan type: "
+                                                 + DataType.findTypeName(leaf.getType())
+ " in LOGenerate" ) ;
                     }
                 }
 

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=668885&r1=668884&r2=668885&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java Tue
Jun 17 17:39:00 2008
@@ -53,6 +53,7 @@
 import org.apache.pig.impl.logicalLayer.ExpressionOperator;
 import org.apache.pig.impl.logicalLayer.LogicalPlan;
 import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
+import org.apache.pig.impl.logicalLayer.LOPrinter;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 
 
@@ -881,6 +882,16 @@
         buildPlan("e = foreach c generate $0, ($1 IS NOT NULL ? $1 - 5 : 0);");
     }
 
+    private void printPlan(LogicalPlan lp) {
+        LOPrinter graphPrinter = new LOPrinter(System.err, lp);
+        System.err.println("Printing the logical plan");
+        try {
+            graphPrinter.visit();
+        } catch (Exception e) {
+            System.err.println(e.getMessage());
+        }
+    }
+    
     // Helper Functions
     
     // Helper Functions



Mime
View raw message