asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [21/85] [abbrv] [partial] incubator-asterixdb-hyracks git commit: Move Pregelix and Hivesterix codebase to new repositories: 1. Move Pregelix codebase to https://github.com/pregelix/pregelix; 2. Move Hivesterix codebase to https://code.google.com/p/hives
Date Fri, 24 Apr 2015 18:45:48 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java
deleted file mode 100644
index a4022d1..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
-import org.apache.hadoop.hive.ql.plan.JoinDesc;
-import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-
-@SuppressWarnings("rawtypes")
-public class JoinVisitor extends DefaultVisitor {
-
-    /**
-     * reduce sink operator to variables
-     */
-    private HashMap<Operator, List<LogicalVariable>> reduceSinkToKeyVariables = new HashMap<Operator, List<LogicalVariable>>();
-
-    /**
-     * reduce sink operator to variables
-     */
-    private HashMap<Operator, List<String>> reduceSinkToFieldNames = new HashMap<Operator, List<String>>();
-
-    /**
-     * reduce sink operator to variables
-     */
-    private HashMap<Operator, List<TypeInfo>> reduceSinkToTypes = new HashMap<Operator, List<TypeInfo>>();
-
-    /**
-     * map a join operator (in hive) to its parent operators (in hive)
-     */
-    private HashMap<Operator, List<Operator>> operatorToHiveParents = new HashMap<Operator, List<Operator>>();
-
-    /**
-     * map a join operator (in hive) to its parent operators (in asterix)
-     */
-    private HashMap<Operator, List<ILogicalOperator>> operatorToAsterixParents = new HashMap<Operator, List<ILogicalOperator>>();
-
-    /**
-     * the latest traversed reduce sink operator
-     */
-    private Operator latestReduceSink = null;
-
-    /**
-     * the latest generated parent for join
-     */
-    private ILogicalOperator latestAlgebricksOperator = null;
-
-    /**
-     * process a join operator
-     */
-    @Override
-    public Mutable<ILogicalOperator> visit(JoinOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
-            Translator t) {
-        latestAlgebricksOperator = AlgebricksParentOperator.getValue();
-        translateJoinOperatorPreprocess(operator, t);
-        List<Operator> parents = operatorToHiveParents.get(operator);
-        if (parents.size() < operator.getParentOperators().size()) {
-            return null;
-        } else {
-            ILogicalOperator joinOp = translateJoinOperator(operator, AlgebricksParentOperator, t);
-            // clearStatus();
-            return new MutableObject<ILogicalOperator>(joinOp);
-        }
-    }
-
-    private void reorder(Byte[] order, List<ILogicalOperator> parents, List<Operator> hiveParents) {
-        ILogicalOperator[] lops = new ILogicalOperator[parents.size()];
-        Operator[] ops = new Operator[hiveParents.size()];
-
-        for (Operator op : hiveParents) {
-            ReduceSinkOperator rop = (ReduceSinkOperator) op;
-            ReduceSinkDesc rdesc = rop.getConf();
-            int tag = rdesc.getTag();
-
-            int index = -1;
-            for (int i = 0; i < order.length; i++)
-                if (order[i] == tag) {
-                    index = i;
-                    break;
-                }
-            lops[index] = parents.get(hiveParents.indexOf(op));
-            ops[index] = op;
-        }
-
-        parents.clear();
-        hiveParents.clear();
-
-        for (int i = 0; i < lops.length; i++) {
-            parents.add(lops[i]);
-            hiveParents.add(ops[i]);
-        }
-    }
-
-    /**
-     * translate a hive join operator to asterix join operator->assign
-     * operator->project operator
-     * 
-     * @param parentOperator
-     * @param operator
-     * @return
-     */
-    private ILogicalOperator translateJoinOperator(Operator operator, Mutable<ILogicalOperator> parentOperator,
-            Translator t) {
-
-        JoinDesc joinDesc = (JoinDesc) operator.getConf();
-
-        // get the projection expression (already re-written) from each source
-        // table
-        Map<Byte, List<ExprNodeDesc>> exprMap = joinDesc.getExprs();
-        reorder(joinDesc.getTagOrder(), operatorToAsterixParents.get(operator), operatorToHiveParents.get(operator));
-
-        // make an reduce join operator
-        ILogicalOperator currentOperator = generateJoinTree(joinDesc.getCondsList(),
-                operatorToAsterixParents.get(operator), operatorToHiveParents.get(operator), 0, t);
-        parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-
-        // add assign and project operator on top of a join
-        // output variables
-        ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-        Set<Entry<Byte, List<ExprNodeDesc>>> entries = exprMap.entrySet();
-        Iterator<Entry<Byte, List<ExprNodeDesc>>> iterator = entries.iterator();
-        while (iterator.hasNext()) {
-            List<ExprNodeDesc> outputExprs = iterator.next().getValue();
-            ILogicalOperator assignOperator = t.getAssignOperator(parentOperator, outputExprs, variables);
-
-            if (assignOperator != null) {
-                currentOperator = assignOperator;
-                parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-            }
-        }
-
-        ILogicalOperator po = new ProjectOperator(variables);
-        po.getInputs().add(parentOperator);
-        t.rewriteOperatorOutputSchema(variables, operator);
-        return po;
-    }
-
-    /**
-     * deal with reduce sink operator for the case of join
-     */
-    @Override
-    public Mutable<ILogicalOperator> visit(ReduceSinkOperator operator, Mutable<ILogicalOperator> parentOperator,
-            Translator t) {
-
-        Operator downStream = (Operator) operator.getChildOperators().get(0);
-        if (!(downStream instanceof JoinOperator))
-            return null;
-
-        ReduceSinkDesc desc = (ReduceSinkDesc) operator.getConf();
-        List<ExprNodeDesc> keys = desc.getKeyCols();
-        List<ExprNodeDesc> values = desc.getValueCols();
-        List<ExprNodeDesc> partitionCols = desc.getPartitionCols();
-
-        /**
-         * rewrite key, value, paritioncol expressions
-         */
-        for (ExprNodeDesc key : keys)
-            t.rewriteExpression(key);
-        for (ExprNodeDesc value : values)
-            t.rewriteExpression(value);
-        for (ExprNodeDesc col : partitionCols)
-            t.rewriteExpression(col);
-
-        ILogicalOperator currentOperator = null;
-
-        // add assign operator for keys if necessary
-        ArrayList<LogicalVariable> keyVariables = new ArrayList<LogicalVariable>();
-        ILogicalOperator assignOperator = t.getAssignOperator(parentOperator, keys, keyVariables);
-        if (assignOperator != null) {
-            currentOperator = assignOperator;
-            parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-        }
-
-        // add assign operator for values if necessary
-        ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-        assignOperator = t.getAssignOperator(parentOperator, values, variables);
-        if (assignOperator != null) {
-            currentOperator = assignOperator;
-            parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-        }
-
-        // unified schema: key, value
-        ArrayList<LogicalVariable> unifiedKeyValues = new ArrayList<LogicalVariable>();
-        unifiedKeyValues.addAll(keyVariables);
-        for (LogicalVariable value : variables)
-            if (keyVariables.indexOf(value) < 0)
-                unifiedKeyValues.add(value);
-
-        // insert projection operator, it is a *must*,
-        // in hive, reduce sink sometimes also do the projection operator's
-        // task
-        currentOperator = new ProjectOperator(unifiedKeyValues);
-        currentOperator.getInputs().add(parentOperator);
-        parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-
-        reduceSinkToKeyVariables.put(operator, keyVariables);
-        List<String> fieldNames = new ArrayList<String>();
-        List<TypeInfo> types = new ArrayList<TypeInfo>();
-        for (LogicalVariable var : unifiedKeyValues) {
-            fieldNames.add(var.toString());
-            types.add(t.getType(var));
-        }
-        reduceSinkToFieldNames.put(operator, fieldNames);
-        reduceSinkToTypes.put(operator, types);
-        t.rewriteOperatorOutputSchema(variables, operator);
-
-        latestAlgebricksOperator = currentOperator;
-        latestReduceSink = operator;
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-
-    /**
-     * partial rewrite a join operator
-     * 
-     * @param operator
-     * @param t
-     */
-    private void translateJoinOperatorPreprocess(Operator operator, Translator t) {
-        JoinDesc desc = (JoinDesc) operator.getConf();
-        ReduceSinkDesc reduceSinkDesc = (ReduceSinkDesc) latestReduceSink.getConf();
-        int tag = reduceSinkDesc.getTag();
-
-        Map<Byte, List<ExprNodeDesc>> exprMap = desc.getExprs();
-        List<ExprNodeDesc> exprs = exprMap.get(Byte.valueOf((byte) tag));
-
-        for (ExprNodeDesc expr : exprs)
-            t.rewriteExpression(expr);
-
-        List<Operator> parents = operatorToHiveParents.get(operator);
-        if (parents == null) {
-            parents = new ArrayList<Operator>();
-            operatorToHiveParents.put(operator, parents);
-        }
-        parents.add(latestReduceSink);
-
-        List<ILogicalOperator> asterixParents = operatorToAsterixParents.get(operator);
-        if (asterixParents == null) {
-            asterixParents = new ArrayList<ILogicalOperator>();
-            operatorToAsterixParents.put(operator, asterixParents);
-        }
-        asterixParents.add(latestAlgebricksOperator);
-    }
-
-    // generate a join tree from a list of exchange/reducesink operator
-    // both exchanges and reduce sinks have the same order
-    private ILogicalOperator generateJoinTree(List<JoinCondDesc> conds, List<ILogicalOperator> exchanges,
-            List<Operator> reduceSinks, int offset, Translator t) {
-        // get a list of reduce sink descs (input descs)
-        int inputSize = reduceSinks.size() - offset;
-
-        if (inputSize == 2) {
-            ILogicalOperator currentRoot;
-
-            List<ReduceSinkDesc> reduceSinkDescs = new ArrayList<ReduceSinkDesc>();
-            for (int i = reduceSinks.size() - 1; i >= offset; i--)
-                reduceSinkDescs.add((ReduceSinkDesc) reduceSinks.get(i).getConf());
-
-            // get the object inspector for the join
-            List<String> fieldNames = new ArrayList<String>();
-            List<TypeInfo> types = new ArrayList<TypeInfo>();
-            for (int i = reduceSinks.size() - 1; i >= offset; i--) {
-                fieldNames.addAll(reduceSinkToFieldNames.get(reduceSinks.get(i)));
-                types.addAll(reduceSinkToTypes.get(reduceSinks.get(i)));
-            }
-
-            // get number of equality conjunctions in the final join condition
-            int size = reduceSinkDescs.get(0).getKeyCols().size();
-
-            // make up the join conditon expression
-            List<ExprNodeDesc> joinConditionChildren = new ArrayList<ExprNodeDesc>();
-            for (int i = 0; i < size; i++) {
-                // create a join key pair
-                List<ExprNodeDesc> keyPair = new ArrayList<ExprNodeDesc>();
-                for (ReduceSinkDesc sink : reduceSinkDescs) {
-                    keyPair.add(sink.getKeyCols().get(i));
-                }
-                // create a hive equal condition
-                ExprNodeDesc equality = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
-                        new GenericUDFOPEqual(), keyPair);
-                // add the equal condition to the conjunction list
-                joinConditionChildren.add(equality);
-            }
-            // get final conjunction expression
-            ExprNodeDesc conjunct = null;
-
-            if (joinConditionChildren.size() > 1)
-                conjunct = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(),
-                        joinConditionChildren);
-            else if (joinConditionChildren.size() == 1)
-                conjunct = joinConditionChildren.get(0);
-            else {
-                // there is no join equality condition, equal-join
-                conjunct = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, new Boolean(true));
-            }
-            // get an ILogicalExpression from hive's expression
-            Mutable<ILogicalExpression> expression = t.translateScalarFucntion(conjunct);
-
-            Mutable<ILogicalOperator> leftBranch = new MutableObject<ILogicalOperator>(
-                    exchanges.get(exchanges.size() - 1));
-            Mutable<ILogicalOperator> rightBranch = new MutableObject<ILogicalOperator>(
-                    exchanges.get(exchanges.size() - 2));
-            // get the join operator
-            if (conds.get(offset).getType() == JoinDesc.LEFT_OUTER_JOIN) {
-                currentRoot = new LeftOuterJoinOperator(expression);
-                Mutable<ILogicalOperator> temp = leftBranch;
-                leftBranch = rightBranch;
-                rightBranch = temp;
-            } else if (conds.get(offset).getType() == JoinDesc.RIGHT_OUTER_JOIN) {
-                currentRoot = new LeftOuterJoinOperator(expression);
-            } else
-                currentRoot = new InnerJoinOperator(expression);
-
-            currentRoot.getInputs().add(leftBranch);
-            currentRoot.getInputs().add(rightBranch);
-
-            // rewriteOperatorOutputSchema(variables, operator);
-            return currentRoot;
-        } else {
-            // get the child join operator and insert and one-to-one exchange
-            ILogicalOperator joinSrcOne = generateJoinTree(conds, exchanges, reduceSinks, offset + 1, t);
-            // joinSrcOne.addInput(childJoin);
-
-            ILogicalOperator currentRoot;
-
-            List<ReduceSinkDesc> reduceSinkDescs = new ArrayList<ReduceSinkDesc>();
-            for (int i = offset; i < offset + 2; i++)
-                reduceSinkDescs.add((ReduceSinkDesc) reduceSinks.get(i).getConf());
-
-            // get the object inspector for the join
-            List<String> fieldNames = new ArrayList<String>();
-            List<TypeInfo> types = new ArrayList<TypeInfo>();
-            for (int i = offset; i < reduceSinks.size(); i++) {
-                fieldNames.addAll(reduceSinkToFieldNames.get(reduceSinks.get(i)));
-                types.addAll(reduceSinkToTypes.get(reduceSinks.get(i)));
-            }
-
-            // get number of equality conjunctions in the final join condition
-            int size = reduceSinkDescs.get(0).getKeyCols().size();
-
-            // make up the join condition expression
-            List<ExprNodeDesc> joinConditionChildren = new ArrayList<ExprNodeDesc>();
-            for (int i = 0; i < size; i++) {
-                // create a join key pair
-                List<ExprNodeDesc> keyPair = new ArrayList<ExprNodeDesc>();
-                for (ReduceSinkDesc sink : reduceSinkDescs) {
-                    keyPair.add(sink.getKeyCols().get(i));
-                }
-                // create a hive equal condition
-                ExprNodeDesc equality = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
-                        new GenericUDFOPEqual(), keyPair);
-                // add the equal condition to the conjunction list
-                joinConditionChildren.add(equality);
-            }
-            // get final conjunction expression
-            ExprNodeDesc conjunct = null;
-
-            if (joinConditionChildren.size() > 1)
-                conjunct = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(),
-                        joinConditionChildren);
-            else if (joinConditionChildren.size() == 1)
-                conjunct = joinConditionChildren.get(0);
-            else {
-                // there is no join equality condition, full outer join
-                conjunct = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, new Boolean(true));
-            }
-            // get an ILogicalExpression from hive's expression
-            Mutable<ILogicalExpression> expression = t.translateScalarFucntion(conjunct);
-
-            Mutable<ILogicalOperator> leftBranch = new MutableObject<ILogicalOperator>(joinSrcOne);
-            Mutable<ILogicalOperator> rightBranch = new MutableObject<ILogicalOperator>(exchanges.get(offset));
-
-            // get the join operator
-            if (conds.get(offset).getType() == JoinDesc.LEFT_OUTER_JOIN) {
-                currentRoot = new LeftOuterJoinOperator(expression);
-                Mutable<ILogicalOperator> temp = leftBranch;
-                leftBranch = rightBranch;
-                rightBranch = temp;
-            } else if (conds.get(offset).getType() == JoinDesc.RIGHT_OUTER_JOIN) {
-                currentRoot = new LeftOuterJoinOperator(expression);
-            } else
-                currentRoot = new InnerJoinOperator(expression);
-
-            // set the inputs from Algebricks join operator
-            // add the current table
-            currentRoot.getInputs().add(leftBranch);
-            currentRoot.getInputs().add(rightBranch);
-
-            return currentRoot;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java
deleted file mode 100644
index aa1837c..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.UDTFOperator;
-import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc;
-import org.apache.hadoop.hive.ql.plan.UDTFDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-/**
- * The lateral view join operator is used for FROM src LATERAL VIEW udtf()...
- * This operator was implemented with the following operator DAG in mind.
- * For a query such as
- * SELECT pageid, adid.* FROM example_table LATERAL VIEW explode(adid_list) AS
- * adid
- * The top of the operator DAG will look similar to
- * [Table Scan] | [Lateral View Forward] / \ [Select](*) [Select](adid_list) | |
- * | [UDTF] (explode) \ / [Lateral View Join] | | [Select] (pageid, adid.*) |
- * ....
- * Rows from the table scan operator are first to a lateral view forward
- * operator that just forwards the row and marks the start of a LV. The select
- * operator on the left picks all the columns while the select operator on the
- * right picks only the columns needed by the UDTF.
- * The output of select in the left branch and output of the UDTF in the right
- * branch are then sent to the lateral view join (LVJ). In most cases, the UDTF
- * will generate > 1 row for every row received from the TS, while the left
- * select operator will generate only one. For each row output from the TS, the
- * LVJ outputs all possible rows that can be created by joining the row from the
- * left select and one of the rows output from the UDTF.
- * Additional lateral views can be supported by adding a similar DAG after the
- * previous LVJ operator.
- */
-
-@SuppressWarnings("rawtypes")
-public class LateralViewJoinVisitor extends DefaultVisitor {
-
-    private UDTFDesc udtf;
-
-    private List<Mutable<ILogicalOperator>> parents = new ArrayList<Mutable<ILogicalOperator>>();
-
-    @Override
-    public Mutable<ILogicalOperator> visit(LateralViewJoinOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException {
-
-        parents.add(AlgebricksParentOperatorRef);
-        if (operator.getParentOperators().size() > parents.size()) {
-            return null;
-        }
-
-        ILogicalOperator parentOperator = null;
-        ILogicalExpression unnestArg = null;
-        List<LogicalVariable> projectVariables = new ArrayList<LogicalVariable>();
-        for (Mutable<ILogicalOperator> parentLOpRef : parents) {
-            VariableUtilities.getLiveVariables(parentLOpRef.getValue(), projectVariables);
-        }
-        for (Operator parentOp : operator.getParentOperators()) {
-            if (parentOp instanceof UDTFOperator) {
-                int index = operator.getParentOperators().indexOf(parentOp);
-                List<LogicalVariable> unnestVars = new ArrayList<LogicalVariable>();
-                VariableUtilities.getLiveVariables(parents.get(index).getValue(), unnestVars);
-                unnestArg = new VariableReferenceExpression(unnestVars.get(0));
-                parentOperator = parents.get(index).getValue();
-            }
-        }
-
-        LogicalVariable var = t.getVariable(udtf.toString(), TypeInfoFactory.unknownTypeInfo);
-        Mutable<ILogicalExpression> unnestExpr = t.translateUnnestFunction(udtf, new MutableObject<ILogicalExpression>(
-                unnestArg));
-        ILogicalOperator currentOperator = new UnnestOperator(var, unnestExpr);
-
-        List<LogicalVariable> outputVars = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(parents.get(0).getValue(), outputVars);
-        outputVars.add(var);
-        ILogicalOperator inputProjectOperator = new ProjectOperator(projectVariables);
-        currentOperator.getInputs().add(new MutableObject<ILogicalOperator>(inputProjectOperator));
-        inputProjectOperator.getInputs().addAll(parentOperator.getInputs());
-
-        parents.clear();
-        udtf = null;
-        List<ColumnInfo> inputSchema = operator.getSchema().getSignature();
-        rewriteOperatorDesc(outputVars, operator.getConf(), inputSchema, t);
-        //t.rewriteOperatorOutputSchema(outputVars, operator);
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(UDTFOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) {
-        Schema currentSchema = t.generateInputSchema(operator.getParentOperators().get(0));
-        udtf = (UDTFDesc) operator.getConf();
-
-        // populate the schema from upstream operator
-        operator.setSchema(operator.getParentOperators().get(0).getSchema());
-        List<LogicalVariable> latestOutputSchema = t.getVariablesFromSchema(currentSchema);
-        t.rewriteOperatorOutputSchema(latestOutputSchema, operator);
-        return null;
-    }
-
-    private void rewriteOperatorDesc(List<LogicalVariable> variables, LateralViewJoinDesc desc,
-            List<ColumnInfo> schema, Translator t) {
-        List<String> outputFieldNames = desc.getOutputInternalColNames();
-        for (int i = 0; i < variables.size(); i++) {
-            LogicalVariable var = variables.get(i);
-            String fieldName = outputFieldNames.get(i);
-            String tabAlias = findTabAlias(fieldName, schema);
-            fieldName = tabAlias + "." + fieldName;
-            if (fieldName.indexOf("$$") < 0) {
-                //outputFieldNames.set(i, var.toString());
-                t.updateVariable(fieldName, var);
-            }
-        }
-    }
-
-    private String findTabAlias(String fieldName, List<ColumnInfo> schema) {
-        for (int i = 0; i < schema.size(); i++) {
-            ColumnInfo column = schema.get(i);
-            if (column.getInternalName().equals(fieldName)) {
-                return column.getTabAlias();
-            }
-        }
-        return "null";
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java
deleted file mode 100644
index 0876642..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.plan.LimitDesc;
-
-import edu.uci.ics.hivesterix.logical.expression.HivesterixConstantValue;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-
-public class LimitVisitor extends DefaultVisitor {
-
-    @Override
-    public Mutable<ILogicalOperator> visit(LimitOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) {
-        Schema currentSchema = t.generateInputSchema(operator.getParentOperators().get(0));
-
-        LimitDesc desc = (LimitDesc) operator.getConf();
-        int limit = desc.getLimit();
-        Integer limitValue = new Integer(limit);
-
-        ILogicalExpression expr = new ConstantExpression(new HivesterixConstantValue(limitValue));
-        ILogicalOperator currentOperator = new edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator(
-                expr, true);
-        currentOperator.getInputs().add(AlgebricksParentOperatorRef);
-
-        operator.setSchema(operator.getParentOperators().get(0).getSchema());
-        List<LogicalVariable> latestOutputSchema = t.getVariablesFromSchema(currentSchema);
-        t.rewriteOperatorOutputSchema(latestOutputSchema, operator);
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java
deleted file mode 100644
index 3ed9786..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-
-@SuppressWarnings("rawtypes")
-public class MapJoinVisitor extends DefaultVisitor {
-
-    /**
-     * map a join operator (in hive) to its parent operators (in asterix)
-     */
-    private HashMap<Operator, List<Mutable<ILogicalOperator>>> opMap = new HashMap<Operator, List<Mutable<ILogicalOperator>>>();
-
-    @Override
-    public Mutable<ILogicalOperator> visit(MapJoinOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) {
-        List<Operator<? extends OperatorDesc>> joinSrc = operator.getParentOperators();
-        List<Mutable<ILogicalOperator>> parents = opMap.get(operator);
-        if (parents == null) {
-            parents = new ArrayList<Mutable<ILogicalOperator>>();
-            opMap.put(operator, parents);
-        }
-        parents.add(AlgebricksParentOperatorRef);
-        if (joinSrc.size() != parents.size())
-            return null;
-
-        ILogicalOperator currentOperator;
-        // make an map join operator
-        // TODO: will have trouble for n-way joins
-        MapJoinDesc joinDesc = (MapJoinDesc) operator.getConf();
-
-        Map<Byte, List<ExprNodeDesc>> keyMap = joinDesc.getKeys();
-        // get the projection expression (already re-written) from each source
-        // table
-        Map<Byte, List<ExprNodeDesc>> exprMap = joinDesc.getExprs();
-
-        int inputSize = operator.getParentOperators().size();
-        // get a list of reduce sink descs (input descs)
-
-        // get the parent operator
-        List<Mutable<ILogicalOperator>> parentOps = parents;
-
-        List<String> fieldNames = new ArrayList<String>();
-        List<TypeInfo> types = new ArrayList<TypeInfo>();
-        for (Operator ts : joinSrc) {
-            List<ColumnInfo> columns = ts.getSchema().getSignature();
-            for (ColumnInfo col : columns) {
-                fieldNames.add(col.getInternalName());
-                types.add(col.getType());
-            }
-        }
-
-        // get number of equality conjunctions in the final join condition
-        Set<Entry<Byte, List<ExprNodeDesc>>> keyEntries = keyMap.entrySet();
-        Iterator<Entry<Byte, List<ExprNodeDesc>>> entry = keyEntries.iterator();
-
-        int size = 0;
-        if (entry.hasNext())
-            size = entry.next().getValue().size();
-
-        // make up the join conditon expression
-        List<ExprNodeDesc> joinConditionChildren = new ArrayList<ExprNodeDesc>();
-        for (int i = 0; i < size; i++) {
-            // create a join key pair
-            List<ExprNodeDesc> keyPair = new ArrayList<ExprNodeDesc>();
-            for (int j = 0; j < inputSize; j++) {
-                keyPair.add(keyMap.get(Byte.valueOf((byte) j)).get(i));
-            }
-            // create a hive equal condition
-            ExprNodeDesc equality = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
-                    new GenericUDFOPEqual(), keyPair);
-            // add the equal condition to the conjunction list
-            joinConditionChildren.add(equality);
-        }
-        // get final conjunction expression
-        ExprNodeDesc conjunct = null;
-
-        if (joinConditionChildren.size() > 1)
-            conjunct = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(),
-                    joinConditionChildren);
-        else if (joinConditionChildren.size() == 1)
-            conjunct = joinConditionChildren.get(0);
-        else {
-            // there is no join equality condition, full outer join
-            conjunct = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, new Boolean(true));
-        }
-        // get an ILogicalExpression from hive's expression
-        Mutable<ILogicalExpression> expression = t.translateScalarFucntion(conjunct);
-
-        ArrayList<LogicalVariable> left = new ArrayList<LogicalVariable>();
-        ArrayList<LogicalVariable> right = new ArrayList<LogicalVariable>();
-
-        Set<Entry<Byte, List<ExprNodeDesc>>> kentries = keyMap.entrySet();
-        Iterator<Entry<Byte, List<ExprNodeDesc>>> kiterator = kentries.iterator();
-        int iteration = 0;
-        ILogicalOperator assignOperator = null;
-        while (kiterator.hasNext()) {
-            List<ExprNodeDesc> outputExprs = kiterator.next().getValue();
-
-            if (iteration == 0)
-                assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, outputExprs, left);
-            else
-                assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, outputExprs, right);
-
-            if (assignOperator != null) {
-                currentOperator = assignOperator;
-                AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-            }
-            iteration++;
-        }
-
-        List<Mutable<ILogicalOperator>> inputs = parentOps;
-
-        // get the join operator
-        currentOperator = new InnerJoinOperator(expression);
-
-        // set the inputs from asterix join operator
-        for (Mutable<ILogicalOperator> input : inputs)
-            currentOperator.getInputs().add(input);
-        AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-
-        // add assign and project operator
-        // output variables
-        ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-        Set<Entry<Byte, List<ExprNodeDesc>>> entries = exprMap.entrySet();
-        Iterator<Entry<Byte, List<ExprNodeDesc>>> iterator = entries.iterator();
-        while (iterator.hasNext()) {
-            List<ExprNodeDesc> outputExprs = iterator.next().getValue();
-            assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, outputExprs, variables);
-
-            if (assignOperator != null) {
-                currentOperator = assignOperator;
-                AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-            }
-        }
-
-        currentOperator = new ProjectOperator(variables);
-        currentOperator.getInputs().add(AlgebricksParentOperatorRef);
-        t.rewriteOperatorOutputSchema(variables, operator);
-        // opMap.clear();
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java
deleted file mode 100644
index d69fb1c..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.SelectDesc;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-
-public class ProjectVisitor extends DefaultVisitor {
-
-    /**
-     * translate project operator
-     */
-    @Override
-    public Mutable<ILogicalOperator> visit(SelectOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
-            Translator t) {
-
-        SelectDesc desc = (SelectDesc) operator.getConf();
-
-        if (desc == null)
-            return null;
-
-        List<ExprNodeDesc> cols = desc.getColList();
-
-        if (cols == null)
-            return null;
-
-        // insert assign operator if necessary
-        ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-
-        for (ExprNodeDesc expr : cols)
-            t.rewriteExpression(expr);
-
-        ILogicalOperator assignOp = t.getAssignOperator(AlgebricksParentOperator, cols, variables);
-        ILogicalOperator currentOperator = null;
-        if (assignOp != null) {
-            currentOperator = assignOp;
-            AlgebricksParentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-        }
-
-        currentOperator = new ProjectOperator(variables);
-        currentOperator.getInputs().add(AlgebricksParentOperator);
-        t.rewriteOperatorOutputSchema(variables, operator);
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java
deleted file mode 100644
index 277af50..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ExtractOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-
-public class SortVisitor extends DefaultVisitor {
-
-    @SuppressWarnings("rawtypes")
-    @Override
-    public Mutable<ILogicalOperator> visit(ReduceSinkOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException {
-        ReduceSinkDesc desc = (ReduceSinkDesc) operator.getConf();
-        Operator downStream = (Operator) operator.getChildOperators().get(0);
-        List<ExprNodeDesc> keys = desc.getKeyCols();
-        if (!(downStream instanceof ExtractOperator && desc.getNumReducers() == 1 && keys.size() > 0)) {
-            return null;
-        }
-
-        List<ExprNodeDesc> schema = new ArrayList<ExprNodeDesc>();
-        List<ExprNodeDesc> values = desc.getValueCols();
-        List<ExprNodeDesc> partitionCols = desc.getPartitionCols();
-        for (ExprNodeDesc key : keys) {
-            t.rewriteExpression(key);
-        }
-        for (ExprNodeDesc value : values) {
-            t.rewriteExpression(value);
-        }
-        for (ExprNodeDesc col : partitionCols) {
-            t.rewriteExpression(col);
-        }
-
-        // add a order-by operator and limit if any
-        List<Pair<IOrder, Mutable<ILogicalExpression>>> pairs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
-        char[] orders = desc.getOrder().toCharArray();
-        int i = 0;
-        for (ExprNodeDesc key : keys) {
-            Mutable<ILogicalExpression> expr = t.translateScalarFucntion(key);
-            IOrder order = orders[i] == '+' ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER;
-
-            Pair<IOrder, Mutable<ILogicalExpression>> pair = new Pair<IOrder, Mutable<ILogicalExpression>>(order, expr);
-            pairs.add(pair);
-            i++;
-        }
-
-        // get input variables
-        ArrayList<LogicalVariable> inputVariables = new ArrayList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(AlgebricksParentOperatorRef.getValue(), inputVariables);
-
-        ArrayList<LogicalVariable> keyVariables = new ArrayList<LogicalVariable>();
-        ILogicalOperator currentOperator;
-        ILogicalOperator assignOp = t.getAssignOperator(AlgebricksParentOperatorRef, keys, keyVariables);
-        if (assignOp != null) {
-            currentOperator = assignOp;
-            AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-        }
-
-        OrderColumn[] keyColumns = new OrderColumn[keyVariables.size()];
-
-        for (int j = 0; j < keyColumns.length; j++)
-            keyColumns[j] = new OrderColumn(keyVariables.get(j), pairs.get(j).first.getKind());
-
-        // handle order operator
-        currentOperator = new OrderOperator(pairs);
-        currentOperator.getInputs().add(AlgebricksParentOperatorRef);
-        AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-
-        // project back, remove generated sort-key columns if any
-        if (assignOp != null) {
-            currentOperator = new ProjectOperator(inputVariables);
-            currentOperator.getInputs().add(AlgebricksParentOperatorRef);
-            AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-        }
-
-        /**
-         * a special rule for hive's order by output schema of reduce sink
-         * operator only contains the columns
-         */
-        for (ExprNodeDesc value : values) {
-            schema.add(value);
-        }
-
-        ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-        ILogicalOperator assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, schema, variables);
-        t.rewriteOperatorOutputSchema(variables, operator);
-
-        if (assignOperator != null) {
-            currentOperator = assignOperator;
-            AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-        }
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java
deleted file mode 100644
index 25abdec..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveDataSink;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveDataSource;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveMetaDataProvider;
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-public class TableScanWriteVisitor extends DefaultVisitor {
-
-    /**
-     * map from alias to partition desc
-     */
-    private HashMap<String, PartitionDesc> aliasToPathMap;
-
-    /**
-     * map from partition desc to data source
-     */
-    private HashMap<PartitionDesc, IDataSource<PartitionDesc>> dataSourceMap = new HashMap<PartitionDesc, IDataSource<PartitionDesc>>();
-
-    /**
-     * constructor
-     * 
-     * @param aliasToPathMap
-     */
-    public TableScanWriteVisitor(HashMap<String, PartitionDesc> aliasToPathMap) {
-        this.aliasToPathMap = aliasToPathMap;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(TableScanOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {    	
-        TableScanDesc desc = (TableScanDesc) operator.getConf();
-        if (desc == null || desc.getAlias()==null) {
-            List<LogicalVariable> schema = new ArrayList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(AlgebricksParentOperator.getValue(), schema);
-            t.rewriteOperatorOutputSchema(schema, operator);
-            return null;
-        }
-
-        List<ColumnInfo> columns = operator.getSchema().getSignature();
-        for (int i = columns.size() - 1; i >= 0; i--)
-            if (columns.get(i).getIsVirtualCol() == true)
-                columns.remove(i);
-
-        // start with empty tuple operator
-        List<TypeInfo> types = new ArrayList<TypeInfo>();
-        ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-        List<String> names = new ArrayList<String>();
-        for (ColumnInfo column : columns) {
-            types.add(column.getType());
-
-            LogicalVariable var = t.getVariableFromFieldName(column.getTabAlias() + "." + column.getInternalName());
-            LogicalVariable varNew;
-
-            if (var != null) {
-                varNew = t.getVariable(column.getTabAlias() + "." + column.getInternalName() + operator.toString(),
-                        column.getType());
-                t.replaceVariable(var, varNew);
-                var = varNew;
-            } else
-                var = t.getNewVariable(column.getTabAlias() + "." + column.getInternalName(), column.getType());
-
-            variables.add(var);
-            names.add(column.getInternalName());
-        }
-        Schema currentSchema = new Schema(names, types);
-
-        String alias = desc.getAlias();
-        PartitionDesc partDesc = aliasToPathMap.get(alias);
-        IDataSource<PartitionDesc> dataSource = new HiveDataSource<PartitionDesc>(partDesc, currentSchema.getSchema());
-        ILogicalOperator currentOperator = new DataSourceScanOperator(variables, dataSource);
-
-        // set empty tuple source operator
-        ILogicalOperator ets = new EmptyTupleSourceOperator();
-        currentOperator.getInputs().add(new MutableObject<ILogicalOperator>(ets));
-
-        // setup data source
-        dataSourceMap.put(partDesc, dataSource);
-        t.rewriteOperatorOutputSchema(variables, operator);
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(FileSinkOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) {
-        if (hiveOperator.getChildOperators() != null && hiveOperator.getChildOperators().size() > 0)
-            return null;
-
-        Schema currentSchema = t.generateInputSchema(hiveOperator.getParentOperators().get(0));
-
-        IDataSink sink = new HiveDataSink(hiveOperator, currentSchema.getSchema());
-        List<Mutable<ILogicalExpression>> exprList = new ArrayList<Mutable<ILogicalExpression>>();
-        for (String column : currentSchema.getNames()) {
-            exprList.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(t.getVariable(column))));
-        }
-
-        ILogicalOperator currentOperator = new WriteOperator(exprList, sink);
-        if (AlgebricksParentOperator != null) {
-            currentOperator.getInputs().add(AlgebricksParentOperator);
-        }
-
-        IMetadataProvider<PartitionDesc, Object> metaData = new HiveMetaDataProvider<PartitionDesc, Object>(
-                hiveOperator, currentSchema, dataSourceMap);
-        t.setMetadataProvider(metaData);
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java
deleted file mode 100644
index c710f3f..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-public class UnionVisitor extends DefaultVisitor {
-
-    List<Mutable<ILogicalOperator>> parents = new ArrayList<Mutable<ILogicalOperator>>();
-
-    @Override
-    public Mutable<ILogicalOperator> visit(UnionOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
-            Translator t) throws AlgebricksException {
-
-        parents.add(AlgebricksParentOperator);
-        if (operator.getParentOperators().size() > parents.size()) {
-            return null;
-        }
-
-        List<LogicalVariable> leftVars = new ArrayList<LogicalVariable>();
-        List<LogicalVariable> rightVars = new ArrayList<LogicalVariable>();
-
-        VariableUtilities.getLiveVariables(parents.get(0).getValue(), leftVars);
-        VariableUtilities.getLiveVariables(parents.get(1).getValue(), rightVars);
-
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> triples = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>();
-        List<LogicalVariable> unionVars = new ArrayList<LogicalVariable>();
-
-        for (int i = 0; i < leftVars.size(); i++) {
-            LogicalVariable unionVar = t.getVariable(
-                    leftVars.get(i).getId() + "union" + AlgebricksParentOperator.hashCode(),
-                    TypeInfoFactory.unknownTypeInfo);
-            unionVars.add(unionVar);
-            Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
-                    leftVars.get(i), rightVars.get(i), unionVar);
-            t.replaceVariable(leftVars.get(i), unionVar);
-            t.replaceVariable(rightVars.get(i), unionVar);
-            triples.add(triple);
-        }
-        ILogicalOperator currentOperator = new edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator(
-                triples);
-        for (Mutable<ILogicalOperator> parent : parents)
-            currentOperator.getInputs().add(parent);
-
-        t.rewriteOperatorOutputSchema(unionVars, operator);
-        parents.clear();
-        return new MutableObject<ILogicalOperator>(currentOperator);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java
deleted file mode 100644
index f79b79f..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor.base;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hadoop.hive.ql.exec.CollectOperator;
-import org.apache.hadoop.hive.ql.exec.ExtractOperator;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.ForwardOperator;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
-import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.MapOperator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.ScriptOperator;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.UDTFOperator;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-
-/**
- * a default empty implementation of visitor
- * 
- * @author yingyib
- */
-public class DefaultVisitor implements Visitor {
-
-    @Override
-    public Mutable<ILogicalOperator> visit(CollectOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(JoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(ExtractOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(MapJoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(SMBMapJoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    public Mutable<ILogicalOperator> visit(FileSinkOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    public Mutable<ILogicalOperator> visit(ReduceSinkOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(FilterOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(ForwardOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(GroupByOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(LateralViewForwardOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(LateralViewJoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(LimitOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(MapOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(ScriptOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(SelectOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(TableScanOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(UDTFOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
-            Translator t) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Mutable<ILogicalOperator> visit(UnionOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
-            Translator t) throws AlgebricksException {
-        return null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java
deleted file mode 100644
index 5b6ac50..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor.base;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.plan.AggregationDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-import org.apache.hadoop.hive.ql.plan.UDTFDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-@SuppressWarnings("rawtypes")
-public interface Translator {
-
-    /**
-     * generate input schema
-     * 
-     * @param operator
-     * @return
-     */
-    public Schema generateInputSchema(Operator operator);
-
-    /**
-     * rewrite the names of output columns for feture expression evaluators to
-     * use
-     * 
-     * @param operator
-     */
-    public void rewriteOperatorOutputSchema(List<LogicalVariable> vars, Operator operator);
-
-    /**
-     * rewrite the names of output columns for feture expression evaluators to
-     * use
-     * 
-     * @param operator
-     */
-    public void rewriteOperatorOutputSchema(Operator operator);
-
-    /**
-     * rewrite an expression and substitute variables
-     * 
-     * @param expr
-     *            hive expression
-     */
-    public void rewriteExpression(ExprNodeDesc expr);
-
-    /**
-     * rewrite an expression and substitute variables
-     * 
-     * @param expr
-     *            hive expression
-     */
-    public void rewriteExpressionPartial(ExprNodeDesc expr);
-
-    /**
-     * get an assign operator as a child of parent
-     * 
-     * @param parent
-     * @param cols
-     * @param variables
-     * @return
-     */
-    public ILogicalOperator getAssignOperator(Mutable<ILogicalOperator> parent, List<ExprNodeDesc> cols,
-            ArrayList<LogicalVariable> variables);
-
-    /**
-     * get type for a logical variable
-     * 
-     * @param var
-     * @return type info
-     */
-    public TypeInfo getType(LogicalVariable var);
-
-    /**
-     * translate an expression from hive to Algebricks
-     * 
-     * @param desc
-     * @return
-     */
-    public Mutable<ILogicalExpression> translateScalarFucntion(ExprNodeDesc desc);
-
-    /**
-     * translate an aggregation from hive to Algebricks
-     * 
-     * @param aggregateDesc
-     * @return
-     */
-    public Mutable<ILogicalExpression> translateAggregation(AggregationDesc aggregateDesc);
-
-    /**
-     * translate unnesting (UDTF) function expression
-     * 
-     * @param aggregator
-     * @return
-     */
-    public Mutable<ILogicalExpression> translateUnnestFunction(UDTFDesc udtfDesc, Mutable<ILogicalExpression> argument);
-
-    /**
-     * get variable from a schema
-     * 
-     * @param schema
-     * @return
-     */
-    public List<LogicalVariable> getVariablesFromSchema(Schema schema);
-
-    /**
-     * get variable from name
-     * 
-     * @param name
-     * @return
-     */
-    public LogicalVariable getVariable(String name);
-
-    /**
-     * get variable from field name
-     * 
-     * @param name
-     * @return
-     */
-    public LogicalVariable getVariableFromFieldName(String name);
-
-    /**
-     * get variable from name, type
-     * 
-     * @param fieldName
-     * @param type
-     * @return
-     */
-    public LogicalVariable getVariable(String fieldName, TypeInfo type);
-
-    /**
-     * get new variable from name, type
-     * 
-     * @param fieldName
-     * @param type
-     * @return
-     */
-    public LogicalVariable getNewVariable(String fieldName, TypeInfo type);
-
-    /**
-     * update a name-variable binding
-     * 
-     * @param fieldName
-     * @param variable
-     */
-    public void updateVariable(String fieldName, LogicalVariable variable);
-
-    /**
-     * set the metadata provider
-     * 
-     * @param metadata
-     */
-    public void setMetadataProvider(IMetadataProvider<PartitionDesc, Object> metadata);
-
-    /**
-     * get the metadata provider
-     * 
-     * @param metadata
-     */
-    public IMetadataProvider<PartitionDesc, Object> getMetadataProvider();
-
-    /**
-     * replace the variable
-     * 
-     * @param oldVar
-     * @param newVar
-     */
-    public void replaceVariable(LogicalVariable oldVar, LogicalVariable newVar);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java
deleted file mode 100644
index 2291d5f..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor.base;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hadoop.hive.ql.exec.CollectOperator;
-import org.apache.hadoop.hive.ql.exec.ExtractOperator;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.ForwardOperator;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
-import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.MapOperator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.ScriptOperator;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.UDTFOperator;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-
-public interface Visitor {
-
-    public Mutable<ILogicalOperator> visit(CollectOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(JoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(ExtractOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(MapJoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(SMBMapJoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(FilterOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(ForwardOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(GroupByOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(LateralViewForwardOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(LateralViewJoinOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(LimitOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(MapOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(ScriptOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(SelectOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(TableScanOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(FileSinkOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(ReduceSinkOperator hiveOperator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(UDTFOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
-    public Mutable<ILogicalOperator> visit(UnionOperator operator,
-            Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/pom.xml
----------------------------------------------------------------------
diff --git a/hivesterix/pom.xml b/hivesterix/pom.xml
deleted file mode 100644
index ccaed66..0000000
--- a/hivesterix/pom.xml
+++ /dev/null
@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ! Copyright 2009-2013 by The Regents of the University of California
- ! Licensed under the Apache License, Version 2.0 (the "License");
- ! you may not use this file except in compliance with the License.
- ! you may obtain a copy of the License from
- ! 
- !     http://www.apache.org/licenses/LICENSE-2.0
- ! 
- ! Unless required by applicable law or agreed to in writing, software
- ! distributed under the License is distributed on an "AS IS" BASIS,
- ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ! See the License for the specific language governing permissions and
- ! limitations under the License.
- !-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<groupId>edu.uci.ics.hyracks</groupId>
-	<artifactId>hivesterix</artifactId>
-	<version>0.2.14-SNAPSHOT</version>
-	<packaging>pom</packaging>
-	<name>hivesterix</name>
-
-	<parent>
-	  <groupId>edu.uci.ics.hyracks</groupId>
-	  <artifactId>fullstack</artifactId>
-	  <version>0.2.14-SNAPSHOT</version>
-	</parent>
-
-	<properties>
-		<jvm.extraargs />
-	</properties>
-
-	<profiles>
-		<profile>
-			<id>macosx</id>
-			<activation>
-				<os>
-					<name>mac os x</name>
-				</os>
-				<jdk>1.7</jdk>
-			</activation>
-			<properties>
-				<jvm.extraargs>-Djava.nio.channels.spi.SelectorProvider=sun.nio.ch.KQueueSelectorProvider</jvm.extraargs>
-			</properties>
-		</profile>
-	</profiles>
-
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-release-plugin</artifactId>
-				<version>2.0</version>
-				<configuration>
-					<goals>package source:jar javadoc:jar deploy:deploy</goals>
-				</configuration>
-			</plugin>
-			<plugin>
-				<groupId>org.codehaus.mojo</groupId>
-				<artifactId>versions-maven-plugin</artifactId>
-				<version>1.2</version>
-			</plugin>
-		</plugins>
-	</build>
-
-	<reporting>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-changelog-plugin</artifactId>
-				<version>2.2</version>
-			</plugin>
-		</plugins>
-	</reporting>
-
-	<distributionManagement>
-		<repository>
-			<id>hyracks-releases</id>
-			<url>http://obelix.ics.uci.edu/nexus/content/repositories/hyracks-releases/</url>
-		</repository>
-		<snapshotRepository>
-			<id>hyracks-snapshots</id>
-			<url>http://obelix.ics.uci.edu/nexus/content/repositories/hyracks-snapshots/</url>
-		</snapshotRepository>
-	</distributionManagement>
-
-	<repositories>
-        <repository>
-        <id>maven-central</id>
-        <url>http://repo1.maven.org/maven2/</url>
-        </repository>
-		<repository>
-			<id>hyracks-public</id>
-			<url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public/</url>
-		</repository>
-		<repository>
-			<id>jboss-public</id>
-			<url>https://repository.jboss.org/nexus/content/groups/public/</url>
-		</repository>
-		<repository>
-			<releases>
-				<enabled>true</enabled>
-				<updatePolicy>always</updatePolicy>
-				<checksumPolicy>warn</checksumPolicy>
-			</releases>
-			<snapshots>
-				<enabled>true</enabled>
-				<updatePolicy>always</updatePolicy>
-				<checksumPolicy>fail</checksumPolicy>
-			</snapshots>
-			<id>third-party</id>
-			<url>http://obelix.ics.uci.edu/nexus/content/repositories/third-party</url>
-		</repository>
-		<repository>
-			<releases>
-				<enabled>true</enabled>
-				<updatePolicy>always</updatePolicy>
-				<checksumPolicy>warn</checksumPolicy>
-			</releases>
-			<snapshots>
-				<enabled>true</enabled>
-				<updatePolicy>always</updatePolicy>
-				<checksumPolicy>fail</checksumPolicy>
-			</snapshots>
-			<id>hyracks-public-release</id>
-			<url>http://obelix.ics.uci.edu/nexus/content/repositories/hyracks-public-releases</url>
-		</repository>
-	</repositories>
-
-	<pluginRepositories>
-		<pluginRepository>
-			<id>hyracks-public</id>
-			<url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public/</url>
-			<releases>
-				<updatePolicy>always</updatePolicy>
-			</releases>
-		</pluginRepository>
-	</pluginRepositories>
-
-	<modules>
-		<module>hivesterix-runtime</module>
-		<module>hivesterix-translator</module>
-		<module>hivesterix-optimizer</module>
-		<module>hivesterix-serde</module>
-		<module>hivesterix-dist</module>
-		<module>hivesterix-common</module>
-	</modules>
-</project>


Mime
View raw message