hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From athu...@apache.org
Subject svn commit: r737291 [2/8] - in /hadoop/hive/trunk: ./ eclipse-templates/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/lib/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apache/hadoop/hive/ql/parse...
Date Sat, 24 Jan 2009 01:58:06 GMT
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java Sat Jan 24 01:58:01 2009
@@ -43,7 +43,7 @@
    * @param nd the reduce sink operator encountered
    * @param opProcCtx context
    */
-  public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+  public Object process(Node nd, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException {
     ReduceSinkOperator op = (ReduceSinkOperator)nd;
     GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
 
@@ -69,6 +69,7 @@
     }
 
     mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
+    return null;
   }
 
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Sat Jan 24 01:58:01 2009
@@ -44,7 +44,7 @@
    * @param nd the table sink operator encountered
    * @param opProcCtx context
    */
-  public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+  public Object process(Node nd, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException {
     TableScanOperator op = (TableScanOperator)nd;
     GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
     ParseContext parseCtx = ctx.getParseCtx();
@@ -62,10 +62,11 @@
         String currAliasId = alias;
         ctx.setCurrAliasId(currAliasId);
         mapCurrCtx.put(op, new GenMapRedCtx(currTask, currTopOp, currAliasId));
-        return;
+        return null;
       }
     }
     assert false;
+    return null;
   }
 
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java Sat Jan 24 01:58:01 2009
@@ -154,7 +154,7 @@
         }
 
         // Create function desc
-        desc = SemanticAnalyzer.getXpathOrFuncExprNodeDesc(expr, isFunction, children);
+        desc = TypeCheckProcFactory.DefaultExprProcessor.getXpathOrFuncExprNodeDesc(expr, isFunction, children);
         
         if (desc instanceof exprNodeFuncDesc && (
             ((exprNodeFuncDesc)desc).getUDFMethod().getDeclaringClass().equals(UDFOPAnd.class) 
@@ -245,7 +245,7 @@
       if (this.prunerExpr == null)
         this.prunerExpr = desc;
       else
-        this.prunerExpr = SemanticAnalyzer.getFuncExprNodeDesc("OR", this.prunerExpr, desc);
+        this.prunerExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("OR", this.prunerExpr, desc);
     }
   }
 
@@ -264,7 +264,7 @@
       if (this.prunerExpr == null)
         this.prunerExpr = desc;
       else
-        this.prunerExpr = SemanticAnalyzer.getFuncExprNodeDesc("AND", this.prunerExpr, desc);
+        this.prunerExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("AND", this.prunerExpr, desc);
     }
   }
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java Sat Jan 24 01:58:01 2009
@@ -70,14 +70,15 @@
     return ret.toString();
   }
   
-  public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException {
+  public Object process(Node nd, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException {
     Operator<? extends Serializable> op = (Operator<? extends Serializable>)nd;
     if (opMap.get(op) == null) {
       opMap.put(op, curNum++);
     }
     out.println("[" + opMap.get(op) + "] " + op.getClass().getName() + " =p=> " + getParents(op) + " =c=> " + getChildren(op));
     if(op.getConf() == null) {
-      return;
+      return null;
     }
+    return null;
   }
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Jan 24 01:58:01 2009
@@ -23,7 +23,6 @@
 import java.io.Serializable;
 import java.lang.reflect.Method;
 
-import org.antlr.runtime.tree.*;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
@@ -35,6 +34,7 @@
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
 import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
 import org.apache.hadoop.hive.ql.lib.Dispatcher;
 import org.apache.hadoop.hive.ql.lib.GraphWalker;
@@ -901,7 +901,7 @@
 
     Operator output = putOpInsertMap(
       OperatorFactory.getAndMakeChild(
-        new filterDesc(genExprNodeDesc(qb.getMetaData(), (ASTNode)whereExpr.getChild(0), inputRR)),
+        new filterDesc(genExprNodeDesc((ASTNode)whereExpr.getChild(0), inputRR)),
           new RowSchema(inputRR.getColumnInfos()), input), inputRR);
  
     LOG.debug("Created Filter Plan for " + qb.getId() + ":" + dest + " row schema: " + inputRR.toString());
@@ -921,7 +921,7 @@
     RowResolver inputRR = inputCtx.getRR();
     Operator output = putOpInsertMap(
       OperatorFactory.getAndMakeChild(
-        new filterDesc(genExprNodeDesc(qb.getMetaData(), condn, inputRR)),
+        new filterDesc(genExprNodeDesc(condn, inputRR)),
           new RowSchema(inputRR.getColumnInfos()), input), inputRR);
  
     LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " + inputRR.toString());
@@ -1150,7 +1150,7 @@
             genColList(tabAlias, alias, expr, col_list, inputRR, pos, out_rwsch);
             selectStar = true;
           } else {
-            exprNodeDesc exp = genExprNodeDesc(qb.getMetaData(), expr, inputRR);
+            exprNodeDesc exp = genExprNodeDesc(expr, inputRR);
             col_list.add(exp);
             if (!StringUtils.isEmpty(alias) &&
                 (out_rwsch.get(null, colAlias) != null)) {
@@ -1164,7 +1164,7 @@
         }
       } else {
         // Case when this is an expression
-        exprNodeDesc exp = genExprNodeDesc(qb.getMetaData(), sel, inputRR);
+        exprNodeDesc exp = genExprNodeDesc(sel, inputRR);
         col_list.add(exp);
         if (!StringUtils.isEmpty(alias) &&
             (out_rwsch.get(null, colAlias) != null)) {
@@ -1203,36 +1203,58 @@
    */
   static class UDAFInfo {
     ArrayList<exprNodeDesc> convertedParameters;
-    Method aggregateMethod;
-    Method evaluateMethod;
+    Class<?> retType;
+    Class<? extends UDAFEvaluator> evalClass;
   }
 
   /**
    * Returns the UDAFInfo struct for the aggregation
    * @param aggName  The name of the UDAF.
    * @param mode     The mode of the aggregation. This affects the evaluate method.
-   * @param aggClasses  The classes of the parameters to the UDAF. 
    * @param aggParameters  The actual exprNodeDesc of the parameters.
    * @param aggTree   The ASTNode node of the UDAF in the query.
    * @return UDAFInfo
    * @throws SemanticException when the UDAF is not found or has problems.
    */
-  UDAFInfo getUDAFInfo(String aggName, groupByDesc.Mode mode, ArrayList<Class<?>> aggClasses,
+  UDAFInfo getUDAFInfo(String aggName, groupByDesc.Mode mode,
       ArrayList<exprNodeDesc> aggParameters, ASTNode aggTree) throws SemanticException {
     UDAFInfo r = new UDAFInfo();
-    r.aggregateMethod = FunctionRegistry.getUDAFMethod(aggName, aggClasses);
-    if (null == r.aggregateMethod) {
-      String reason = "Looking for UDAF \"" + aggName + "\" with parameters " + aggClasses;
+    ArrayList<Class<?>> aggClasses = new ArrayList<Class<?>>();
+    for(exprNodeDesc expr: aggParameters) {
+      aggClasses.add(expr.getTypeInfo().getPrimitiveClass());
+    }
+    r.evalClass = FunctionRegistry.getUDAFEvaluator(aggName, aggClasses);
+    if (null == r.evalClass) {
+      String reason = "Looking for UDAF Evaluator\"" + aggName + "\" with parameters " + aggClasses;
+      throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)aggTree.getChild(0), reason));
+    }
+    
+    Method aggregateMethod = null;
+    for(Method m: r.evalClass.getMethods()) {
+      if (m.getName().equalsIgnoreCase("iterate")) {
+        aggregateMethod = m;
+      }
+    }
+    
+    if (null == aggregateMethod) {
+      String reason = "Looking for UDAF Evaluator Iterator\"" + aggName + "\" with parameters " + aggClasses;
       throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)aggTree.getChild(0), reason));
     }
 
-    r.convertedParameters = convertParameters(r.aggregateMethod, aggParameters);
+    r.convertedParameters = convertParameters(aggregateMethod, aggParameters);
     
-    r.evaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
-    if (r.evaluateMethod == null) {
-      String reason = "UDAF \"" + aggName + "\" does not have evaluate()/evaluatePartial() methods.";
+    Method evaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
+    String funcName = (mode == groupByDesc.Mode.COMPLETE || mode == groupByDesc.Mode.FINAL) ? "terminate" : "terminatePartial";
+    for(Method m: r.evalClass.getMethods()) {
+      if (m.getName().equalsIgnoreCase(funcName)) {
+        evaluateMethod = m;
+      }
+    }
+    if (evaluateMethod == null) {
+      String reason = "UDAF \"" + aggName + "\" does not have terminate()/terminatePartial() methods.";
       throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)aggTree.getChild(0), reason)); 
     }
+    r.retType = evaluateMethod.getReturnType();
     
     return r;
   }
@@ -1279,7 +1301,6 @@
       Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
       assert (aggClass != null);
       ArrayList<exprNodeDesc> aggParameters = new ArrayList<exprNodeDesc>();
-      ArrayList<Class<?>> aggClasses = new ArrayList<Class<?>>();
       // 0 is the function name
       for (int i = 1; i < value.getChildCount(); i++) {
         String text = value.getChild(i).toStringTree();
@@ -1292,16 +1313,15 @@
         String paraExpression = paraExprInfo.getInternalName();
         assert(paraExpression != null);
         aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName()));
-        aggClasses.add(paraExprInfo.getType().getPrimitiveClass());
       }
 
-      UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value);
+      UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);
       
-      aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters,
+      aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters,
           value.getToken().getType() == HiveParser.TOK_FUNCTIONDI));
       groupByOutputRowResolver.put("",value.toStringTree(),
                                    new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(),
-                                       udaf.evaluateMethod.getReturnType()));
+                                       udaf.retType));
     }
 
     return 
@@ -1352,12 +1372,9 @@
       String aggName = value.getChild(0).getText();
       Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
       assert (aggClass != null);
-      Method aggEvaluateMethod = null;
-      ArrayList<exprNodeDesc> aggParameters = null;
+      ArrayList<exprNodeDesc> aggParameters = new ArrayList<exprNodeDesc>();
 
       if (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI) {
-        ArrayList<Class<?>> aggClasses = new ArrayList<Class<?>>();
-        ArrayList<exprNodeDesc> params = new ArrayList<exprNodeDesc>();
         // 0 is the function name
         for (int i = 1; i < value.getChildCount(); i++) {
           String text = value.getChild(i).toStringTree();
@@ -1369,17 +1386,11 @@
 
           String paraExpression = paraExprInfo.getInternalName();
           assert(paraExpression != null);
-          params.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName()));
-          aggClasses.add(paraExprInfo.getType().getPrimitiveClass());
+          aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName()));
         }
         
-        UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, params, value);
-        aggParameters = udaf.convertedParameters;
-        aggEvaluateMethod = udaf.evaluateMethod;
       }
       else {
-        aggParameters = new ArrayList<exprNodeDesc>();
-        aggEvaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
         String text = entry.getKey();
         ColumnInfo paraExprInfo = groupByInputRowResolver.get("",text);
         if (paraExprInfo == null) {
@@ -1390,10 +1401,12 @@
         aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExpression));
       }
 
-      aggregations.add(new aggregationDesc(aggClass, aggParameters, ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
+      UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);
+      aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters, 
+          ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
       groupByOutputRowResolver.put("", value.toStringTree(),
                                     new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(),
-                                        aggEvaluateMethod.getReturnType()));
+                                        udaf.retType));
     }
 
     return putOpInsertMap(
@@ -1423,7 +1436,7 @@
     List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest);
     for (int i = 0; i < grpByExprs.size(); ++i) {
       ASTNode grpbyExpr = grpByExprs.get(i);
-      exprNodeDesc grpByExprNode = genExprNodeDesc(qb.getMetaData(), grpbyExpr, groupByInputRowResolver);
+      exprNodeDesc grpByExprNode = genExprNodeDesc(grpbyExpr, groupByInputRowResolver);
 
       groupByKeys.add(grpByExprNode);
       String field = (Integer.valueOf(i)).toString();
@@ -1440,7 +1453,7 @@
         ASTNode parameter = (ASTNode) value.getChild(i);
         String text = parameter.toStringTree();
         if (groupByOutputRowResolver.get("",text) == null) {
-          exprNodeDesc distExprNode = genExprNodeDesc(qb.getMetaData(), parameter, groupByInputRowResolver);
+          exprNodeDesc distExprNode = genExprNodeDesc(parameter, groupByInputRowResolver);
           groupByKeys.add(distExprNode);
           numDistn++;
           String field = (Integer.valueOf(grpByExprs.size() + numDistn -1)).toString();
@@ -1464,19 +1477,18 @@
       // 0 is the function name
       for (int i = 1; i < value.getChildCount(); i++) {
         ASTNode paraExpr = (ASTNode)value.getChild(i);
-        exprNodeDesc paraExprNode = genExprNodeDesc(qb.getMetaData(), paraExpr, groupByInputRowResolver);
+        exprNodeDesc paraExprNode = genExprNodeDesc(paraExpr, groupByInputRowResolver);
 
         aggParameters.add(paraExprNode);
-        aggClasses.add(paraExprNode.getTypeInfo().getPrimitiveClass());
       }
 
-      UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value);
+      UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);
       
-      aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters,
+      aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters,
                                            value.getToken().getType() == HiveParser.TOK_FUNCTIONDI));
       groupByOutputRowResolver.put("",value.toStringTree(),
                                    new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(),
-                                       udaf.evaluateMethod.getReturnType()));
+                                       udaf.retType));
     }
 
     return putOpInsertMap(
@@ -1506,7 +1518,7 @@
         Class<?> from = desc.getTypeInfo().getPrimitiveClass();
         Class<?> to = pType;
         assert(FunctionRegistry.implicitConvertable(from, to));
-        Method conv = FunctionRegistry.getUDFMethod(to.getName(), true, from);
+        Method conv = FunctionRegistry.getUDFMethod(to.getName(), from);
         assert(conv != null);
         Class<? extends UDF> c = FunctionRegistry.getUDFClass(to.getName());
         assert(c != null);
@@ -1635,7 +1647,7 @@
     List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest);
     for (int i = 0; i < grpByExprs.size(); ++i) {
       ASTNode grpbyExpr = grpByExprs.get(i);
-      reduceKeys.add(genExprNodeDesc(qb.getMetaData(), grpbyExpr, reduceSinkInputRowResolver));
+      reduceKeys.add(genExprNodeDesc(grpbyExpr, reduceSinkInputRowResolver));
       String text = grpbyExpr.toStringTree();
       if (reduceSinkOutputRowResolver.get("", text) == null) {
         reduceSinkOutputRowResolver.put("", text,
@@ -1654,7 +1666,7 @@
         ASTNode parameter = (ASTNode) value.getChild(i);
         String text = parameter.toStringTree();
         if (reduceSinkOutputRowResolver.get("",text) == null) {
-          reduceKeys.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver));
+          reduceKeys.add(genExprNodeDesc(parameter, reduceSinkInputRowResolver));
           reduceSinkOutputRowResolver.put("", text,
                                           new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(),
                                               reduceKeys.get(reduceKeys.size()-1).getTypeInfo()));
@@ -1673,7 +1685,7 @@
         ASTNode parameter = (ASTNode) value.getChild(i);
         String text = parameter.toStringTree();
         if (reduceSinkOutputRowResolver.get("",text) == null) {
-          reduceValues.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver));
+          reduceValues.add(genExprNodeDesc(parameter, reduceSinkInputRowResolver));
           reduceSinkOutputRowResolver.put("", text,
                                           new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
                                               reduceValues.get(reduceValues.size()-1).getTypeInfo()));
@@ -1783,12 +1795,9 @@
     HashMap<String, ASTNode> aggregationTrees = parseInfo
         .getAggregationExprsForClause(dest);
     for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
-      ASTNode value = entry.getValue();
-      String aggName = value.getChild(0).getText();
-      Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
-      Method aggEvaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
-      assert (aggClass != null);
       ArrayList<exprNodeDesc> aggParameters = new ArrayList<exprNodeDesc>();
+      ArrayList<Class<?>> aggParamTypes = new ArrayList<Class<?>>();
+      ASTNode value = entry.getValue();
       String text = entry.getKey();
       ColumnInfo paraExprInfo = groupByInputRowResolver2.get("",text);
       if (paraExprInfo == null) {
@@ -1797,10 +1806,18 @@
       String paraExpression = paraExprInfo.getInternalName();
       assert(paraExpression != null);
       aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExpression));
-      aggregations.add(new aggregationDesc(aggClass, aggParameters, ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
+      aggParamTypes.add(paraExprInfo.getType().getPrimitiveClass());
+
+      String aggName = value.getChild(0).getText();
+      Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
+      assert (aggClass != null);
+
+      UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);      
+      aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters, 
+                                           ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
       groupByOutputRowResolver2.put("", value.toStringTree(),
                                     new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(),
-                                        aggEvaluateMethod.getReturnType()));
+                                        udaf.retType));
     }
 
     return putOpInsertMap(
@@ -2099,7 +2116,7 @@
             // cannot convert to complex types
             column = null; 
           } else {
-            column = getFuncExprNodeDesc(tableFieldTypeInfo.getPrimitiveClass().getName(), column);
+            column = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(tableFieldTypeInfo.getPrimitiveClass().getName(), column);
           }
           if (column == null) {
             String reason = "Cannot convert column " + i + " from " + rowFieldTypeInfo + " to " 
@@ -2183,7 +2200,7 @@
       int ccount = partitionExprs.getChildCount();
       for(int i=0; i<ccount; ++i) {
         ASTNode cl = (ASTNode)partitionExprs.getChild(i);
-        partitionCols.add(genExprNodeDescFromColRef(cl, inputRR));
+        partitionCols.add(genExprNodeDesc(cl, inputRR));
       }
     }
 
@@ -2212,7 +2229,7 @@
           order.append("+");
         }
 
-        sortCols.add(genExprNodeDescFromColRef(cl, inputRR));
+        sortCols.add(genExprNodeDesc(cl, inputRR));
       }
     }
 
@@ -2312,7 +2329,7 @@
     Vector<ASTNode> exprs = joinTree.getExpressions().get(pos);
     for (int i = 0; i < exprs.size(); i++) {
       ASTNode expr = exprs.get(i);
-      reduceKeys.add(genExprNodeDesc(qb.getMetaData(), expr, inputRS));
+      reduceKeys.add(genExprNodeDesc(expr, inputRS));
     }
 
     // Walk over the input row resolver and copy in the output
@@ -2403,7 +2420,7 @@
       // Add implicit type conversion if necessary
       for(int i=0; i<right.length; i++) {
         if (!commonClass.isAssignableFrom(keys.get(i).get(k).getTypeInfo().getPrimitiveClass())) {
-          keys.get(i).set(k, getFuncExprNodeDesc(commonClass.getName(), keys.get(i).get(k)));
+          keys.get(i).set(k, TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(commonClass.getName(), keys.get(i).get(k)));
         }
       }
     }
@@ -2842,21 +2859,21 @@
     }
     else {
       for(ASTNode expr: ts.getExprs()) {
-    	  args.add(genExprNodeDesc(qbm, expr, rwsch));
+    	  args.add(genExprNodeDesc(expr, rwsch));
       }
     }
 
-    exprNodeDesc hashfnExpr = getFuncExprNodeDesc("default_sample_hashfn", args);
+    exprNodeDesc hashfnExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("default_sample_hashfn", args);
     assert(hashfnExpr != null);
     LOG.info("hashfnExpr = " + hashfnExpr);
-    exprNodeDesc andExpr = getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr);
+    exprNodeDesc andExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr);
     assert(andExpr != null);
     LOG.info("andExpr = " + andExpr);
-    exprNodeDesc modExpr = getFuncExprNodeDesc("%", andExpr, denominatorExpr);
+    exprNodeDesc modExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("%", andExpr, denominatorExpr);
     assert(modExpr != null);
     LOG.info("modExpr = " + modExpr);
     LOG.info("numeratorExpr = " + numeratorExpr);
-    exprNodeDesc equalsExpr = getFuncExprNodeDesc("==", modExpr, numeratorExpr);
+    exprNodeDesc equalsExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("==", modExpr, numeratorExpr);
     LOG.info("equalsExpr = " + equalsExpr);
     assert(equalsExpr != null);
     return equalsExpr;
@@ -3159,7 +3176,7 @@
     GraphWalker ogw = new GenMapRedWalker(disp);
     ArrayList<Node> topNodes = new ArrayList<Node>();
     topNodes.addAll(this.topOps.values());
-    ogw.startWalking(topNodes);
+    ogw.startWalking(topNodes, null);
 
     // reduce sink does not have any kids
     breakOperatorTree(procCtx.getRootOps());
@@ -3240,379 +3257,65 @@
 
     return;
   }
-  
-  /**
-   * Get the exprNodeDesc
-   * @param name
-   * @param children
-   * @return
-   */
-  public static exprNodeDesc getFuncExprNodeDesc(String name, exprNodeDesc... children) {
-    return getFuncExprNodeDesc(name, Arrays.asList(children));
-  }
-  
-  /**
-   * This function create an ExprNodeDesc for a UDF function given the children (arguments).
-   * It will insert implicit type conversion functions if necessary. 
-   * @throws SemanticException 
-   */
-  public static exprNodeDesc getFuncExprNodeDesc(String udfName, List<exprNodeDesc> children) {
-    // Find the corresponding method
-    ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
-    for(int i=0; i<children.size(); i++) {
-      exprNodeDesc child = children.get(i);
-      assert(child != null);
-      TypeInfo childTypeInfo = child.getTypeInfo();
-      assert(childTypeInfo != null);
-      
-      // Note: we don't pass the element types of MAP/LIST to UDF.
-      // That will work for null test and size but not other more complex functionalities like list slice etc.
-      // For those more complex functionalities, we plan to have a ComplexUDF interface which has an evaluate
-      // method that accepts a list of objects and a list of objectinspectors. 
-      switch (childTypeInfo.getCategory()) {
-        case PRIMITIVE: {
-          argumentClasses.add(childTypeInfo.getPrimitiveClass());
-          break;
-        }
-        case MAP: {
-          argumentClasses.add(Map.class);
-          break;
-        }
-        case LIST: {
-          argumentClasses.add(List.class);
-          break;
-        }
-        case STRUCT: {
-          argumentClasses.add(Object.class);
-          break;
-        }
-        default: {
-          // should never happen
-          assert(false);
-        }
-      }
-    }
-    Method udfMethod = FunctionRegistry.getUDFMethod(udfName, false, argumentClasses);
-    if (udfMethod == null) return null;
-
-    ArrayList<exprNodeDesc> ch = new ArrayList<exprNodeDesc>();
-    Class<?>[] pTypes = udfMethod.getParameterTypes();
-
-    for (int i = 0; i < children.size(); i++)
-    {
-      exprNodeDesc desc = children.get(i);
-      Class<?> pType = ObjectInspectorUtils.generalizePrimitive(pTypes[i]);
-      if (desc instanceof exprNodeNullDesc) {
-        exprNodeConstantDesc newCh = new exprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), null);
-        ch.add(newCh);
-      } else if (pType.isAssignableFrom(argumentClasses.get(i))) {
-        // no type conversion needed
-        ch.add(desc);
-      } else {
-        // must be implicit type conversion
-        Class<?> from = argumentClasses.get(i);
-        Class<?> to = pType;
-        assert(FunctionRegistry.implicitConvertable(from, to));
-        Method m = FunctionRegistry.getUDFMethod(to.getName(), true, from);
-        assert(m != null);
-        Class<? extends UDF> c = FunctionRegistry.getUDFClass(to.getName());
-        assert(c != null);
-
-        // get the conversion method
-        ArrayList<exprNodeDesc> conversionArg = new ArrayList<exprNodeDesc>(1);
-        conversionArg.add(desc);
-        ch.add(new exprNodeFuncDesc(
-              TypeInfoFactory.getPrimitiveTypeInfo(pType),
-              c, m, conversionArg));
-      }
-    }
-
-    exprNodeFuncDesc desc = new exprNodeFuncDesc(
-      TypeInfoFactory.getPrimitiveTypeInfo(udfMethod.getReturnType()),
-      FunctionRegistry.getUDFClass(udfName),
-      udfMethod, ch);
-    return desc;
-  }
-
 
   /**
    * Generates and expression node descriptor for the expression passed in the arguments. This
    * function uses the row resolver and the metadata informatinon that are passed as arguments
    * to resolve the column names to internal names.
-   * @param qbm The metadata infromation for the query block
    * @param expr The expression
    * @param input The row resolver
    * @return exprNodeDesc
    * @throws SemanticException
    */
   @SuppressWarnings("nls")
-  private exprNodeDesc genExprNodeDesc(QBMetaData qbm, ASTNode expr, RowResolver input)
+  private exprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input)
   throws SemanticException {
     //  We recursively create the exprNodeDesc.  Base cases:  when we encounter 
     //  a column ref, we convert that into an exprNodeColumnDesc;  when we encounter 
     //  a constant, we convert that into an exprNodeConstantDesc.  For others we just 
     //  build the exprNodeFuncDesc with recursively built children.
     
-    exprNodeDesc desc = null;
-
     //  If the current subExpression is pre-calculated, as in Group-By etc.
     ColumnInfo colInfo = input.get("", expr.toStringTree());
     if (colInfo != null) {
-      desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()); 
-      return desc;
+      return new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
     }
 
-    //  Is this a simple expr node (not a TOK_COLREF or a TOK_FUNCTION or an operator)?
-    desc = genSimpleExprNodeDesc(expr);
-    if (desc != null) {
-      return desc;
-    }
+    // Create the walker, the rules dispatcher and the context.
+    TypeCheckCtx tcCtx = new TypeCheckCtx(input);
     
-    int tokType = expr.getType();
-    switch (tokType) {
-      case HiveParser.TOK_COLREF: {
-        desc = genExprNodeDescFromColRef(expr, input);
-        break;
-      }
-  
-      default: {
-        boolean isFunction = (expr.getType() == HiveParser.TOK_FUNCTION);
-        
-        // Create all children
-        int childrenBegin = (isFunction ? 1 : 0);
-        ArrayList<exprNodeDesc> children = new ArrayList<exprNodeDesc>(expr.getChildCount() - childrenBegin);
-        for (int ci=childrenBegin; ci<expr.getChildCount(); ci++) {
-          children.add(genExprNodeDesc(qbm, (ASTNode)expr.getChild(ci), input));
-        }
-        
-        // Create function desc
-        desc = getXpathOrFuncExprNodeDesc(expr, isFunction, children);
-        break;
-      }
-    }
-    assert(desc != null);
-    return desc;
-  }
-
-  /**
-   * Generates expression node from a TOK_COLREF AST Node
-   * @param expr Antrl node
-   * @param input row resolver for this col reference
-   * @return exprNodeDesc or null if ASTNode is not a TOK_COLREF
-   * @throws SemanticException
-   */
-  private exprNodeDesc genExprNodeDescFromColRef(ASTNode expr, RowResolver input)
-      throws SemanticException {
-    if(expr.getType() != HiveParser.TOK_COLREF) {
-      throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(expr));
-    }
-    exprNodeDesc desc;
-    ColumnInfo colInfo;
-    String tabAlias = null;
-    String colName = null;
-    if (expr.getChildCount() != 1) {
-      tabAlias = unescapeIdentifier(expr.getChild(0).getText());
-      colName = unescapeIdentifier(expr.getChild(1).getText());
-    }
-    else {
-      colName = unescapeIdentifier(expr.getChild(0).getText());
-    }
-
-    if (colName == null) {
-      throw new SemanticException(ErrorMsg.INVALID_XPATH.getMsg(expr));
-    }
-
-    colInfo = input.get(tabAlias, colName);
+    // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher
+    // generates the plan from the operator tree
+    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+    StringBuilder sb = new StringBuilder();
+    Formatter fm = new Formatter(sb);
+    opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"), TypeCheckProcFactory.getNullExprProcessor());
+    opRules.put(new RuleRegExp("R2", HiveParser.Number + "%"), TypeCheckProcFactory.getNumExprProcessor());
+    opRules.put(new RuleRegExp("R3", HiveParser.Identifier + "%|" + 
+                                     HiveParser.StringLiteral + "%|" + 
+                                     HiveParser.TOK_CHARSETLITERAL + "%"), 
+                               TypeCheckProcFactory.getStrExprProcessor());
+    opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|" + HiveParser.KW_FALSE + "%"), 
+                               TypeCheckProcFactory.getBoolExprProcessor());
+    opRules.put(new RuleRegExp("R4", HiveParser.TOK_COLREF + "%"), TypeCheckProcFactory.getColumnExprProcessor());
 
-    if (colInfo == null && input.getIsExprResolver()) {
-      throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(expr));
-    }         
-    else if (tabAlias != null && !input.hasTableAlias(tabAlias)) {
-      throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr.getChild(0)));
-    } else if (colInfo == null) {
-      throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(tabAlias == null? expr.getChild(0) : expr.getChild(1)));
+    // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
+    Dispatcher disp = new DefaultRuleDispatcher(TypeCheckProcFactory.getDefaultExprProcessor(), opRules, tcCtx);
+    GraphWalker ogw = new DefaultGraphWalker(disp);
+   
+    // Create a list of topop nodes
+    ArrayList<Node> topNodes = new ArrayList<Node>();
+    topNodes.add(expr);
+    HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
+    ogw.startWalking(topNodes, nodeOutputs);
+    exprNodeDesc desc = (exprNodeDesc)nodeOutputs.get(expr);
+    if (desc == null) {
+      throw new SemanticException(tcCtx.getError());
     }
 
-    desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
     return desc;
   }
-
-  static HashMap<Integer, String> specialUnaryOperatorTextHashMap;
-  static HashMap<Integer, String> specialFunctionTextHashMap;
-  static HashMap<Integer, String> conversionFunctionTextHashMap;
-  static {
-    specialUnaryOperatorTextHashMap = new HashMap<Integer, String>();
-    specialUnaryOperatorTextHashMap.put(HiveParser.PLUS, "positive");
-    specialUnaryOperatorTextHashMap.put(HiveParser.MINUS, "negative");
-    specialFunctionTextHashMap = new HashMap<Integer, String>();
-    specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull");
-    specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull");
-    conversionFunctionTextHashMap = new HashMap<Integer, String>();
-    conversionFunctionTextHashMap.put(HiveParser.TOK_BOOLEAN, Boolean.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_TINYINT, Byte.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_SMALLINT, Short.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_INT, Integer.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_BIGINT, Long.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_FLOAT, Float.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_DOUBLE, Double.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_STRING, String.class.getName());
-    conversionFunctionTextHashMap.put(HiveParser.TOK_DATE, java.sql.Date.class.getName());
-  }
-  
-  public static boolean isRedundantConversionFunction(ASTNode expr, boolean isFunction, ArrayList<exprNodeDesc> children) {
-    if (!isFunction) return false;
-    // children is always one less than the expr.getChildCount(), since the latter contains function name.
-    assert(children.size() == expr.getChildCount() - 1);
-    // conversion functions take a single parameter
-    if (children.size() != 1) return false;
-    String funcText = conversionFunctionTextHashMap.get(((ASTNode)expr.getChild(0)).getType());
-    // not a conversion function 
-    if (funcText == null) return false;
-    // return true when the child type and the conversion target type is the same
-    return children.get(0).getTypeInfo().getPrimitiveClass().getName().equals(funcText);
-  }
-  
-  public static String getFunctionText(ASTNode expr, boolean isFunction) {
-    String funcText = null;
-    if (!isFunction) {
-      // For operator, the function name is the operator text, unless it's in our special dictionary
-      if (expr.getChildCount() == 1) {
-        funcText = specialUnaryOperatorTextHashMap.get(expr.getType());
-      }
-      if (funcText == null) {
-        funcText = expr.getText();
-      }
-    } else {
-      // For TOK_FUNCTION, the function name is stored in the first child, unless it's in our
-      // special dictionary.
-      assert(expr.getChildCount() >= 1);
-      int funcType = ((ASTNode)expr.getChild(0)).getType();
-      funcText = specialFunctionTextHashMap.get(funcType);
-      if (funcText == null) {
-        funcText = conversionFunctionTextHashMap.get(funcType);
-      }
-      if (funcText == null) {
-        funcText = ((ASTNode)expr.getChild(0)).getText();
-      }
-    }
-    return funcText;
-  }
-  
-  static exprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, boolean isFunction,
-      ArrayList<exprNodeDesc> children)
-      throws SemanticException {
-    // return the child directly if the conversion is redundant.
-    if (isRedundantConversionFunction(expr, isFunction, children)) {
-      assert(children.size() == 1);
-      assert(children.get(0) != null);
-      return children.get(0);
-    }
-    String funcText = getFunctionText(expr, isFunction);
-    exprNodeDesc desc;
-    if (funcText.equals(".")) {
-      // "." :  FIELD Expression
-      assert(children.size() == 2);
-      // Only allow constant field name for now
-      assert(children.get(1) instanceof exprNodeConstantDesc);
-      exprNodeDesc object = children.get(0);
-      exprNodeConstantDesc fieldName = (exprNodeConstantDesc)children.get(1);
-      assert(fieldName.getValue() instanceof String);
-      
-      // Calculate result TypeInfo
-      String fieldNameString = (String)fieldName.getValue();
-      TypeInfo objectTypeInfo = object.getTypeInfo();
-      
-      // Allow accessing a field of list element structs directly from a list  
-      boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST);
-      if (isList) {
-        objectTypeInfo = objectTypeInfo.getListElementTypeInfo();
-      }
-      if (objectTypeInfo.getCategory() != Category.STRUCT) {
-        throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr));
-      }
-      TypeInfo t = objectTypeInfo.getStructFieldTypeInfo(fieldNameString);
-      if (isList) {
-        t = TypeInfoFactory.getListTypeInfo(t);
-      }
-      
-      desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString, isList);
-      
-    } else if (funcText.equals("[")){
-      // "[]" : LSQUARE/INDEX Expression
-      assert(children.size() == 2);
-      
-      // Check whether this is a list or a map
-      TypeInfo myt = children.get(0).getTypeInfo();
-
-      if (myt.getCategory() == Category.LIST) {
-        // Only allow constant integer index for now
-        if (!(children.get(1) instanceof exprNodeConstantDesc)
-            || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) {
-          throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr));
-        }
-      
-        // Calculate TypeInfo
-        TypeInfo t = myt.getListElementTypeInfo();
-        desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
-      }
-      else if (myt.getCategory() == Category.MAP) {
-        // Only allow only constant indexes for now
-        if (!(children.get(1) instanceof exprNodeConstantDesc)) {
-          throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_CONSTANT.getMsg(expr));
-        }
-        if (!(((exprNodeConstantDesc)children.get(1)).getValue().getClass() == 
-              myt.getMapKeyTypeInfo().getPrimitiveClass())) {
-          throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE.getMsg(expr));
-        }
-        // Calculate TypeInfo
-        TypeInfo t = myt.getMapValueTypeInfo();
-        
-        desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
-      }
-      else {
-        throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr, 
-            myt.getTypeName()));
-      }
-    } else {
-      // other operators or functions
-      Class<? extends UDF> udf = FunctionRegistry.getUDFClass(funcText);
-      if (udf == null) {
-      	if (isFunction)
-          throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr.getChild(0)));
-      	else
-          throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr));
-      }
-      
-      desc = getFuncExprNodeDesc(funcText, children);
-      if (desc == null) {
-        ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
-        for(int i=0; i<children.size(); i++) {
-          argumentClasses.add(children.get(i).getTypeInfo().getPrimitiveClass());
-        }
   
-        if (isFunction) {
-          String reason = "Looking for UDF \"" + expr.getChild(0).getText() + "\" with parameters " + argumentClasses;
-          throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)expr.getChild(0), reason));
-        } else {
-          String reason = "Looking for Operator \"" + expr.getText() + "\" with parameters " + argumentClasses;
-          throw new SemanticException(ErrorMsg.INVALID_OPERATOR_SIGNATURE.getMsg(expr, reason));
-        }
-      }
-    }
-    // UDFOPPositive is a no-op.
-    // However, we still create it, and then remove it here, to make sure we only allow
-    // "+" for numeric types.
-    if (desc instanceof exprNodeFuncDesc) {
-      exprNodeFuncDesc funcDesc = (exprNodeFuncDesc)desc;
-      if (funcDesc.getUDFClass().equals(UDFOPPositive.class)) {
-        assert(funcDesc.getChildren().size() == 1);
-        desc = funcDesc.getChildren().get(0);
-      }
-    }
-    assert(desc != null);
-    return desc;
-  }
-
   static exprNodeDesc genSimpleExprNodeDesc(ASTNode expr) throws SemanticException {
     exprNodeDesc desc = null;
     switch(expr.getType()) {

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java?rev=737291&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java Sat Jan 24 01:58:01 2009
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+
+/**
+ * This class implements the context information that is used for typechecking phase
+ * in query compilation.
+ */
+public class TypeCheckCtx implements NodeProcessorCtx {
+  
+  /**
+   * The row resolver of the previous operator. This field is used to generate expression
+   * descriptors from the expression ASTs.
+   */
+  private RowResolver inputRR;
+
+  /**
+   * Potential typecheck error reason.
+   */
+  private String error;
+  
+  /**
+   * Constructor.
+   * 
+   * @param inputRR The input row resolver of the previous operator.
+   */
+  public TypeCheckCtx(RowResolver inputRR) {
+    this.setInputRR(inputRR);
+    this.error = null;
+  }
+
+  /**
+   * @param inputRR the inputRR to set
+   */
+  public void setInputRR(RowResolver inputRR) {
+    this.inputRR = inputRR;
+  }
+
+  /**
+   * @return the inputRR
+   */
+  public RowResolver getInputRR() {
+    return inputRR;
+  }
+
+  /**
+   * @param error the error to set
+   */
+  public void setError(String error) {
+    this.error = error;
+  }
+
+  /**
+   * @return the error
+   */
+  public String getError() {
+    return error;
+  }
+
+}

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=737291&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Sat Jan 24 01:58:01 2009
@@ -0,0 +1,628 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
+import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.ql.udf.UDFOPPositive;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+/**
+ * The Factory for creating typecheck processors. The typecheck processors are used to
+ * processes the syntax trees for expressions and convert them into expression Node
+ * Descriptor trees. They also introduce the correct conversion functions to do proper
+ * implicit conversion.
+ */
+public class TypeCheckProcFactory {
+
+  /**
+   * Function to do groupby subexpression elimination. This is called by all the processors initially.
+   * As an example, consider the query
+   *   select a+b, count(1) from T group by a+b;
+   * Then a+b is already precomputed in the group by operators key, so we substitute a+b in the select
+   * list with the internal column name of the a+b expression that appears in the in input row resolver.
+   * 
+   * @param nd The node that is being inspected.
+   * @param procCtx The processor context.
+   * 
+   * @return exprNodeColumnDesc.
+   */
+  public static exprNodeDesc processGByExpr(Node nd, Object procCtx) 
+    throws SemanticException {
+    //  We recursively create the exprNodeDesc.  Base cases:  when we encounter 
+    //  a column ref, we convert that into an exprNodeColumnDesc;  when we encounter 
+    //  a constant, we convert that into an exprNodeConstantDesc.  For others we just 
+    //  build the exprNodeFuncDesc with recursively built children.
+    ASTNode expr = (ASTNode)nd;
+    TypeCheckCtx ctx = (TypeCheckCtx) procCtx;
+    RowResolver input = ctx.getInputRR();
+    exprNodeDesc desc = null;
+
+    //  If the current subExpression is pre-calculated, as in Group-By etc.
+    ColumnInfo colInfo = input.get("", expr.toStringTree());
+    if (colInfo != null) {
+      desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()); 
+      return desc;
+    }    
+    return desc;
+  }
+  
+  /**
+   * Processor for processing NULL expression.
+   */
+  public static class NullExprProcessor implements NodeProcessor {
+
+    @Override
+    public Object process(Node nd, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+
+      exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+      if (desc != null) {
+        return desc;
+      }
+      
+      return new exprNodeNullDesc();
+    }
+    
+  }
+  
+  /**
+   * Factory method to get NullExprProcessor.
+   * @return NullExprProcessor.
+   */
+  public static NullExprProcessor getNullExprProcessor() {
+    return new NullExprProcessor();
+  }
+  
+  /**
+   * Processor for processing numeric constants.
+   */
+  public static class NumExprProcessor implements NodeProcessor {
+
+    @Override
+    public Object process(Node nd, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+
+      exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+      if (desc != null) {
+        return desc;
+      }
+      
+      Number v = null;
+      ASTNode expr = (ASTNode)nd;
+      // The expression can be any one of Double, Long and Integer. We
+      // try to parse the expression in that order to ensure that the
+      // most specific type is used for conversion.
+      try {
+        v = Double.valueOf(expr.getText());
+        v = Long.valueOf(expr.getText());
+        v = Integer.valueOf(expr.getText());
+      } catch (NumberFormatException e) {
+        // do nothing here, we will throw an exception in the following block
+      }
+      if (v == null) {
+        throw new SemanticException(ErrorMsg.INVALID_NUMERICAL_CONSTANT.getMsg(expr));
+      }
+      return new exprNodeConstantDesc(v);
+    }
+    
+  }
+  
+  /**
+   * Factory method to get NumExprProcessor.
+   * @return NumExprProcessor.
+   */
+  public static NumExprProcessor getNumExprProcessor() {
+    return new NumExprProcessor();
+  }
+  
+  /**
+   * Processor for processing string constants.
+   */
+  public static class StrExprProcessor implements NodeProcessor {
+
+    @Override
+    public Object process(Node nd, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+
+      exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+      if (desc != null) {
+        return desc;
+      }
+      
+      ASTNode expr = (ASTNode)nd;
+      String str = null;
+      
+      switch (expr.getToken().getType()) {
+      case HiveParser.Identifier:
+        str = BaseSemanticAnalyzer.unescapeIdentifier(expr.getText());
+        break;
+      case HiveParser.StringLiteral:
+        str = BaseSemanticAnalyzer.unescapeSQLString(expr.getText());
+        break;
+      case HiveParser.TOK_CHARSETLITERAL:
+        str = BaseSemanticAnalyzer.charSetString(expr.getChild(0).getText(), expr.getChild(1).getText());
+        break;
+      default:
+        assert false;
+      }
+      return new exprNodeConstantDesc(String.class, str);
+    }
+    
+  }
+  
+  /**
+   * Factory method to get StrExprProcessor.
+   * @return StrExprProcessor.
+   */
+  public static StrExprProcessor getStrExprProcessor() {
+    return new StrExprProcessor();
+  }
+  
+  /**
+   * Processor for boolean constants.
+   */
+  public static class BoolExprProcessor implements NodeProcessor {
+
+    @Override
+    public Object process(Node nd, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+
+      exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+      if (desc != null) {
+        return desc;
+      }
+
+      ASTNode expr = (ASTNode)nd;
+      Boolean bool = null;
+
+      switch (expr.getToken().getType()) {
+      case HiveParser.KW_TRUE:
+        bool = Boolean.TRUE;
+        break;
+      case HiveParser.KW_FALSE:
+        bool = Boolean.FALSE;
+        break;
+      default:
+        assert false;
+      }
+      return new exprNodeConstantDesc(Boolean.class, bool);      
+    }
+    
+  }
+  
+  /**
+   * Factory method to get BoolExprProcessor.
+   * @return BoolExprProcessor.
+   */
+  public static BoolExprProcessor getBoolExprProcessor() {
+    return new BoolExprProcessor();
+  }
+  
+  /**
+   * Processor for table columns
+   */
+  public static class ColumnExprProcessor implements NodeProcessor {
+
+    @Override
+    public Object process(Node nd, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+
+      exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+      if (desc != null) {
+        return desc;
+      }
+
+      ASTNode expr = (ASTNode)nd;
+      TypeCheckCtx ctx = (TypeCheckCtx)procCtx;
+      RowResolver input = ctx.getInputRR();
+
+      if(expr.getType() != HiveParser.TOK_COLREF) {
+        ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr));
+        return null;
+      }
+      
+      String tabAlias = null;
+      String colName = null;
+      
+      if (expr.getChildCount() != 1) {
+        tabAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
+        colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(1).getText());
+      }
+      else {
+        colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
+      }
+
+      if (colName == null) {
+        ctx.setError(ErrorMsg.INVALID_XPATH.getMsg(expr));
+        return null;
+      }
+
+      ColumnInfo colInfo = input.get(tabAlias, colName);
+
+      if (colInfo == null && input.getIsExprResolver()) {
+        ctx.setError(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(expr));
+        return null;
+      }         
+      else if (tabAlias != null && !input.hasTableAlias(tabAlias)) {
+        ctx.setError(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr.getChild(0)));
+        return null;
+      } else if (colInfo == null) {
+        ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(tabAlias == null? expr.getChild(0) : expr.getChild(1)));
+        return null;
+      }
+
+      return new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
+    }
+    
+  }
+  
+  /**
+   * Factory method to get ColumnExprProcessor.
+   * @return ColumnExprProcessor.
+   */
+  public static ColumnExprProcessor getColumnExprProcessor() {
+    return new ColumnExprProcessor();
+  }
+  
+  /**
+   * The default processor for typechecking.
+   */
+  public static class DefaultExprProcessor implements NodeProcessor {
+
+    static HashMap<Integer, String> specialUnaryOperatorTextHashMap;
+    static HashMap<Integer, String> specialFunctionTextHashMap;
+    static HashMap<Integer, String> conversionFunctionTextHashMap;
+    static {
+      specialUnaryOperatorTextHashMap = new HashMap<Integer, String>();
+      specialUnaryOperatorTextHashMap.put(HiveParser.PLUS, "positive");
+      specialUnaryOperatorTextHashMap.put(HiveParser.MINUS, "negative");
+      specialFunctionTextHashMap = new HashMap<Integer, String>();
+      specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull");
+      specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull");
+      conversionFunctionTextHashMap = new HashMap<Integer, String>();
+      conversionFunctionTextHashMap.put(HiveParser.TOK_BOOLEAN, Boolean.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_TINYINT, Byte.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_SMALLINT, Short.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_INT, Integer.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_BIGINT, Long.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_FLOAT, Float.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_DOUBLE, Double.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_STRING, String.class.getName());
+      conversionFunctionTextHashMap.put(HiveParser.TOK_DATE, java.sql.Date.class.getName());
+    }
+
+    public static boolean isRedundantConversionFunction(ASTNode expr, boolean isFunction, ArrayList<exprNodeDesc> children) {
+      if (!isFunction) return false;
+      // children is always one less than the expr.getChildCount(), since the latter contains function name.
+      assert(children.size() == expr.getChildCount() - 1);
+      // conversion functions take a single parameter
+      if (children.size() != 1) return false;
+      String funcText = conversionFunctionTextHashMap.get(((ASTNode)expr.getChild(0)).getType());
+      // not a conversion function 
+      if (funcText == null) return false;
+      // return true when the child type and the conversion target type is the same
+      return children.get(0).getTypeInfo().getPrimitiveClass().getName().equals(funcText);
+    }
+    
+    public static String getFunctionText(ASTNode expr, boolean isFunction) {
+      String funcText = null;
+      if (!isFunction) {
+        // For operator, the function name is the operator text, unless it's in our special dictionary
+        if (expr.getChildCount() == 1) {
+          funcText = specialUnaryOperatorTextHashMap.get(expr.getType());
+        }
+        if (funcText == null) {
+          funcText = expr.getText();
+        }
+      } else {
+        // For TOK_FUNCTION, the function name is stored in the first child, unless it's in our
+        // special dictionary.
+        assert(expr.getChildCount() >= 1);
+        int funcType = ((ASTNode)expr.getChild(0)).getType();
+        funcText = specialFunctionTextHashMap.get(funcType);
+        if (funcText == null) {
+          funcText = conversionFunctionTextHashMap.get(funcType);
+        }
+        if (funcText == null) {
+          funcText = ((ASTNode)expr.getChild(0)).getText();
+        }
+      }
+      return funcText;
+    }
+
+
+    /**
+     * Get the exprNodeDesc
+     * @param name
+     * @param children
+     * @return
+     */
+    public static exprNodeDesc getFuncExprNodeDesc(String name, exprNodeDesc... children) {
+      return getFuncExprNodeDesc(name, Arrays.asList(children));
+    }
+    
+    /**
+     * This function create an ExprNodeDesc for a UDF function given the children (arguments).
+     * It will insert implicit type conversion functions if necessary. 
+     * @throws SemanticException 
+     */
+    public static exprNodeDesc getFuncExprNodeDesc(String udfName, List<exprNodeDesc> children) {
+      // Find the corresponding method
+      ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
+      for(int i=0; i<children.size(); i++) {
+        exprNodeDesc child = children.get(i);
+        assert(child != null);
+        TypeInfo childTypeInfo = child.getTypeInfo();
+        assert(childTypeInfo != null);
+        
+        // Note: we don't pass the element types of MAP/LIST to UDF.
+        // That will work for null test and size but not other more complex functionalities like list slice etc.
+        // For those more complex functionalities, we plan to have a ComplexUDF interface which has an evaluate
+        // method that accepts a list of objects and a list of objectinspectors. 
+        switch (childTypeInfo.getCategory()) {
+          case PRIMITIVE: {
+            argumentClasses.add(childTypeInfo.getPrimitiveClass());
+            break;
+          }
+          case MAP: {
+            argumentClasses.add(Map.class);
+            break;
+          }
+          case LIST: {
+            argumentClasses.add(List.class);
+            break;
+          }
+          case STRUCT: {
+            argumentClasses.add(Object.class);
+            break;
+          }
+          default: {
+            // should never happen
+            assert(false);
+          }
+        }
+      }
+      Method udfMethod = FunctionRegistry.getUDFMethod(udfName, argumentClasses);
+      if (udfMethod == null) return null;
+
+      ArrayList<exprNodeDesc> ch = new ArrayList<exprNodeDesc>();
+      Class<?>[] pTypes = udfMethod.getParameterTypes();
+
+      for (int i = 0; i < children.size(); i++)
+      {
+        exprNodeDesc desc = children.get(i);
+        Class<?> pType = ObjectInspectorUtils.generalizePrimitive(pTypes[i]);
+        if (desc instanceof exprNodeNullDesc) {
+          exprNodeConstantDesc newCh = new exprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), null);
+          ch.add(newCh);
+        } else if (pType.isAssignableFrom(argumentClasses.get(i))) {
+          // no type conversion needed
+          ch.add(desc);
+        } else {
+          // must be implicit type conversion
+          Class<?> from = argumentClasses.get(i);
+          Class<?> to = pType;
+          assert(FunctionRegistry.implicitConvertable(from, to));
+          Method m = FunctionRegistry.getUDFMethod(to.getName(), from);
+          assert(m != null);
+          Class<? extends UDF> c = FunctionRegistry.getUDFClass(to.getName());
+          assert(c != null);
+
+          // get the conversion method
+          ArrayList<exprNodeDesc> conversionArg = new ArrayList<exprNodeDesc>(1);
+          conversionArg.add(desc);
+          ch.add(new exprNodeFuncDesc(
+                TypeInfoFactory.getPrimitiveTypeInfo(pType),
+                c, m, conversionArg));
+        }
+      }
+
+      exprNodeFuncDesc desc = new exprNodeFuncDesc(
+        TypeInfoFactory.getPrimitiveTypeInfo(udfMethod.getReturnType()),
+        FunctionRegistry.getUDFClass(udfName),
+        udfMethod, ch);
+      return desc;
+    }
+
+    static exprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, boolean isFunction,
+        ArrayList<exprNodeDesc> children)
+        throws SemanticException {
+      // return the child directly if the conversion is redundant.
+      if (isRedundantConversionFunction(expr, isFunction, children)) {
+        assert(children.size() == 1);
+        assert(children.get(0) != null);
+        return children.get(0);
+      }
+      String funcText = getFunctionText(expr, isFunction);
+      exprNodeDesc desc;
+      if (funcText.equals(".")) {
+        // "." :  FIELD Expression
+        assert(children.size() == 2);
+        // Only allow constant field name for now
+        assert(children.get(1) instanceof exprNodeConstantDesc);
+        exprNodeDesc object = children.get(0);
+        exprNodeConstantDesc fieldName = (exprNodeConstantDesc)children.get(1);
+        assert(fieldName.getValue() instanceof String);
+        
+        // Calculate result TypeInfo
+        String fieldNameString = (String)fieldName.getValue();
+        TypeInfo objectTypeInfo = object.getTypeInfo();
+        
+        // Allow accessing a field of list element structs directly from a list  
+        boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST);
+        if (isList) {
+          objectTypeInfo = objectTypeInfo.getListElementTypeInfo();
+        }
+        if (objectTypeInfo.getCategory() != Category.STRUCT) {
+          throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr));
+        }
+        TypeInfo t = objectTypeInfo.getStructFieldTypeInfo(fieldNameString);
+        if (isList) {
+          t = TypeInfoFactory.getListTypeInfo(t);
+        }
+        
+        desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString, isList);
+        
+      } else if (funcText.equals("[")){
+        // "[]" : LSQUARE/INDEX Expression
+        assert(children.size() == 2);
+        
+        // Check whether this is a list or a map
+        TypeInfo myt = children.get(0).getTypeInfo();
+
+        if (myt.getCategory() == Category.LIST) {
+          // Only allow constant integer index for now
+          if (!(children.get(1) instanceof exprNodeConstantDesc)
+              || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) {
+            throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr));
+          }
+        
+          // Calculate TypeInfo
+          TypeInfo t = myt.getListElementTypeInfo();
+          desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
+        }
+        else if (myt.getCategory() == Category.MAP) {
+          // Only allow only constant indexes for now
+          if (!(children.get(1) instanceof exprNodeConstantDesc)) {
+            throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_CONSTANT.getMsg(expr));
+          }
+          if (!(((exprNodeConstantDesc)children.get(1)).getValue().getClass() == 
+                myt.getMapKeyTypeInfo().getPrimitiveClass())) {
+            throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE.getMsg(expr));
+          }
+          // Calculate TypeInfo
+          TypeInfo t = myt.getMapValueTypeInfo();
+          
+          desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
+        }
+        else {
+          throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr, 
+              myt.getTypeName()));
+        }
+      } else {
+        // other operators or functions
+        Class<? extends UDF> udf = FunctionRegistry.getUDFClass(funcText);
+        if (udf == null) {
+          if (isFunction)
+            throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr.getChild(0)));
+          else
+            throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr));
+        }
+        
+        desc = getFuncExprNodeDesc(funcText, children);
+        if (desc == null) {
+          ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
+          for(int i=0; i<children.size(); i++) {
+            argumentClasses.add(children.get(i).getTypeInfo().getPrimitiveClass());
+          }
+    
+          if (isFunction) {
+            String reason = "Looking for UDF \"" + expr.getChild(0).getText() + "\" with parameters " + argumentClasses;
+            throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)expr.getChild(0), reason));
+          } else {
+            String reason = "Looking for Operator \"" + expr.getText() + "\" with parameters " + argumentClasses;
+            throw new SemanticException(ErrorMsg.INVALID_OPERATOR_SIGNATURE.getMsg(expr, reason));
+          }
+        }
+      }
+      // UDFOPPositive is a no-op.
+      // However, we still create it, and then remove it here, to make sure we only allow
+      // "+" for numeric types.
+      if (desc instanceof exprNodeFuncDesc) {
+        exprNodeFuncDesc funcDesc = (exprNodeFuncDesc)desc;
+        if (funcDesc.getUDFClass().equals(UDFOPPositive.class)) {
+          assert(funcDesc.getChildren().size() == 1);
+          desc = funcDesc.getChildren().get(0);
+        }
+      }
+      assert(desc != null);
+      return desc;
+    }
+
+    @Override
+    public Object process(Node nd, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+
+      ASTNode expr = (ASTNode)nd;
+      
+      // Return nulls for conversion operators
+      if (conversionFunctionTextHashMap.keySet().contains(expr.getType()) ||
+          specialFunctionTextHashMap.keySet().contains(expr.getType()) ||
+          expr.getToken().getType() == HiveParser.CharSetName ||
+          expr.getToken().getType() == HiveParser.CharSetLiteral) {
+        return null;
+      }
+      
+      exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+      if (desc != null) {
+        return desc;
+      }
+
+      boolean isFunction = (expr.getType() == HiveParser.TOK_FUNCTION);
+      
+      // Create all children
+      int childrenBegin = (isFunction ? 1 : 0);
+      ArrayList<exprNodeDesc> children = new ArrayList<exprNodeDesc>(expr.getChildCount() - childrenBegin);
+      for (int ci=childrenBegin; ci<expr.getChildCount(); ci++) {
+        children.add((exprNodeDesc)nodeOutputs[ci]);
+      }
+      
+      // If any of the children contains null, then return a null
+      // this is a hack for now to handle the group by case
+      if (children.contains(null)) {
+        return null;
+      }
+      
+      // Create function desc
+      return getXpathOrFuncExprNodeDesc(expr, isFunction, children);
+    }
+    
+  }
+  
+  /**
+   * Factory method to get DefaultExprProcessor.
+   * @return DefaultExprProcessor.
+   */
+  public static DefaultExprProcessor getDefaultExprProcessor() {
+    return new DefaultExprProcessor();
+  }
+}

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Sat Jan 24 01:58:01 2009
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
 import org.apache.hadoop.hive.ql.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.hive.serde.Constants;
 import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
@@ -222,7 +223,7 @@
     } else {
       // numPartitionFields = -1 means random partitioning
       partitionCols = new ArrayList<exprNodeDesc>(1);
-      partitionCols.add(SemanticAnalyzer.getFuncExprNodeDesc("rand"));
+      partitionCols.add(TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("rand"));
     }
     
     StringBuilder order = new StringBuilder();

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java Sat Jan 24 01:58:01 2009
@@ -21,25 +21,26 @@
 import org.apache.hadoop.hive.ql.exec.FunctionInfo;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.UDAF;
+import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
 
 public class aggregationDesc implements java.io.Serializable {
   private static final long serialVersionUID = 1L;
-  private Class<? extends UDAF> aggregationClass;
+  private Class<? extends UDAFEvaluator> aggregationClass;
   private java.util.ArrayList<exprNodeDesc> parameters;
   private boolean distinct;
   public aggregationDesc() {}
   public aggregationDesc(
-    final Class<? extends UDAF> aggregationClass,
+    final Class<? extends UDAFEvaluator> aggregationClass,
     final java.util.ArrayList<exprNodeDesc> parameters,
     final boolean distinct) {
     this.aggregationClass = aggregationClass;
     this.parameters = parameters;
     this.distinct = distinct;
   }
-  public Class<? extends UDAF> getAggregationClass() {
+  public Class<? extends UDAFEvaluator> getAggregationClass() {
     return this.aggregationClass;
   }
-  public void setAggregationClass(final Class<? extends UDAF> aggregationClass) {
+  public void setAggregationClass(final Class<? extends UDAFEvaluator> aggregationClass) {
     this.aggregationClass = aggregationClass;
   }
   public java.util.ArrayList<exprNodeDesc> getParameters() {

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java Sat Jan 24 01:58:01 2009
@@ -24,8 +24,6 @@
 
 import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 
 public class exprNodeColumnDesc extends exprNodeDesc implements Serializable {
   private static final long serialVersionUID = 1L;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java Sat Jan 24 01:58:01 2009
@@ -22,8 +22,6 @@
 import java.util.List;
 
 import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
 
 public class exprNodeDesc implements Serializable {  
   private static final long serialVersionUID = 1L;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java Sat Jan 24 01:58:01 2009
@@ -23,13 +23,11 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.ql.exec.FunctionInfo;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
 
 /**
  * The reason that we have to store UDFClass as well as UDFMethod is because
@@ -39,12 +37,13 @@
 public class exprNodeFuncDesc extends exprNodeDesc implements Serializable {
 
   private static final long serialVersionUID = 1L;
-  private Class UDFClass;
+  private Class<? extends UDF> UDFClass;
   private Method UDFMethod;
   private ArrayList<exprNodeDesc> children; 
   
   public exprNodeFuncDesc() {}
-  public exprNodeFuncDesc(TypeInfo typeInfo, Class UDFClass, Method UDFMethod, ArrayList<exprNodeDesc> children) {
+  public exprNodeFuncDesc(TypeInfo typeInfo, Class<? extends UDF> UDFClass, 
+                          Method UDFMethod, ArrayList<exprNodeDesc> children) {
     super(typeInfo);
     assert(UDFClass != null);
     this.UDFClass = UDFClass;
@@ -53,10 +52,11 @@
     this.children = children;
   }
   
-  public Class getUDFClass() {
+  public Class<? extends UDF> getUDFClass() {
     return UDFClass;
   }
-  public void setUDFClass(Class UDFClass) {
+  
+  public void setUDFClass(Class<? extends UDF> UDFClass) {
     this.UDFClass = UDFClass;
   }
   public Method getUDFMethod() {

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java Sat Jan 24 01:58:01 2009
@@ -30,8 +30,8 @@
 import org.apache.hadoop.hive.ql.lib.Dispatcher;
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
 import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
 import org.apache.hadoop.hive.ql.lib.Rule;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.HiveParser;
@@ -76,7 +76,7 @@
   /**
    * Implements the process method for the NodeProcessor interface.
    */
-  public void process(Node nd, NodeProcessorCtx procCtx)
+  public Object process(Node nd, NodeProcessorCtx procCtx, Object... nodeOutputs)
   throws SemanticException {
     ASTNode pt = (ASTNode)nd;
 
@@ -91,7 +91,7 @@
       inputTableList.add(table_name);
       break;
     }
-
+    return null;
   }
 
   /**
@@ -128,7 +128,7 @@
     // Create a list of topop nodes
     ArrayList<Node> topNodes = new ArrayList<Node>();
     topNodes.add(tree);
-    ogw.startWalking(topNodes);
+    ogw.startWalking(topNodes, null);
   }
 
   public static void main(String[] args) throws IOException, ParseException,

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java Sat Jan 24 01:58:01 2009
@@ -18,11 +18,12 @@
 
 package org.apache.hadoop.hive.ql.udf;
 
-import org.apache.hadoop.hive.ql.exec.UDAF;
+import org.apache.hadoop.hive.ql.exec.NumericUDAF;
+import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
 
 
 
-public class UDAFAvg extends UDAF {
+public class UDAFAvg extends NumericUDAF implements UDAFEvaluator {
 
   private long mCount;
   private double mSum;
@@ -37,7 +38,7 @@
     mCount = 0;
   }
 
-  public boolean aggregate(Double o) {
+  public boolean iterate(Double o) {
     if (o != null) {
       mSum += o;
       mCount ++;
@@ -45,12 +46,12 @@
     return true;
   }
   
-  public String evaluatePartial() {
+  public String terminatePartial() {
     // This is SQL standard - average of zero items should be null.
     return mCount == 0 ? null : String.valueOf(mSum) + '/' + String.valueOf(mCount);
   }
 
-  public boolean aggregatePartial(String o) {
+  public boolean merge(String o) {
     if (o != null && !o.isEmpty()) {
       int pos = o.indexOf('/');
       assert(pos != -1);
@@ -60,7 +61,7 @@
     return true;
   }
 
-  public Double evaluate() {
+  public Double terminate() {
     // This is SQL standard - average of zero items should be null.
     return mCount == 0 ? null : Double.valueOf(mSum / mCount);
   }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java Sat Jan 24 01:58:01 2009
@@ -19,9 +19,10 @@
 package org.apache.hadoop.hive.ql.udf;
 
 import org.apache.hadoop.hive.ql.exec.UDAF;
+import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
 
 
-public class UDAFCount extends UDAF {
+public class UDAFCount extends UDAF implements UDAFEvaluator {
 
   private long mCount;
   
@@ -34,7 +35,7 @@
     mCount = 0;
   }
   
-  public boolean aggregate(Object o) {
+  public boolean iterate(Object o) {
     // Our SerDe between map/reduce boundary may convert MetadataTypedSerDe to 
     if (o != null && !o.equals("")) {
       mCount ++;
@@ -42,18 +43,17 @@
     return true;
   }
 
-  public Long evaluatePartial() {
+  public Long terminatePartial() {
     return Long.valueOf(mCount);
   }
 
-  public boolean aggregatePartial(Long count) {
+  public boolean merge(Long count) {
     mCount += count;
     return true;
   }
 
-  public Long evaluate() {
+  public Long terminate() {
     return Long.valueOf(mCount);
   }
 
-  
 }



Mime
View raw message