hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1447989 [5/7] - in /hive/branches/ptf-windowing/ql/src: java/org/apache/hadoop/hive/ql/ java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/ java/org/apache/hadoop/hive/ql/udf/gene...
Date Wed, 20 Feb 2013 02:27:18 GMT
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java Wed Feb 20 02:27:17 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.udf.pt
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
@@ -29,20 +28,22 @@ import org.apache.hadoop.hive.ql.exec.Ex
 import org.apache.hadoop.hive.ql.exec.PTFPartition;
 import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator;
 import org.apache.hadoop.hive.ql.exec.PTFUtils;
+import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.PTFSpec.SelectSpec;
 import org.apache.hadoop.hive.ql.parse.PTFTranslator;
 import org.apache.hadoop.hive.ql.parse.RowResolver;
 import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.parse.TypeCheckCtx;
+import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec;
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.ArgDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef;
 import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFInputDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
 import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
@@ -51,24 +52,30 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 
 /**
- * return rows that meet a specified pattern. Use symbols to specify a list of expressions to match.
- * Pattern is used to specify a Path. The results list can contain expressions based on the input columns
- * and also the matched Path.
+ * return rows that meet a specified pattern. Use symbols to specify a list of expressions
+ * to match.
+ * Pattern is used to specify a Path. The results list can contain expressions based on
+ * the input columns and also the matched Path.
  * <ol>
  * <li><b>pattern:</b> pattern for the Path. Path is 'dot' separated list of symbols.
- * Each element is treated as a symbol. Elements that end in '*' or '+' are interpreted with the
- * usual meaning of zero or more, one or more respectively. For e.g. "LATE.EARLY*.ONTIMEOREARLY" implies a sequence of flights
- * where the first occurrence was LATE, followed by zero or more EARLY flights, followed by a ONTIME or EARLY flight.
+ * Each element is treated as a symbol. Elements that end in '*' or '+' are interpreted with
+ * the usual meaning of zero or more, one or more respectively. For e.g.
+ * "LATE.EARLY*.ONTIMEOREARLY" implies a sequence of flights
+ * where the first occurrence was LATE, followed by zero or more EARLY flights,
+ * followed by a ONTIME or EARLY flight.
  * <li><b>symbols</b> specify a list of name, expression pairs. For e.g.
  * 'LATE', arrival_delay > 0, 'EARLY', arrival_delay < 0 , 'ONTIME', arrival_delay == 0.
  * These symbols can be used in the Pattern defined above.
  * <li><b>resultSelectList</b> specified as a select list.
- * The expressions in the selectList are evaluated in the context where all the input columns are available, plus the attribute
+ * The expressions in the selectList are evaluated in the context where all the
+ * input columns are available, plus the attribute
  * "tpath" is available. Path is a collection of rows that represents the matching Path.
  * </ol>
  */
@@ -83,6 +90,7 @@ public class NPath extends TableFunction
    * the names of the Columns of the input to NPath. Used to setup the tpath Struct column.
    */
   private ArrayList<String> inputColumnNames;
+  private ArrayList<String> selectListNames;
 
   @Override
   public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException
@@ -95,7 +103,8 @@ public class NPath extends TableFunction
       if (syFnRes.matches )
       {
         int sz = syFnRes.nextRow - (pItr.getIndex() - 1);
-        Object selectListInput = NPath.getSelectListInput(iRow, tDef.getInput().getOI(), pItr, sz);
+        Object selectListInput = NPath.getSelectListInput(iRow,
+            tDef.getInput().getOutputShape().getOI(), pItr, sz);
         ArrayList<Object> oRow = new ArrayList<Object>();
         for(ExprNodeEvaluator resExprEval : resultExprInfo.resultExprEvals)
         {
@@ -109,14 +118,33 @@ public class NPath extends TableFunction
   static void throwErrorWithSignature(String message) throws SemanticException
   {
     throw new SemanticException(PTFUtils.sprintf(
-        "NPath signature is: SymbolPattern, one or more SymbolName, expression pairs, the result expression as a select list. Error %s",
+        "NPath signature is: SymbolPattern, one or more SymbolName, " +
+        "expression pairs, the result expression as a select list. Error %s",
         message));
   }
+
+  public ArrayList<String> getInputColumnNames() {
+    return inputColumnNames;
+  }
+
+  public void setInputColumnNames(ArrayList<String> inputColumnNames) {
+    this.inputColumnNames = inputColumnNames;
+  }
+
+  public ArrayList<String> getSelectListNames() {
+    return selectListNames;
+  }
+
+  public void setSelectListNames(ArrayList<String> selectListNames) {
+    this.selectListNames = selectListNames;
+  }
+
   public static class NPathResolver extends TableFunctionResolver
   {
 
     @Override
-    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef)
+    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc,
+        PartitionedTableFunctionDef tDef)
     {
 
       return new NPath();
@@ -127,12 +155,14 @@ public class NPath extends TableFunction
      * <li> check structure of Arguments:
      * <ol>
      * <li> First arg should be a String
-     * <li> then there should be an even number of Arguments: String, expression; expression should be Convertible to Boolean.
+     * <li> then there should be an even number of Arguments:
+     * String, expression; expression should be Convertible to Boolean.
      * <li> finally there should be a String.
      * </ol>
      * <li> convert pattern into a NNode chain.
      * <li> convert symbol args into a Symbol Map.
-     * <li> parse selectList into SelectList struct. The inputOI used to translate these expressions should be based on the
+     * <li> parse selectList into SelectList struct. The inputOI used to translate
+     * these expressions should be based on the
      * columns in the Input, the 'path.attr'
      * </ul>
      */
@@ -140,9 +170,9 @@ public class NPath extends TableFunction
     public void setupOutputOI() throws SemanticException
     {
       NPath evaluator = (NPath) getEvaluator();
-      TableFuncDef tDef = evaluator.getTableDef();
+      PartitionedTableFunctionDef tDef = evaluator.getTableDef();
 
-      ArrayList<ArgDef> args = tDef.getArgs();
+      ArrayList<PTFExpressionDef> args = tDef.getArgs();
       int argsNum = args == null ? 0 : args.size();
 
       if ( argsNum < 4 )
@@ -163,7 +193,8 @@ public class NPath extends TableFunction
       /*
        * parse ResultExpr Str and setup OI.
        */
-      ResultExpressionParser resultExprParser = new ResultExpressionParser(evaluator.resultExprStr, selectListInputRR);
+      ResultExpressionParser resultExprParser =
+          new ResultExpressionParser(evaluator.resultExprStr, selectListInputRR);
       try {
         resultExprParser.translate();
       }
@@ -172,55 +203,70 @@ public class NPath extends TableFunction
       }
       evaluator.resultExprInfo = resultExprParser.getResultExprInfo();
       StructObjectInspector OI = evaluator.resultExprInfo.resultOI;
+      evaluator.selectListNames = new ArrayList<String>();
+      extractOIColumnNames(resultExprParser.selectListInputOI, evaluator.selectListNames);
+
       setOutputOI(OI);
     }
     /*
      * validate and setup patternStr
      */
-    private void validateAndSetupPatternStr(NPath evaluator, ArrayList<ArgDef> args) throws SemanticException {
-      ArgDef symboPatternArg = args.get(0);
+    private void validateAndSetupPatternStr(NPath evaluator,
+        ArrayList<PTFExpressionDef> args) throws SemanticException {
+      PTFExpressionDef symboPatternArg = args.get(0);
       ObjectInspector symbolPatternArgOI = symboPatternArg.getOI();
 
       if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) ||
           (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
-          ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
+          ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() !=
+          PrimitiveObjectInspector.PrimitiveCategory.STRING )
       {
         throwErrorWithSignature("Currently the symbol Pattern must be a Constant String.");
       }
 
-      evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI).getWritableConstantValue().toString();
+      evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI).
+          getWritableConstantValue().toString();
     }
 
     /*
      * validate and setup SymbolInfo
      */
-    private void validateAndSetupSymbolInfo(NPath evaluator, ArrayList<ArgDef> args, int argsNum) throws SemanticException {
+    private void validateAndSetupSymbolInfo(NPath evaluator,
+        ArrayList<PTFExpressionDef> args,
+        int argsNum) throws SemanticException {
       int symbolArgsSz = argsNum - 2;
       if ( symbolArgsSz % 2 != 0)
       {
-        throwErrorWithSignature("Symbol Name, Expression need to be specified in pairs: there are odd number of symbol args");
+        throwErrorWithSignature("Symbol Name, Expression need to be specified in pairs: " +
+        		"there are odd number of symbol args");
       }
 
       evaluator.symInfo = new SymbolsInfo(symbolArgsSz/2);
       for(int i=1; i <= symbolArgsSz; i += 2)
       {
-        ArgDef symbolNameArg = args.get(i);
+        PTFExpressionDef symbolNameArg = args.get(i);
         ObjectInspector symbolNameArgOI = symbolNameArg.getOI();
 
         if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolNameArgOI) ||
             (symbolNameArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
-            ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
+            ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() !=
+            PrimitiveObjectInspector.PrimitiveCategory.STRING )
         {
-          throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Name(%s) must be a Constant String", symbolNameArg.getExpression().toStringTree()));
+          throwErrorWithSignature(
+              PTFUtils.sprintf("Currently a Symbol Name(%s) must be a Constant String",
+                  symbolNameArg.getExpressionTreeString()));
         }
-        String symbolName = ((ConstantObjectInspector)symbolNameArgOI).getWritableConstantValue().toString();
+        String symbolName = ((ConstantObjectInspector)symbolNameArgOI).
+            getWritableConstantValue().toString();
 
-        ArgDef symolExprArg = args.get(i+1);
+        PTFExpressionDef symolExprArg = args.get(i+1);
         ObjectInspector symolExprArgOI = symolExprArg.getOI();
         if ( (symolExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
-              ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN )
+              ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() !=
+              PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN )
         {
-          throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Expression(%s) must be a boolean expression", symolExprArg.getExpression().toStringTree()));
+          throwErrorWithSignature(PTFUtils.sprintf("Currently a Symbol Expression(%s) " +
+          		"must be a boolean expression", symolExprArg.getExpressionTreeString()));
         }
         evaluator.symInfo.add(symbolName, symolExprArg);
       }
@@ -229,18 +275,22 @@ public class NPath extends TableFunction
     /*
      * validate and setup resultExprStr
      */
-    private void validateAndSetupResultExprStr(NPath evaluator, ArrayList<ArgDef> args, int argsNum) throws SemanticException {
-      ArgDef resultExprArg = args.get(argsNum - 1);
+    private void validateAndSetupResultExprStr(NPath evaluator,
+        ArrayList<PTFExpressionDef> args,
+        int argsNum) throws SemanticException {
+      PTFExpressionDef resultExprArg = args.get(argsNum - 1);
       ObjectInspector resultExprArgOI = resultExprArg.getOI();
 
       if ( !ObjectInspectorUtils.isConstantObjectInspector(resultExprArgOI) ||
             (resultExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
-            ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING )
+            ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() !=
+            PrimitiveObjectInspector.PrimitiveCategory.STRING )
       {
         throwErrorWithSignature("Currently the result Expr parameter must be a Constant String.");
       }
 
-      evaluator.resultExprStr = ((ConstantObjectInspector)resultExprArgOI).getWritableConstantValue().toString();
+      evaluator.resultExprStr = ((ConstantObjectInspector)resultExprArgOI).
+          getWritableConstantValue().toString();
     }
 
     /*
@@ -264,9 +314,9 @@ public class NPath extends TableFunction
     public void initializeOutputOI() throws HiveException {
       try {
         NPath evaluator = (NPath) getEvaluator();
-        TableFuncDef tDef = evaluator.getTableDef();
+        PartitionedTableFunctionDef tDef = evaluator.getTableDef();
 
-        ArrayList<ArgDef> args = tDef.getArgs();
+        ArrayList<PTFExpressionDef> args = tDef.getArgs();
         int argsNum = args.size();
 
         validateAndSetupPatternStr(evaluator, args);
@@ -277,15 +327,16 @@ public class NPath extends TableFunction
         /*
          * setup OI for input to resultExpr select list
          */
-        RowResolver selectListInputRR = NPath.createSelectListRR(evaluator, tDef.getInput());
-        StructObjectInspector selectListInputOI = (StructObjectInspector) PTFTranslator.getInputOI(selectListInputRR);
+        StructObjectInspector selectListInputOI = NPath.createSelectListOI( evaluator,
+            tDef.getInput());
         ResultExprInfo resultExprInfo = evaluator.resultExprInfo;
         ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
         resultExprInfo.resultExprEvals = new ArrayList<ExprNodeEvaluator>();
 
         for(int i=0 ; i < resultExprInfo.resultExprNodes.size(); i++) {
           ExprNodeDesc selectColumnExprNode =resultExprInfo.resultExprNodes.get(i);
-          ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode);
+          ExprNodeEvaluator selectColumnExprEval =
+              ExprNodeEvaluatorFactory.get(selectColumnExprNode);
           ObjectInspector selectColumnOI = selectColumnExprEval.initialize(selectListInputOI);
           resultExprInfo.resultExprEvals.add(selectColumnExprEval);
           selectListExprOIs.add(selectColumnOI);
@@ -306,6 +357,15 @@ public class NPath extends TableFunction
       return evaluator.resultExprInfo.getResultExprNames();
     }
 
+
+
+    private static void extractOIColumnNames(StructObjectInspector OI,
+        ArrayList<String> oiColumnNames) {
+      StructTypeInfo t = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(OI);
+      ArrayList<String> fnames = t.getAllStructFieldNames();
+      oiColumnNames.addAll(fnames);
+    }
+
   }
 
   public ResultExprInfo getResultExprInfo() {
@@ -330,7 +390,7 @@ public class NPath extends TableFunction
       symbolExprsNames = new ArrayList<String>(sz);
     }
 
-    void add(String name, ArgDef arg)
+    void add(String name, PTFExpressionDef arg)
     {
       symbolExprsNames.add(name);
       symbolExprsEvaluators.add(arg.getExprEvaluator());
@@ -584,7 +644,8 @@ public class NPath extends TableFunction
         String symbolName = symbolNames.get(i);
         ExprNodeEvaluator symbolExprEval = symbolExprEvals.get(i);
         ObjectInspector symbolExprOI = symbolExprOIs.get(i);
-        symbolExprEvalMap.put(symbolName.toLowerCase(), new Object[] {symbolExprEval, symbolExprOI});
+        symbolExprEvalMap.put(symbolName.toLowerCase(),
+            new Object[] {symbolExprEval, symbolExprOI});
       }
     }
 
@@ -633,7 +694,8 @@ public class NPath extends TableFunction
    * - the select keyword is optional. The parser checks if the expression doesn't start with
    * select; if not it prefixes it.
    * - Window Fn clauses are not permitted.
-   * - expressions can operate on the input columns plus the psuedo column 'path' which is array of
+   * - expressions can operate on the input columns plus the psuedo column 'path'
+   * which is array of
    * structs. The shape of the struct is
    * the same as the input.
    */
@@ -644,11 +706,12 @@ public class NPath extends TableFunction
     TypeCheckCtx selectListInputTypeCheckCtx;
     StructObjectInspector selectListInputOI;
 
-    SelectSpec selectSpec;
+    ArrayList<WindowExpressionSpec> selectSpec;
 
     ResultExprInfo resultExprInfo;
 
-    public ResultExpressionParser(String resultExprString, RowResolver selectListInputRowResolver)
+    public ResultExpressionParser(String resultExprString,
+        RowResolver selectListInputRowResolver)
     {
       this.resultExprString = resultExprString;
       this.selectListInputRowResolver = selectListInputRowResolver;
@@ -676,15 +739,15 @@ public class NPath extends TableFunction
       //result
       ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
       int i = 0;
-      Iterator<Object> it = selectSpec.getColumnListAndAlias();
-      while (it.hasNext())
+      for(WindowExpressionSpec expr : selectSpec)
       {
-        Object[] selectColDetails = (Object[]) it.next();
-        String selectColName = (String) selectColDetails[1];
-        ASTNode selectColumnNode = (ASTNode) selectColDetails[2];
-        ExprNodeDesc selectColumnExprNode = PTFTranslator.buildExprNode(selectColumnNode,
+        String selectColName = expr.getAlias();
+        ASTNode selectColumnNode = expr.getExpression();
+        ExprNodeDesc selectColumnExprNode =
+            ResultExpressionParser.buildExprNode(selectColumnNode,
             selectListInputTypeCheckCtx);
-        ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode);
+        ExprNodeEvaluator selectColumnExprEval =
+            ExprNodeEvaluatorFactory.get(selectColumnExprNode);
         ObjectInspector selectColumnOI = null;
         selectColumnOI = selectColumnExprEval.initialize(selectListInputOI);
 
@@ -708,7 +771,8 @@ public class NPath extends TableFunction
       /*
        * create SelectListOI
        */
-      selectListInputOI = (StructObjectInspector) PTFTranslator.getInputOI(selectListInputRowResolver);
+      selectListInputOI = (StructObjectInspector)
+          PTFTranslator.getStandardStructOI(selectListInputRowResolver);
     }
 
     private void fixResultExprString()
@@ -729,16 +793,9 @@ public class NPath extends TableFunction
 
     private void validateSelectExpr() throws SemanticException
     {
-      if (selectSpec.getWindowFuncs() != null)
-      {
-        throw new SemanticException(
-            "NPath Result Expression cannot have Windowing Function expressions");
-      }
-
-      for (ASTNode node : selectSpec.getExpressions())
+      for (WindowExpressionSpec expr : selectSpec)
       {
-        PTFTranslator.validateNoLeadLagInValueBoundarySpec(node,
-            "Lead/Lag not allowed in NPath Result Expression");
+        PTFTranslator.validateNoLeadLagInValueBoundarySpec(expr.getExpression());
       }
     }
 
@@ -755,6 +812,24 @@ public class NPath extends TableFunction
       }
       return "npath_col_" + colIdx;
     }
+
+    public static ExprNodeDesc buildExprNode(ASTNode expr,
+        TypeCheckCtx typeCheckCtx) throws SemanticException
+    {
+      // todo: use SemanticAnalyzer::genExprNodeDesc
+      // currently SA not available to PTFTranslator.
+      HashMap<Node, Object> map = TypeCheckProcFactory
+          .genExprNode(expr, typeCheckCtx);
+      ExprNodeDesc desc = (ExprNodeDesc) map.get(expr);
+      if (desc == null) {
+        String errMsg = typeCheckCtx.getError();
+        if ( errMsg == null) {
+          errMsg = "Error in parsing ";
+        }
+        throw new SemanticException(errMsg);
+      }
+      return desc;
+    }
   }
 
   public static final String PATHATTR_NAME = "tpath";
@@ -762,9 +837,10 @@ public class NPath extends TableFunction
   /*
    * add array<struct> to the list of columns
    */
-  protected static RowResolver createSelectListRR(NPath evaluator, PTFInputDef inpDef) throws SemanticException {
+  protected static RowResolver createSelectListRR(NPath evaluator,
+      PTFInputDef inpDef) throws SemanticException {
     RowResolver rr = new RowResolver();
-    RowResolver inputRR = inpDef.getInputInfo().getRowResolver();
+    RowResolver inputRR = inpDef.getOutputShape().getRr();
     boolean inputColNamesKnown = evaluator.inputColumnNames != null;
 
     if ( !inputColNamesKnown ) {
@@ -797,8 +873,10 @@ public class NPath extends TableFunction
       inpColOIs.add(cInfo.getObjectInspector());
     }
 
-    StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector(
-        ObjectInspectorFactory.getStandardStructObjectInspector(evaluator.inputColumnNames, inpColOIs));
+    StandardListObjectInspector pathAttrOI =
+        ObjectInspectorFactory.getStandardListObjectInspector(
+        ObjectInspectorFactory.getStandardStructObjectInspector(evaluator.inputColumnNames,
+            inpColOIs));
 
     ColumnInfo pathColumn = new ColumnInfo(PATHATTR_NAME,
         TypeInfoUtils.getTypeInfoFromObjectInspector(pathAttrOI),
@@ -809,6 +887,25 @@ public class NPath extends TableFunction
     return rr;
   }
 
+  protected static StructObjectInspector createSelectListOI(NPath evaluator, PTFInputDef inpDef) {
+    StructObjectInspector inOI = inpDef.getOutputShape().getOI();
+    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
+    for(StructField f : inOI.getAllStructFieldRefs()) {
+      fieldOIs.add(f.getFieldObjectInspector());
+    }
+
+    StandardListObjectInspector pathAttrOI =
+        ObjectInspectorFactory.getStandardListObjectInspector(
+        ObjectInspectorFactory.getStandardStructObjectInspector(evaluator.inputColumnNames,
+            fieldOIs));
+
+    ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>();
+    selectFieldOIs.addAll(fieldOIs);
+    selectFieldOIs.add(pathAttrOI);
+    return ObjectInspectorFactory.getStandardStructObjectInspector(
+        evaluator.selectListNames, selectFieldOIs);
+  }
+
   public static Object getSelectListInput(Object currRow, ObjectInspector rowOI,
       PTFPartitionIterator<Object> pItr, int sz) {
     ArrayList<Object> oRow = new ArrayList<Object>();

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java Wed Feb 20 02:27:17 2013
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.PT
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 
 public class Noop extends TableFunctionEvaluator
@@ -47,7 +47,7 @@ public class Noop extends TableFunctionE
   {
 
     @Override
-    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef)
+    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef)
     {
       return new Noop();
     }
@@ -55,7 +55,7 @@ public class Noop extends TableFunctionE
     @Override
     public void setupOutputOI() throws SemanticException
     {
-      StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI();
+      StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI();
       setOutputOI(OI);
     }
 

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java Wed Feb 20 02:27:17 2013
@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.exec.PT
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 
 public class NoopWithMap extends Noop
@@ -45,7 +45,7 @@ public class NoopWithMap extends Noop
   {
 
     @Override
-    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef)
+    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef)
     {
       return new NoopWithMap();
     }
@@ -53,7 +53,7 @@ public class NoopWithMap extends Noop
     @Override
     public void setupOutputOI() throws SemanticException
     {
-      StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI();
+      StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI();
       setOutputOI(OI);
     }
 
@@ -80,7 +80,7 @@ public class NoopWithMap extends Noop
     @Override
     public void setupRawInputOI() throws SemanticException
     {
-      StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOI();
+      StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI();
       setRawInputOI(OI);
     }
 

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java Wed Feb 20 02:27:17 2013
@@ -26,7 +26,7 @@ import org.apache.hadoop.hive.ql.exec.PT
 import org.apache.hadoop.hive.ql.exec.PTFUtils;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 
@@ -50,9 +50,18 @@ import org.apache.hadoop.hive.serde2.obj
  */
 public abstract class TableFunctionEvaluator
 {
+  /*
+   * how is this different from the OutpuShape set on the TableDef.
+   * This is the OI of the object coming out of the PTF.
+   * It is put in an output Partition whose Serde is usually LazyBinarySerde.
+   * So the next PTF (or Operator) in the chain gets a LazyBinaryStruct.
+   */
   transient protected StructObjectInspector OI;
+  /*
+   * same comment as OI applies here.
+   */
   transient protected StructObjectInspector rawInputOI;
-  protected TableFuncDef tDef;
+  protected PartitionedTableFunctionDef tDef;
   protected PTFDesc ptfDesc;
   String partitionClass;
   int partitionMemSize;
@@ -74,12 +83,12 @@ public abstract class TableFunctionEvalu
     OI = outputOI;
   }
 
-  public TableFuncDef getTableDef()
+  public PartitionedTableFunctionDef getTableDef()
   {
     return tDef;
   }
 
-  public void setTableDef(TableFuncDef tDef)
+  public void setTableDef(PartitionedTableFunctionDef tDef)
   {
     this.tDef = tDef;
   }
@@ -138,7 +147,7 @@ public abstract class TableFunctionEvalu
     PTFPartitionIterator<Object> pItr = iPart.iterator();
     PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr);
     PTFPartition outP = new PTFPartition(getPartitionClass(),
-        getPartitionMemSize(), tDef.getSerde(), OI);
+        getPartitionMemSize(), tDef.getOutputShape().getSerde(), OI);
     execute(pItr, outP);
     return outP;
   }
@@ -159,3 +168,4 @@ public abstract class TableFunctionEvalu
     return null;
   }
 }
+

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java Wed Feb 20 02:27:17 2013
@@ -28,7 +28,7 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 
@@ -65,11 +65,10 @@ public abstract class TableFunctionResol
    * - sets up the evaluator with references to the TableDef, PartitionClass, PartitonMemsize and
    *   the transformsRawInput boolean.
    */
-  public void initialize(PTFDesc ptfDesc, TableFuncDef tDef)
+  public void initialize(HiveConf cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef)
       throws SemanticException
   {
     this.ptfDesc = ptfDesc;
-    HiveConf cfg = ptfDesc.getTranslationInfo().getHiveCfg();
     String partitionClass = HiveConf.getVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENCE_CLASS);
     int partitionMemSize = HiveConf.getIntVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENT_SIZE);
 
@@ -85,7 +84,7 @@ public abstract class TableFunctionResol
   /*
    * called during deserialization of a QueryDef during runtime.
    */
-  public void initialize(PTFDesc ptfDesc, TableFuncDef tDef, TableFunctionEvaluator evaluator)
+  public void initialize(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef, TableFunctionEvaluator evaluator)
       throws HiveException
   {
     this.evaluator = evaluator;
@@ -211,6 +210,5 @@ public abstract class TableFunctionResol
   /*
    * a subclass must provide the {@link TableFunctionEvaluator} instance.
    */
-  protected abstract TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc,
-      TableFuncDef tDef);
+  protected abstract TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef);
 }

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java Wed Feb 20 02:27:17 2013
@@ -22,32 +22,25 @@ import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.PTFOperator;
 import org.apache.hadoop.hive.ql.exec.PTFPartition;
 import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator;
-import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.BoundarySpec;
-import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFrameSpec.Direction;
-import org.apache.hadoop.hive.ql.parse.PTFSpec.WindowFunctionSpec;
-import org.apache.hadoop.hive.ql.parse.PTFTranslator;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.ArgDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.SelectDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.TableFuncDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.BoundaryDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.CurrentRowDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.RangeBoundaryDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef.ValueBoundaryDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.CurrentRowDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.RangeBoundaryDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef;
 import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef;
+import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
 import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
@@ -56,7 +49,19 @@ import org.apache.hadoop.hive.serde2.obj
 
 public class WindowingTableFunction extends TableFunctionEvaluator
 {
-  ArrayList<WindowFunctionDef> wFnDefs;
+
+  @Override
+  public PTFPartition execute(PTFPartition iPart)
+      throws HiveException
+  {
+    WindowTableFunctionDef wFnDef = (WindowTableFunctionDef) getTableDef();
+    PTFPartitionIterator<Object> pItr = iPart.iterator();
+    PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr);
+    PTFPartition outP = new PTFPartition(getPartitionClass(),
+        getPartitionMemSize(), wFnDef.getOutputFromWdwFnProcessing().getSerde(), OI);
+    execute(pItr, outP);
+    return outP;
+  }
 
   @SuppressWarnings({ "unchecked", "rawtypes" })
   @Override
@@ -71,13 +76,15 @@ public class WindowingTableFunction exte
       throw new HiveException(se);
     }
 
-    for(WindowFunctionDef wFn : wFnDefs)
+    WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
+
+    for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions())
     {
-      boolean processWindow = wFn.getWindow() != null && wFn.getWindow().getWindow() != null;
+      boolean processWindow = wFn.getWindowFrame() != null;
       pItr.reset();
       if ( !processWindow )
       {
-        GenericUDAFEvaluator fEval = wFn.getEvaluator();
+        GenericUDAFEvaluator fEval = wFn.getwFnEval();
         Object[] args = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()];
         AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer();
         while(pItr.hasNext())
@@ -85,7 +92,7 @@ public class WindowingTableFunction exte
           Object row = pItr.next();
           int i =0;
           if ( wFn.getArgs() != null ) {
-            for(ArgDef arg : wFn.getArgs())
+            for(PTFExpressionDef arg : wFn.getArgs())
             {
               args[i++] = arg.getExprEvaluator().evaluate(row);
             }
@@ -93,8 +100,7 @@ public class WindowingTableFunction exte
           fEval.aggregate(aggBuffer, args);
         }
         Object out = fEval.evaluate(aggBuffer);
-        WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getSpec().getName());
-        if ( !wFnInfo.isPivotResult())
+        if ( !wFn.isPivotResult())
         {
           out = new SameList(iPart.size(), out);
         }
@@ -133,58 +139,30 @@ public class WindowingTableFunction exte
 
   public static class WindowingTableFunctionResolver extends TableFunctionResolver
   {
+    /*
+     * OI of object constructed from output of Wdw Fns; before it is put
+     * in the Wdw Processing Partition. Set by Translator/Deserializer.
+     */
+    private transient StructObjectInspector wdwProcessingOutputOI;
 
-    @Override
-    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, TableFuncDef tDef)
-    {
+    public StructObjectInspector getWdwProcessingOutputOI() {
+      return wdwProcessingOutputOI;
+    }
 
-      return new WindowingTableFunction();
+    public void setWdwProcessingOutputOI(StructObjectInspector wdwProcessingOutputOI) {
+      this.wdwProcessingOutputOI = wdwProcessingOutputOI;
     }
 
     @Override
-    public void setupOutputOI() throws SemanticException
+    protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef)
     {
-      ArrayList<WindowFunctionDef> wFnDefs = new ArrayList<WindowFunctionDef>();
-      PTFDesc ptfDesc = getPtfDesc();
-      SelectDef select = ptfDesc.getSelectList();
-      ArrayList<WindowFunctionSpec> wFnSpecs = ptfDesc.getSpec().getSelectList().getWindowFuncs();
-      ArrayList<String> aliases = new ArrayList<String>();
-      ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
-
-      /*
-       * Setup the columns in the OI in the following order
-       * - the columns representing the Window Fns
-       * - the columns from the input
-       * Why?
-       * - during translation the input contains Virtual columns that are not represent during runtime
-       * - this messes with the Column Numbers (and hence internal Names) if we add the columns in a different order.
-       */
-
-      for(WindowFunctionSpec wFnS : wFnSpecs)
-      {
-          WindowFunctionDef wFnDef = PTFTranslator.translate(ptfDesc, getEvaluator().getTableDef(), wFnS);
-          WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnS.getName());
-          wFnDefs.add(wFnDef);
-          aliases.add(wFnS.getAlias());
-          if ( wFnInfo.isPivotResult())
-          {
-            ListObjectInspector lOI = (ListObjectInspector) wFnDef.getOI();
-            fieldOIs.add(lOI.getListElementObjectInspector());
-          }
-          else
-          {
-            fieldOIs.add(wFnDef.getOI());
-          }
-      }
-
-      PTFTranslator.addInputColumnsToList(ptfDesc, getEvaluator().getTableDef(), aliases, fieldOIs);
 
-      select.setWindowFuncs(wFnDefs);
-      WindowingTableFunction wTFn = (WindowingTableFunction) getEvaluator();
-      wTFn.wFnDefs = wFnDefs;
+      return new WindowingTableFunction();
+    }
 
-      StructObjectInspector OI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs);
-      setOutputOI(OI);
+    @Override
+    public void setupOutputOI() throws SemanticException {
+      setOutputOI(wdwProcessingOutputOI);
     }
 
     /*
@@ -195,42 +173,7 @@ public class WindowingTableFunction exte
     @Override
     public void initializeOutputOI() throws HiveException
     {
-      PTFDesc ptfDesc = getPtfDesc();
-      TableFuncDef tblFuncDef = evaluator.getTableDef();
-      WindowingTableFunction wTFn = (WindowingTableFunction) tblFuncDef.getFunction();
-      ArrayList<WindowFunctionDef> wFnDefs = ptfDesc.getSelectList().getWindowFuncs();
-      ArrayList<String> aliases = new ArrayList<String>();
-      ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
-
-      /*
-       * Setup the columns in the OI in the following order
-       * - the columns representing the Window Fns
-       * - the columns from the input
-       * Why?
-       * - during translation the input contains Virtual columns that are not present during runtime
-       * - this messes with the Column Numbers (and hence internal Names) if we add the columns in a different order.
-       */
-
-      for (WindowFunctionDef wFnDef : wFnDefs) {
-        WindowFunctionSpec wFnS = wFnDef.getSpec();
-        WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnS.getName());
-        aliases.add(wFnS.getAlias());
-        if ( wFnInfo.isPivotResult())
-        {
-          ListObjectInspector lOI = (ListObjectInspector) wFnDef.getOI();
-          fieldOIs.add(lOI.getListElementObjectInspector());
-        }
-        else
-        {
-          fieldOIs.add(wFnDef.getOI());
-        }
-
-      }
-      PTFTranslator.addInputColumnsToList(ptfDesc, getEvaluator().getTableDef(), aliases, fieldOIs);
-
-      wTFn.wFnDefs = wFnDefs;
-      StructObjectInspector OI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs);
-      setOutputOI(OI);
+      setupOutputOI();
     }
 
 
@@ -267,7 +210,7 @@ public class WindowingTableFunction exte
   {
     ArrayList<Object> vals = new ArrayList<Object>();
 
-    GenericUDAFEvaluator fEval = wFnDef.getEvaluator();
+    GenericUDAFEvaluator fEval = wFnDef.getwFnEval();
     Object[] args = new Object[wFnDef.getArgs().size()];
     for(int i=0; i < iPart.size(); i++)
     {
@@ -279,7 +222,7 @@ public class WindowingTableFunction exte
       {
         Object row = rItr.next();
         int j = 0;
-        for(ArgDef arg : wFnDef.getArgs())
+        for(PTFExpressionDef arg : wFnDef.getArgs())
         {
           args[j++] = arg.getExprEvaluator().evaluate(row);
         }
@@ -294,8 +237,8 @@ public class WindowingTableFunction exte
 
   static Range getRange(WindowFunctionDef wFnDef, int currRow, PTFPartition p) throws HiveException
   {
-    BoundaryDef startB = wFnDef.getWindow().getWindow().getStart();
-    BoundaryDef endB = wFnDef.getWindow().getWindow().getEnd();
+    BoundaryDef startB = wFnDef.getWindowFrame().getStart();
+    BoundaryDef endB = wFnDef.getWindowFrame().getEnd();
 
     int start = getIndex(startB, currRow, p, false);
     int end = getIndex(endB, currRow, p, true);
@@ -440,6 +383,7 @@ public class WindowingTableFunction exte
     public abstract boolean isGreater(Object v1, Object v2, int amt);
 
 
+    @SuppressWarnings("incomplete-switch")
     public static ValueBoundaryScanner getScanner(ValueBoundaryDef vbDef)
     {
       PrimitiveObjectInspector pOI = (PrimitiveObjectInspector) vbDef.getOI();

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_general_queries.q Wed Feb 20 02:27:17 2013
@@ -32,7 +32,7 @@ LOAD DATA LOCAL INPATH '../data/files/fl
 select p_mfgr, p_name, p_size,
 rank() as r,
 dense_rank() as dr,
-sum(p_retailprice) as s1 over (rows between unbounded preceding and current row)
+sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row)
 from noop(part 
   distribute by p_mfgr
   sort by p_name
@@ -51,7 +51,7 @@ sort by p_name;
 select p_mfgr, p_name,
 rank() as r,
 dense_rank() as dr,
-p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row),
+p_retailprice, sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row),
 p_size, p_size - lag(p_size,1) as deltaSz
 from noop(part
 distribute by p_mfgr
@@ -74,19 +74,14 @@ from part
 distribute by p_mfgr
 sort by p_name ;
 
--- 6. testJoinWithLeadLag
-select p1.p_mfgr, p1.p_name,
-p1.p_size, p1.p_size - lag(p1.p_size,1) as deltaSz
-from part p1 join part p2 on p1.p_partkey = p2.p_partkey
-distribute by p1.p_mfgr
-sort by p1.p_name ;
-        
 -- 7. testJoinWithNoop
 select p_mfgr, p_name,
 p_size, p_size - lag(p_size,1) as deltaSz
 from noop ( (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j
 distribute by j.p_mfgr
-sort by j.p_name);    
+sort by j.p_name)
+distribute by p_mfgr
+sort by p_name;    
         
 -- 8. testGroupByWithSWQ
 select p_mfgr, p_name, p_size, min(p_retailprice),
@@ -119,7 +114,7 @@ sort by p_name);
 select p_mfgr, p_name, p_size,
 rank() as r,
 dense_rank() as dr,
-sum(p_retailprice) as s1 over (rows between unbounded preceding and current row)
+sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row)
 from noop(part 
   distribute by p_mfgr
   sort by p_name
@@ -156,9 +151,9 @@ select p_mfgr, p_name, 
 rank() as r, 
 dense_rank() as dr, 
 count(p_size) as cd 
-from noop(part 
+from part 
 distribute by p_mfgr 
-sort by p_name);
+sort by p_name;
 
 -- 15. testCountWithWindowing
 select p_mfgr, p_name, 
@@ -167,9 +162,9 @@ dense_rank() as dr, 
 count(p_size) as cd, 
 p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), 
 p_size, p_size - lag(p_size,1) as deltaSz 
-from noop(part 
+from part 
 distribute by p_mfgr 
-sort by p_name);
+sort by p_name;
 
 
 -- 16. testCountInSubQ
@@ -180,9 +175,9 @@ dense_rank() as dr, 
 count(p_size) as cd, 
 p_retailprice, sum(p_retailprice) as s1 over (rows between unbounded preceding and current row), 
 p_size, p_size - lag(p_size,1) as deltaSz 
-from noop(part 
+from part 
 distribute by p_mfgr 
-sort by p_name)
+sort by p_name
 ) sub1;
 
 -- 17. testJoin
@@ -214,12 +209,13 @@ sort by abc.p_name ;
 
 -- 20. testMixedCaseAlias
 select p_mfgr, p_name, p_size, rank() as R
-from noop(part 
+from part 
 distribute by p_mfgr 
-sort by p_name, p_size desc);
+sort by p_name, p_size desc;
 
 -- 21. testNoopWithMap
-select p_mfgr, p_name, p_size, rank() as r
+select p_mfgr, p_name, p_size, 
+rank() as r over (distribute by p_mfgr sort by p_name, p_size desc)
 from noopwithmap(part
 distribute by p_mfgr
 sort by p_name, p_size desc);
@@ -228,7 +224,7 @@ sort by p_name, p_size desc);
 select p_mfgr, p_name, p_size,
 rank() as r,
 dense_rank() as dr,
-sum(p_retailprice) as s1 over (rows between unbounded preceding and current row)
+sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row)
 from noopwithmap(part 
   distribute by p_mfgr
   sort by p_name);
@@ -242,14 +238,14 @@ from part 
 having p_size > 5 
 distribute by p_mfgr 
 sort by p_name; 
-    		
+
 -- 24. testHavingWithWindowingCondRankNoGBY
 select p_mfgr, p_name, p_size, 
 rank() as r, 
 dense_rank() as dr, 
 sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) 
 from part 
-having r < 4 
+having rank() < 4 
 distribute by p_mfgr 
 sort by p_name;
 
@@ -257,12 +253,12 @@ sort by p_name;
 select p_mfgr, p_name, p_size,
 rank() as r,
 dense_rank() as dr,
-sum(p_retailprice) as s1 over (rows between unbounded preceding and current row)
+sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row)
 from noop(part
 distribute by p_mfgr
 sort by p_name)
-having r < 4;
-
+having rank() < 4;
+    		
 -- 26. testFirstLast   
 select  p_mfgr,p_name, p_size, 
 sum(p_size) as s2 over (rows between current row and current row), 
@@ -345,7 +341,7 @@ window w1 as (rows between 2 preceding a
 select p_mfgr, p_name, p_size, 
 rank() as r, 
 dense_rank() as dr, 
-sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) 
+sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) 
 from noop(noopwithmap(noop(part 
 distribute by p_mfgr 
 sort by p_mfgr, p_name
@@ -361,7 +357,7 @@ sum(p_retailprice) as s1 over w1 
 from noop(part 
 distribute by p_mfgr 
 sort by p_name) 
-window w1 as (rows between 2 preceding and 2 following) 
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following) 
 ) sub1 ;
 
 -- 35. testCountStar
@@ -479,7 +475,7 @@ sum(s) as s1 over w1 
 from noop(mfgr_price_view 
 distribute by p_mfgr 
 sort by p_mfgr)  
-window w1 as (rows between 2 preceding and current row);
+window w1 as ( distribute by p_mfgr sort by p_brand rows between 2 preceding and current row);
 
 -- 46. testCreateViewWithWindowingQuery
 create view IF NOT EXISTS mfgr_brand_price_view as 
@@ -566,7 +562,7 @@ select * from part_1;
 select * from part_2;
 
 select * from part_3;
-	 
+
 -- 50. testGroupByHavingWithSWQAndAlias
 select p_mfgr, p_name, p_size, min(p_retailprice) as mi,
 rank() as r,
@@ -577,7 +573,7 @@ group by p_mfgr, p_name, p_size
 having p_size > 0
 distribute by p_mfgr
 sort by p_name;
-
+	 
 -- 51. testMultipleRangeWindows
 select  p_mfgr,p_name, p_size, 
 sum(p_size) as s2 over (range between p_size 10 less and current row), 
@@ -707,7 +703,7 @@ window w1 as (rows between 2 preceding a
 select p_mfgr, p_name,  
 rank() as r, 
 dense_rank() as dr, 
-p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) 
+p_size, sum(p_size) as s1 over (distribute by p_mfgr,p_name rows between unbounded preceding and current row) 
 from noop(
         noopwithmap(
           noop(
@@ -724,7 +720,7 @@ from noop(
 select p_mfgr, p_name,  
 rank() as r, 
 dense_rank() as dr, 
-p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row) 
+p_size, sum(p_size) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) 
 from noop(
         noop(
           noop(
@@ -739,7 +735,7 @@ from noop(
         
 -- 63. testMultiOperatorChainWithNoWindowing
 select p_mfgr, p_name,  
-rank() as r, 
+rank() as r over (distribute by p_mfgr sort by p_name), 
 dense_rank() as dr, 
 p_size, sum(p_size) as s1 
 from noop(
@@ -757,7 +753,7 @@ from noop(
 select p_mfgr, p_name,  
 rank() as r, 
 dense_rank() as dr, 
-p_size, sum(p_size) as s1 over (rows between unbounded preceding and current row)  
+p_size, sum(p_size) as s1 over (distribute by p_mfgr,p_name rows between unbounded preceding and current row)  
 from noopwithmap(
         noop(
           noop(

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_rcfile.q Wed Feb 20 02:27:17 2013
@@ -18,7 +18,7 @@ LOAD DATA LOCAL INPATH '../data/files/pa
 select p_mfgr, p_name, p_size, 
 rank() as r, 
 dense_rank() as dr, 
-sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) 
+sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) 
 from noop(part_rc 
 distribute by p_mfgr 
 sort by p_name);

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q?rev=1447989&r1=1447988&r2=1447989&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_seqfile.q Wed Feb 20 02:27:17 2013
@@ -18,7 +18,7 @@ LOAD DATA LOCAL INPATH '../data/files/pa
 select p_mfgr, p_name, p_size, 
 rank() as r, 
 dense_rank() as dr, 
-sum(p_retailprice) as s1 over (rows between unbounded preceding and current row) 
+sum(p_retailprice) as s1 over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) 
 from noop(part_seq 
 distribute by p_mfgr 
 sort by p_name);  



Mime
View raw message