hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1451274 - in /hive/branches/ptf-windowing/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/udf/generic/ test/queries/clientpositive/ test/results/clientpositive/
Date Thu, 28 Feb 2013 18:20:46 GMT
Author: hashutosh
Date: Thu Feb 28 18:20:45 2013
New Revision: 1451274

URL: http://svn.apache.org/r1451274
Log:
HIVE-4080: Add Lead & Lag UDAFs (Harish Butani via Ashutosh Chauhan)

Added:
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLag.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLead.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java
    hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/leadlag_queries.q
    hive/branches/ptf-windowing/ql/src/test/results/clientpositive/leadlag_queries.q.out
Modified:
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1451274&r1=1451273&r2=1451274&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
(original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
Thu Feb 28 18:20:45 2013
@@ -149,6 +149,8 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFPercentRank;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFPercentileApprox;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFRank;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFLead;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFLag;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver2;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFRowNumber;
@@ -528,6 +530,8 @@ public final class FunctionRegistry {
     registerWindowFunction("ntile", new GenericUDAFNTile());
     registerWindowFunction("first_value", new GenericUDAFFirstValue());
     registerWindowFunction("last_value", new GenericUDAFLastValue());
+    registerWindowFunction(LEAD_FUNC_NAME, new GenericUDAFLead(), false);
+    registerWindowFunction(LAG_FUNC_NAME, new GenericUDAFLag(), false);
 
     RANKING_FUNCTIONS.add("rank");
     RANKING_FUNCTIONS.add("dense_rank");
@@ -855,6 +859,26 @@ public final class FunctionRegistry {
     return udafEvaluator;
   }
 
+  @SuppressWarnings("deprecation")
+  public static GenericUDAFEvaluator getGenericWindowingEvaluator(String name,
+      List<ObjectInspector> argumentOIs, boolean isDistinct,
+      boolean isAllColumns) throws SemanticException {
+
+    WindowFunctionInfo finfo = windowFunctions.get(name.toLowerCase());
+    if (finfo == null) { return null;}
+    if ( !name.toLowerCase().equals(LEAD_FUNC_NAME) &&
+    !name.toLowerCase().equals(LAG_FUNC_NAME) ) {
+    return getGenericUDAFEvaluator(name, argumentOIs, isDistinct, isAllColumns);
+    }
+
+    // this must be lead/lag UDAF
+    ObjectInspector args[] = new ObjectInspector[argumentOIs.size()];
+    GenericUDAFResolver udafResolver = finfo.getfInfo().getGenericUDAFResolver();
+    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(
+    argumentOIs.toArray(args), isDistinct, isAllColumns);
+    return ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo);
+  }
+
   /**
    * This method is shared between UDFRegistry and UDAFRegistry. methodName will
    * be "evaluate" for UDFRegistry, and "aggregate"/"evaluate"/"evaluatePartial"
@@ -1417,8 +1441,34 @@ public final class FunctionRegistry {
 
   public static void registerWindowFunction(String name, GenericUDAFResolver wFn)
   {
-    registerGenericUDAF(true, name, wFn);
-    FunctionInfo fInfo = getFunctionInfo(name);
+    registerWindowFunction(name, wFn, true);
+  }
+
+  /**
+   * Typically a WindowFunction is the same as a UDAF. The only exceptions are Lead &
Lag UDAFs. These
+   * are not registered as regular UDAFs because
+   * - we plan to support Lead & Lag as UDFs (usable only within argument expressions
+   *   of UDAFs when windowing is involved). Since mFunctions holds both UDFs and UDAFs we
cannot
+   *   add both FunctionInfos to mFunctions.
+   * We choose to only register UDFs in mFunctions. The implication of this is that Lead/Lag
UDAFs
+   * are only usable when windowing is involved.
+   *
+   * @param name
+   * @param wFn
+   * @param registerAsUDAF
+   */
+  public static void registerWindowFunction(String name, GenericUDAFResolver wFn, boolean
registerAsUDAF)
+  {
+    FunctionInfo fInfo = null;
+    if (registerAsUDAF) {
+      registerGenericUDAF(true, name, wFn);
+      fInfo = getFunctionInfo(name);
+    }
+    else {
+      fInfo = new FunctionInfo(true,
+          name.toLowerCase(), wFn);
+    }
+
     WindowFunctionInfo wInfo = new WindowFunctionInfo(fInfo);
     windowFunctions.put(name.toLowerCase(), wInfo);
   }

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1451274&r1=1451273&r2=1451274&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
(original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
Thu Feb 28 18:20:45 2013
@@ -686,7 +686,7 @@ public class PTFTranslator {
       funcArgOIs = argOIs.toArray(funcArgOIs);
     }
 
-    GenericUDAFEvaluator wFnEval = FunctionRegistry.getGenericUDAFEvaluator(def.getName(),
argOIs,
+    GenericUDAFEvaluator wFnEval = FunctionRegistry.getGenericWindowingEvaluator(def.getName(),
argOIs,
         def.isDistinct(), def.isStar());
     ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs);
     def.setWFnEval(wFnEval);

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1451274&r1=1451273&r2=1451274&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Thu Feb 28 18:20:45 2013
@@ -9937,6 +9937,17 @@ public class SemanticAnalyzer extends Ba
     boolean hasLLArgs = false;
     boolean isRankingOrNavFunction = FunctionRegistry.RANKING_FUNCTIONS.contains(fnName)
||
         FunctionRegistry.NAVIGATION_FUNCTIONS.contains(fnName);
+
+    /*
+     * treat Lead & Lag as a UDF if there is no WindowSpec.
+     */
+    if (!hasWindowSpec
+        &&
+        (fnName.equals(FunctionRegistry.LAG_FUNC_NAME) || fnName
+            .equals(FunctionRegistry.LEAD_FUNC_NAME))) {
+      return false;
+    }
+
     /*
      * If Windowing Function has LeadLag expression in its args,
      * then it will be handled by WindowingTabFunc.

Added: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLag.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLag.java?rev=1451274&view=auto
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLag.java
(added)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLag.java
Thu Feb 28 18:20:45 2013
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+@WindowFunctionDescription
+(
+    description = @Description(
+                name = "lag",
+                value = "_FUNC_(expr, amt, default)"
+                ),
+    supportsWindow = false,
+    pivotResult = true
+)
+public class GenericUDAFLag extends GenericUDAFLeadLag {
+
+  static final Log LOG = LogFactory.getLog(GenericUDAFLag.class.getName());
+
+
+  @Override
+  protected String functionName() {
+    return "Lag";
+  }
+
+  @Override
+  protected GenericUDAFLeadLagEvaluator createLLEvaluator() {
+    return new GenericUDAFLagEvaluator();
+  }
+
+  public static class GenericUDAFLagEvaluator extends GenericUDAFLeadLagEvaluator {
+
+    @Override
+    protected LeadLagBuffer getNewLLBuffer() throws HiveException {
+     return new LagBuffer();
+    }
+  }
+
+  static class LagBuffer implements LeadLagBuffer {
+    ArrayList<Object> values;
+    int lagAmt;
+    ArrayList<Object> lagValues;
+    int lastRowIdx;
+
+    public void initialize(int lagAmt) {
+      this.lagAmt = lagAmt;
+      lagValues = new ArrayList<Object>(lagAmt);
+      values = new ArrayList<Object>();
+      lastRowIdx = -1;
+    }
+
+    public void addRow(Object currValue, Object defaultValue) {
+      int row = lastRowIdx + 1;
+      if ( row < lagAmt) {
+        lagValues.add(defaultValue);
+      }
+      values.add(currValue);
+      lastRowIdx++;
+    }
+
+    public Object terminate() {
+      int lastIdx = values.size() - 1;
+      for(int i = 0; i < lagAmt; i++) {
+        values.remove(lastIdx - i);
+      }
+      values.addAll(0, lagValues);
+      return values;
+    }
+  }
+}

Added: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLead.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLead.java?rev=1451274&view=auto
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLead.java
(added)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLead.java
Thu Feb 28 18:20:45 2013
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+@WindowFunctionDescription
+(
+    description = @Description(
+                name = "lead",
+                value = "_FUNC_(expr, amt, default)"
+                ),
+    supportsWindow = false,
+    pivotResult = true
+)
+public class GenericUDAFLead extends GenericUDAFLeadLag {
+
+  static final Log LOG = LogFactory.getLog(GenericUDAFLead.class.getName());
+
+
+  @Override
+  protected String functionName() {
+    return "Lead";
+  }
+
+  @Override
+  protected GenericUDAFLeadLagEvaluator createLLEvaluator() {
+   return new GenericUDAFLeadEvaluator();
+  }
+
+  public static class GenericUDAFLeadEvaluator extends GenericUDAFLeadLagEvaluator {
+
+    @Override
+    protected LeadLagBuffer getNewLLBuffer() throws HiveException {
+     return new LeadBuffer();
+    }
+
+  }
+
+  static class LeadBuffer implements LeadLagBuffer {
+    ArrayList<Object> values;
+    int leadAmt;
+    Object[] leadWindow;
+    int nextPosInWindow;
+    int lastRowIdx;
+
+    public void initialize(int leadAmt) {
+      this.leadAmt = leadAmt;
+      values = new ArrayList<Object>();
+      leadWindow = new Object[leadAmt];
+      nextPosInWindow = 0;
+      lastRowIdx = -1;
+    }
+
+    public void addRow(Object leadExprValue, Object defaultValue) {
+      int row = lastRowIdx + 1;
+      int leadRow = row - leadAmt;
+      if ( leadRow >= 0 ) {
+        values.add(leadExprValue);
+      }
+      leadWindow[nextPosInWindow] = defaultValue;
+      nextPosInWindow = (nextPosInWindow + 1) % leadAmt;
+      lastRowIdx++;
+    }
+
+    public Object terminate() {
+      /*
+       * if there are fewer than leadAmt values in leadWindow; start reading from the first
position.
+       * Otherwise the window starts from nextPosInWindow.
+       */
+      if ( lastRowIdx < leadAmt ) {
+        nextPosInWindow = 0;
+      }
+      for(int i=0; i < leadAmt; i++) {
+        values.add(leadWindow[nextPosInWindow]);
+        nextPosInWindow = (nextPosInWindow + 1) % leadAmt;
+      }
+      return values;
+    }
+
+  }
+
+}

Added: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java?rev=1451274&view=auto
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java
(added)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java
Thu Feb 28 18:20:45 2013
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.io.IntWritable;
+
+/**
+ * abstract class for Lead & lag UDAFs
+ * GenericUDAFLeadLag.
+ *
+ */
+public abstract class GenericUDAFLeadLag extends AbstractGenericUDAFResolver {
+
+  static final Log LOG = LogFactory.getLog(GenericUDAFLead.class.getName());
+
+  @Override
+  public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo parameters)
+      throws SemanticException {
+
+    ObjectInspector[] paramOIs = parameters.getParameterObjectInspectors();
+    String fNm = functionName();
+
+    if (!(paramOIs.length >= 1 && paramOIs.length <= 3) ) {
+      throw new UDFArgumentTypeException(paramOIs.length - 1,
+          "Incorrect invocation of " + fNm + ": _FUNC_(expr, amt, default)");
+    }
+
+    int amt = 1;
+    if ( paramOIs.length > 1 ) {
+      ObjectInspector amtOI = paramOIs[1];
+
+      if ( !ObjectInspectorUtils.isConstantObjectInspector(amtOI) ||
+          (amtOI.getCategory() != ObjectInspector.Category.PRIMITIVE) ||
+          ((PrimitiveObjectInspector)amtOI).getPrimitiveCategory() !=
+          PrimitiveObjectInspector.PrimitiveCategory.INT )
+      {
+        throw new UDFArgumentTypeException(0,
+            fNm + " amount must be a integer value "
+            + amtOI.getTypeName() + " was passed as parameter 1.");
+      }
+      Object o = ((ConstantObjectInspector)amtOI).
+          getWritableConstantValue();
+      amt = ((IntWritable)o).get();
+    }
+
+    if (paramOIs.length == 3) {
+      ObjectInspectorConverters.getConverter(paramOIs[2], paramOIs[0]);
+    }
+
+    GenericUDAFLeadLagEvaluator eval = createLLEvaluator();
+    eval.setAmt(amt);
+    return eval;
+  }
+
+  protected abstract String functionName();
+
+  protected abstract GenericUDAFLeadLagEvaluator createLLEvaluator();
+
+  static interface LeadLagBuffer extends AggregationBuffer {
+    void initialize(int leadAmt);
+    void addRow(Object leadExprValue, Object defaultValue) ;
+    Object terminate();
+
+  }
+
+  public static abstract class GenericUDAFLeadLagEvaluator extends GenericUDAFEvaluator {
+
+    ObjectInspector[] inputOI;
+    int amt;
+    String fnName;
+    Converter defaultValueConverter;
+
+    @Override
+    public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException
+    {
+      super.init(m, parameters);
+      if (m != Mode.COMPLETE)
+      {
+        throw new HiveException(
+            "Only COMPLETE mode supported for " + fnName + " function");
+      }
+
+      inputOI = parameters;
+
+      if (parameters.length == 3) {
+        defaultValueConverter = ObjectInspectorConverters.getConverter(parameters[2], parameters[0]);
+      }
+
+      return ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorUtils
+          .getStandardObjectInspector(parameters[0]));
+    }
+
+    public int getAmt() {
+      return amt;
+    }
+
+    public void setAmt(int amt) {
+      this.amt = amt;
+    }
+
+    public String getFnName() {
+      return fnName;
+    }
+
+    public void setFnName(String fnName) {
+      this.fnName = fnName;
+    }
+
+    protected abstract LeadLagBuffer getNewLLBuffer() throws HiveException;
+
+    @Override
+    public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+      LeadLagBuffer lb =  getNewLLBuffer();
+      lb.initialize(amt);
+      return lb;
+    }
+
+    @Override
+    public void reset(AggregationBuffer agg) throws HiveException {
+      ((LeadLagBuffer)agg).initialize(amt);
+    }
+
+    @Override
+    public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException
{
+      Object rowExprVal = ObjectInspectorUtils.copyToStandardObject(parameters[0],
+          inputOI[0]);
+      Object defaultVal = parameters.length > 2 ? ObjectInspectorUtils.copyToStandardObject(
+          defaultValueConverter.convert(parameters[2]),
+          inputOI[0]) : null;
+      ((LeadLagBuffer)agg).addRow(rowExprVal, defaultVal);
+    }
+
+    @Override
+    public Object terminatePartial(AggregationBuffer agg) throws HiveException {
+      throw new HiveException("terminatePartial not supported");
+    }
+
+    @Override
+    public void merge(AggregationBuffer agg, Object partial) throws HiveException {
+      throw new HiveException("merge not supported");
+    }
+
+    @Override
+    public Object terminate(AggregationBuffer agg) throws HiveException {
+      return ((LeadLagBuffer)agg).terminate();
+    }
+
+  }
+
+}

Added: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/leadlag_queries.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/leadlag_queries.q?rev=1451274&view=auto
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/leadlag_queries.q (added)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/leadlag_queries.q Thu Feb
28 18:20:45 2013
@@ -0,0 +1,53 @@
+-- data setup
+CREATE TABLE part( 
+    p_partkey INT,
+    p_name STRING,
+    p_mfgr STRING,
+    p_brand STRING,
+    p_type STRING,
+    p_size INT,
+    p_container STRING,
+    p_retailprice DOUBLE,
+    p_comment STRING
+);
+
+LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part;
+
+-- 1. testLeadUDAF
+select p_mfgr, p_retailprice,
+lead(p_retailprice) as l1 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1) as l2 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1,10) as l3 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1, p_retailprice) as l4 over (partition by p_mfgr order by p_name),
+p_retailprice - lead(p_retailprice,1)
+from part;
+
+-- 2.testLeadUDAFPartSz1
+select p_mfgr, p_name, p_retailprice,
+lead(p_retailprice,1) over (partition by p_mfgr, p_name ),
+p_retailprice - lead(p_retailprice,1)
+from part;
+
+-- 3.testLagUDAF
+select p_mfgr, p_retailprice,
+lag(p_retailprice,1) as l1 over (partition by p_mfgr order by p_name),
+lag(p_retailprice) as l2 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1, p_retailprice) as l3 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1,10) as l4 over (partition by p_mfgr order by p_name),
+p_retailprice - lag(p_retailprice,1)
+from part;
+
+-- 4.testLagUDAFPartSz1
+select p_mfgr, p_name, p_retailprice,
+lag(p_retailprice,1) over (partition by p_mfgr, p_name ),
+p_retailprice - lag(p_retailprice,1)
+from part;
+
+-- 5.testLeadLagUDAF
+select p_mfgr, p_retailprice,
+lead(p_retailprice,1) as l1 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1, p_retailprice) as l2 over (partition by p_mfgr order by p_name),
+p_retailprice - lead(p_retailprice,1),
+lag(p_retailprice,1) as l3 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1, p_retailprice) as l4 over (partition by p_mfgr order by p_name) 
+from part;
\ No newline at end of file

Added: hive/branches/ptf-windowing/ql/src/test/results/clientpositive/leadlag_queries.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientpositive/leadlag_queries.q.out?rev=1451274&view=auto
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientpositive/leadlag_queries.q.out (added)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientpositive/leadlag_queries.q.out Thu
Feb 28 18:20:45 2013
@@ -0,0 +1,261 @@
+PREHOOK: query: -- data setup
+CREATE TABLE part( 
+    p_partkey INT,
+    p_name STRING,
+    p_mfgr STRING,
+    p_brand STRING,
+    p_type STRING,
+    p_size INT,
+    p_container STRING,
+    p_retailprice DOUBLE,
+    p_comment STRING
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- data setup
+CREATE TABLE part( 
+    p_partkey INT,
+    p_name STRING,
+    p_mfgr STRING,
+    p_brand STRING,
+    p_type STRING,
+    p_size INT,
+    p_container STRING,
+    p_retailprice DOUBLE,
+    p_comment STRING
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@part
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table
part
+PREHOOK: type: LOAD
+PREHOOK: Output: default@part
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table
part
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@part
+PREHOOK: query: -- 1. testLeadUDAF
+select p_mfgr, p_retailprice,
+lead(p_retailprice) as l1 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1) as l2 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1,10) as l3 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1, p_retailprice) as l4 over (partition by p_mfgr order by p_name),
+p_retailprice - lead(p_retailprice,1)
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1. testLeadUDAF
+select p_mfgr, p_retailprice,
+lead(p_retailprice) as l1 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1) as l2 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1,10) as l3 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1, p_retailprice) as l4 over (partition by p_mfgr order by p_name),
+p_retailprice - lead(p_retailprice,1)
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1	1173.15	1173.15	1173.15	1173.15	1173.15	0.0
+Manufacturer#1	1173.15	1753.76	1753.76	1753.76	1753.76	-580.6099999999999
+Manufacturer#1	1753.76	1602.59	1602.59	1602.59	1602.59	151.17000000000007
+Manufacturer#1	1602.59	1414.42	1414.42	1414.42	1414.42	188.16999999999985
+Manufacturer#1	1414.42	1632.66	1632.66	1632.66	1632.66	-218.24
+Manufacturer#1	1632.66	NULL	NULL	10.0	1632.66	0.0
+Manufacturer#2	1690.68	1800.7	1800.7	1800.7	1800.7	-110.01999999999998
+Manufacturer#2	1800.7	2031.98	2031.98	2031.98	2031.98	-231.27999999999997
+Manufacturer#2	2031.98	1698.66	1698.66	1698.66	1698.66	333.31999999999994
+Manufacturer#2	1698.66	1701.6	1701.6	1701.6	1701.6	-2.939999999999827
+Manufacturer#2	1701.6	NULL	NULL	10.0	1701.6	0.0
+Manufacturer#3	1671.68	1190.27	1190.27	1190.27	1190.27	481.4100000000001
+Manufacturer#3	1190.27	1410.39	1410.39	1410.39	1410.39	-220.12000000000012
+Manufacturer#3	1410.39	1922.98	1922.98	1922.98	1922.98	-512.5899999999999
+Manufacturer#3	1922.98	1337.29	1337.29	1337.29	1337.29	585.69
+Manufacturer#3	1337.29	NULL	NULL	10.0	1337.29	0.0
+Manufacturer#4	1620.67	1375.42	1375.42	1375.42	1375.42	245.25
+Manufacturer#4	1375.42	1206.26	1206.26	1206.26	1206.26	169.16000000000008
+Manufacturer#4	1206.26	1844.92	1844.92	1844.92	1844.92	-638.6600000000001
+Manufacturer#4	1844.92	1290.35	1290.35	1290.35	1290.35	554.5700000000002
+Manufacturer#4	1290.35	NULL	NULL	10.0	1290.35	0.0
+Manufacturer#5	1789.69	1611.66	1611.66	1611.66	1611.66	178.02999999999997
+Manufacturer#5	1611.66	1788.73	1788.73	1788.73	1788.73	-177.06999999999994
+Manufacturer#5	1788.73	1018.1	1018.1	1018.1	1018.1	770.63
+Manufacturer#5	1018.1	1464.48	1464.48	1464.48	1464.48	-446.38
+Manufacturer#5	1464.48	NULL	NULL	10.0	1464.48	0.0
+PREHOOK: query: -- 2.testLeadUDAFPartSz1
+select p_mfgr, p_name, p_retailprice,
+lead(p_retailprice,1) over (partition by p_mfgr, p_name ),
+p_retailprice - lead(p_retailprice,1)
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2.testLeadUDAFPartSz1
+select p_mfgr, p_name, p_retailprice,
+lead(p_retailprice,1) over (partition by p_mfgr, p_name ),
+p_retailprice - lead(p_retailprice,1)
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1	almond antique burnished rose metallic	1173.15	1173.15	0.0
+Manufacturer#1	almond antique burnished rose metallic	1173.15	NULL	0.0
+Manufacturer#1	almond antique chartreuse lavender yellow	1753.76	NULL	0.0
+Manufacturer#1	almond antique salmon chartreuse burlywood	1602.59	NULL	0.0
+Manufacturer#1	almond aquamarine burnished black steel	1414.42	NULL	0.0
+Manufacturer#1	almond aquamarine pink moccasin thistle	1632.66	NULL	0.0
+Manufacturer#2	almond antique violet chocolate turquoise	1690.68	NULL	0.0
+Manufacturer#2	almond antique violet turquoise frosted	1800.7	NULL	0.0
+Manufacturer#2	almond aquamarine midnight light salmon	2031.98	NULL	0.0
+Manufacturer#2	almond aquamarine rose maroon antique	1698.66	NULL	0.0
+Manufacturer#2	almond aquamarine sandy cyan gainsboro	1701.6	NULL	0.0
+Manufacturer#3	almond antique chartreuse khaki white	1671.68	NULL	0.0
+Manufacturer#3	almond antique forest lavender goldenrod	1190.27	NULL	0.0
+Manufacturer#3	almond antique metallic orange dim	1410.39	NULL	0.0
+Manufacturer#3	almond antique misty red olive	1922.98	NULL	0.0
+Manufacturer#3	almond antique olive coral navajo	1337.29	NULL	0.0
+Manufacturer#4	almond antique gainsboro frosted violet	1620.67	NULL	0.0
+Manufacturer#4	almond antique violet mint lemon	1375.42	NULL	0.0
+Manufacturer#4	almond aquamarine floral ivory bisque	1206.26	NULL	0.0
+Manufacturer#4	almond aquamarine yellow dodger mint	1844.92	NULL	0.0
+Manufacturer#4	almond azure aquamarine papaya violet	1290.35	NULL	0.0
+Manufacturer#5	almond antique blue firebrick mint	1789.69	NULL	0.0
+Manufacturer#5	almond antique medium spring khaki	1611.66	NULL	0.0
+Manufacturer#5	almond antique sky peru orange	1788.73	NULL	0.0
+Manufacturer#5	almond aquamarine dodger light gainsboro	1018.1	NULL	0.0
+Manufacturer#5	almond azure blanched chiffon midnight	1464.48	NULL	0.0
+PREHOOK: query: -- 3.testLagUDAF
+select p_mfgr, p_retailprice,
+lag(p_retailprice,1) as l1 over (partition by p_mfgr order by p_name),
+lag(p_retailprice) as l2 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1, p_retailprice) as l3 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1,10) as l4 over (partition by p_mfgr order by p_name),
+p_retailprice - lag(p_retailprice,1)
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: -- 3.testLagUDAF
+select p_mfgr, p_retailprice,
+lag(p_retailprice,1) as l1 over (partition by p_mfgr order by p_name),
+lag(p_retailprice) as l2 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1, p_retailprice) as l3 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1,10) as l4 over (partition by p_mfgr order by p_name),
+p_retailprice - lag(p_retailprice,1)
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1	1173.15	NULL	NULL	1173.15	10.0	0.0
+Manufacturer#1	1173.15	1173.15	1173.15	1173.15	1173.15	0.0
+Manufacturer#1	1753.76	1173.15	1173.15	1173.15	1173.15	580.6099999999999
+Manufacturer#1	1602.59	1753.76	1753.76	1753.76	1753.76	-151.17000000000007
+Manufacturer#1	1414.42	1602.59	1602.59	1602.59	1602.59	-188.16999999999985
+Manufacturer#1	1632.66	1414.42	1414.42	1414.42	1414.42	218.24
+Manufacturer#2	1690.68	NULL	NULL	1690.68	10.0	0.0
+Manufacturer#2	1800.7	1690.68	1690.68	1690.68	1690.68	110.01999999999998
+Manufacturer#2	2031.98	1800.7	1800.7	1800.7	1800.7	231.27999999999997
+Manufacturer#2	1698.66	2031.98	2031.98	2031.98	2031.98	-333.31999999999994
+Manufacturer#2	1701.6	1698.66	1698.66	1698.66	1698.66	2.939999999999827
+Manufacturer#3	1671.68	NULL	NULL	1671.68	10.0	0.0
+Manufacturer#3	1190.27	1671.68	1671.68	1671.68	1671.68	-481.4100000000001
+Manufacturer#3	1410.39	1190.27	1190.27	1190.27	1190.27	220.12000000000012
+Manufacturer#3	1922.98	1410.39	1410.39	1410.39	1410.39	512.5899999999999
+Manufacturer#3	1337.29	1922.98	1922.98	1922.98	1922.98	-585.69
+Manufacturer#4	1620.67	NULL	NULL	1620.67	10.0	0.0
+Manufacturer#4	1375.42	1620.67	1620.67	1620.67	1620.67	-245.25
+Manufacturer#4	1206.26	1375.42	1375.42	1375.42	1375.42	-169.16000000000008
+Manufacturer#4	1844.92	1206.26	1206.26	1206.26	1206.26	638.6600000000001
+Manufacturer#4	1290.35	1844.92	1844.92	1844.92	1844.92	-554.5700000000002
+Manufacturer#5	1789.69	NULL	NULL	1789.69	10.0	0.0
+Manufacturer#5	1611.66	1789.69	1789.69	1789.69	1789.69	-178.02999999999997
+Manufacturer#5	1788.73	1611.66	1611.66	1611.66	1611.66	177.06999999999994
+Manufacturer#5	1018.1	1788.73	1788.73	1788.73	1788.73	-770.63
+Manufacturer#5	1464.48	1018.1	1018.1	1018.1	1018.1	446.38
+PREHOOK: query: -- 4.testLagUDAFPartSz1
+select p_mfgr, p_name, p_retailprice,
+lag(p_retailprice,1) over (partition by p_mfgr, p_name ),
+p_retailprice - lag(p_retailprice,1)
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: -- 4.testLagUDAFPartSz1
+select p_mfgr, p_name, p_retailprice,
+lag(p_retailprice,1) over (partition by p_mfgr, p_name ),
+p_retailprice - lag(p_retailprice,1)
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1	almond antique burnished rose metallic	1173.15	NULL	0.0
+Manufacturer#1	almond antique burnished rose metallic	1173.15	1173.15	0.0
+Manufacturer#1	almond antique chartreuse lavender yellow	1753.76	NULL	0.0
+Manufacturer#1	almond antique salmon chartreuse burlywood	1602.59	NULL	0.0
+Manufacturer#1	almond aquamarine burnished black steel	1414.42	NULL	0.0
+Manufacturer#1	almond aquamarine pink moccasin thistle	1632.66	NULL	0.0
+Manufacturer#2	almond antique violet chocolate turquoise	1690.68	NULL	0.0
+Manufacturer#2	almond antique violet turquoise frosted	1800.7	NULL	0.0
+Manufacturer#2	almond aquamarine midnight light salmon	2031.98	NULL	0.0
+Manufacturer#2	almond aquamarine rose maroon antique	1698.66	NULL	0.0
+Manufacturer#2	almond aquamarine sandy cyan gainsboro	1701.6	NULL	0.0
+Manufacturer#3	almond antique chartreuse khaki white	1671.68	NULL	0.0
+Manufacturer#3	almond antique forest lavender goldenrod	1190.27	NULL	0.0
+Manufacturer#3	almond antique metallic orange dim	1410.39	NULL	0.0
+Manufacturer#3	almond antique misty red olive	1922.98	NULL	0.0
+Manufacturer#3	almond antique olive coral navajo	1337.29	NULL	0.0
+Manufacturer#4	almond antique gainsboro frosted violet	1620.67	NULL	0.0
+Manufacturer#4	almond antique violet mint lemon	1375.42	NULL	0.0
+Manufacturer#4	almond aquamarine floral ivory bisque	1206.26	NULL	0.0
+Manufacturer#4	almond aquamarine yellow dodger mint	1844.92	NULL	0.0
+Manufacturer#4	almond azure aquamarine papaya violet	1290.35	NULL	0.0
+Manufacturer#5	almond antique blue firebrick mint	1789.69	NULL	0.0
+Manufacturer#5	almond antique medium spring khaki	1611.66	NULL	0.0
+Manufacturer#5	almond antique sky peru orange	1788.73	NULL	0.0
+Manufacturer#5	almond aquamarine dodger light gainsboro	1018.1	NULL	0.0
+Manufacturer#5	almond azure blanched chiffon midnight	1464.48	NULL	0.0
+PREHOOK: query: -- 5.testLeadLagUDAF
+select p_mfgr, p_retailprice,
+lead(p_retailprice,1) as l1 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1, p_retailprice) as l2 over (partition by p_mfgr order by p_name),
+p_retailprice - lead(p_retailprice,1),
+lag(p_retailprice,1) as l3 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1, p_retailprice) as l4 over (partition by p_mfgr order by p_name) 
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: -- 5.testLeadLagUDAF
+select p_mfgr, p_retailprice,
+lead(p_retailprice,1) as l1 over (partition by p_mfgr order by p_name),
+lead(p_retailprice,1, p_retailprice) as l2 over (partition by p_mfgr order by p_name),
+p_retailprice - lead(p_retailprice,1),
+lag(p_retailprice,1) as l3 over (partition by p_mfgr order by p_name),
+lag(p_retailprice,1, p_retailprice) as l4 over (partition by p_mfgr order by p_name) 
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1	1173.15	1173.15	1173.15	0.0	NULL	1173.15
+Manufacturer#1	1173.15	1753.76	1753.76	-580.6099999999999	1173.15	1173.15
+Manufacturer#1	1753.76	1602.59	1602.59	151.17000000000007	1173.15	1173.15
+Manufacturer#1	1602.59	1414.42	1414.42	188.16999999999985	1753.76	1753.76
+Manufacturer#1	1414.42	1632.66	1632.66	-218.24	1602.59	1602.59
+Manufacturer#1	1632.66	NULL	1632.66	0.0	1414.42	1414.42
+Manufacturer#2	1690.68	1800.7	1800.7	-110.01999999999998	NULL	1690.68
+Manufacturer#2	1800.7	2031.98	2031.98	-231.27999999999997	1690.68	1690.68
+Manufacturer#2	2031.98	1698.66	1698.66	333.31999999999994	1800.7	1800.7
+Manufacturer#2	1698.66	1701.6	1701.6	-2.939999999999827	2031.98	2031.98
+Manufacturer#2	1701.6	NULL	1701.6	0.0	1698.66	1698.66
+Manufacturer#3	1671.68	1190.27	1190.27	481.4100000000001	NULL	1671.68
+Manufacturer#3	1190.27	1410.39	1410.39	-220.12000000000012	1671.68	1671.68
+Manufacturer#3	1410.39	1922.98	1922.98	-512.5899999999999	1190.27	1190.27
+Manufacturer#3	1922.98	1337.29	1337.29	585.69	1410.39	1410.39
+Manufacturer#3	1337.29	NULL	1337.29	0.0	1922.98	1922.98
+Manufacturer#4	1620.67	1375.42	1375.42	245.25	NULL	1620.67
+Manufacturer#4	1375.42	1206.26	1206.26	169.16000000000008	1620.67	1620.67
+Manufacturer#4	1206.26	1844.92	1844.92	-638.6600000000001	1375.42	1375.42
+Manufacturer#4	1844.92	1290.35	1290.35	554.5700000000002	1206.26	1206.26
+Manufacturer#4	1290.35	NULL	1290.35	0.0	1844.92	1844.92
+Manufacturer#5	1789.69	1611.66	1611.66	178.02999999999997	NULL	1789.69
+Manufacturer#5	1611.66	1788.73	1788.73	-177.06999999999994	1789.69	1789.69
+Manufacturer#5	1788.73	1018.1	1018.1	770.63	1611.66	1611.66
+Manufacturer#5	1018.1	1464.48	1464.48	-446.38	1788.73	1788.73
+Manufacturer#5	1464.48	NULL	1464.48	0.0	1018.1	1018.1



Mime
View raw message