hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecapri...@apache.org
Subject svn commit: r1532108 - in /hive/trunk: data/files/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Mon, 14 Oct 2013 21:40:45 GMT
Author: ecapriolo
Date: Mon Oct 14 21:40:44 2013
New Revision: 1532108

URL: http://svn.apache.org/r1532108
Log:
An explode function that includes the item's position in the array (Niko Stahl via egc)

Added:
    hive/trunk/data/files/posexplode_data.txt   (with props)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java
    hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q
    hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out

Added: hive/trunk/data/files/posexplode_data.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/posexplode_data.txt?rev=1532108&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hive/trunk/data/files/posexplode_data.txt
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1532108&r1=1532107&r2=1532108&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Mon Oct 14
21:40:44 2013
@@ -432,6 +432,7 @@ public final class FunctionRegistry {
     registerGenericUDTF("inline", GenericUDTFInline.class);
     registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class);
     registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class);
+    registerGenericUDTF("posexplode", GenericUDTFPosExplode.class);
     registerGenericUDTF("stack", GenericUDTFStack.class);
 
     //PTF declarations

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java?rev=1532108&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java
(added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java
Mon Oct 14 21:40:44 2013
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+/**
+ * PosExplode.
+ *
+ */
+@Description(
+    name = "posexplode",
+    value = "_FUNC_(a) - behaves like explode for arrays, "
+    + "but includes the position of items in the original array")
+public class GenericUDTFPosExplode extends GenericUDTF {
+  private ListObjectInspector listOI = null;
+  private final Object[] forwardObj = new Object[2];
+
+  @Override
+  public void close() throws HiveException {
+  }
+
+  @Override
+  public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException
{
+
+    if (args.length != 1) {
+      throw new UDFArgumentException("posexplode() takes only one argument");
+    }
+
+    if (args[0].getCategory() != ObjectInspector.Category.LIST) {
+      throw new UDFArgumentException("posexplode() takes an array as a parameter");
+    }
+    listOI = (ListObjectInspector) args[0];
+
+    ArrayList<String> fieldNames = new ArrayList<String>();
+    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
+    fieldNames.add("pos");
+    fieldNames.add("val");
+    fieldOIs.add(PrimitiveObjectInspectorFactory
+        .getPrimitiveJavaObjectInspector(PrimitiveCategory.INT));
+    fieldOIs.add(listOI.getListElementObjectInspector());
+    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+  }
+
+  @Override
+  public void process(Object[] o) throws HiveException {
+    List<?> list = listOI.getList(o[0]);
+    for (int i = 0; i < list.size(); i++) {
+      Object r = list.get(i);
+      forwardObj[0] = new Integer(i);
+      forwardObj[1] = r;
+      forward(forwardObj);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "posexplode";
+  }
+}

Added: hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q?rev=1532108&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q Mon Oct 14 21:40:44 2013
@@ -0,0 +1,15 @@
+CREATE TABLE employees (
+name STRING,
+salary FLOAT,
+subordinates ARRAY<STRING>,
+deductions MAP<STRING, FLOAT>,
+address STRUCT<street:STRING, city:STRING, state:STRING, zip:INT>);
+
+LOAD DATA LOCAL INPATH '../data/files/posexplode_data.txt' INTO TABLE employees;
+
+SELECT
+  name, pos, sub
+FROM
+  employees
+LATERAL VIEW
+  posexplode(subordinates) subView AS pos, sub;

Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1532108&r1=1532107&r2=1532108&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Mon Oct 14 21:40:44
2013
@@ -128,6 +128,7 @@ percentile
 percentile_approx
 pi
 pmod
+posexplode
 positive
 pow
 power
@@ -242,6 +243,7 @@ negative
 ntile
 parse_url_tuple
 percentile
+posexplode
 positive
 regexp_replace
 reverse

Added: hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out?rev=1532108&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out Mon Oct 14 21:40:44
2013
@@ -0,0 +1,42 @@
+PREHOOK: query: CREATE TABLE employees (
+name STRING,
+salary FLOAT,
+subordinates ARRAY<STRING>,
+deductions MAP<STRING, FLOAT>,
+address STRUCT<street:STRING, city:STRING, state:STRING, zip:INT>)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE employees (
+name STRING,
+salary FLOAT,
+subordinates ARRAY<STRING>,
+deductions MAP<STRING, FLOAT>,
+address STRUCT<street:STRING, city:STRING, state:STRING, zip:INT>)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@employees
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/posexplode_data.txt' INTO TABLE employees
+PREHOOK: type: LOAD
+PREHOOK: Output: default@employees
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/posexplode_data.txt' INTO TABLE employees
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@employees
+PREHOOK: query: SELECT
+  name, pos, sub
+FROM
+  employees
+LATERAL VIEW
+  posexplode(subordinates) subView AS pos, sub
+PREHOOK: type: QUERY
+PREHOOK: Input: default@employees
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+  name, pos, sub
+FROM
+  employees
+LATERAL VIEW
+  posexplode(subordinates) subView AS pos, sub
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@employees
+#### A masked pattern was here ####
+John Doe	0	Mary Smith
+John Doe	1	Todd Jones
+Mary Smith	0	Jeremy King



Mime
View raw message