hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject svn commit: r1209384 - in /hive/trunk: ./ bin/ builtins/ builtins/src/ builtins/src/org/ builtins/src/org/apache/ builtins/src/org/apache/hive/ builtins/src/org/apache/hive/builtins/ builtins/test/ eclipse-templates/ pdk/scripts/ ql/src/java/org/apache...
Date Fri, 02 Dec 2011 08:14:14 GMT
Author: cws
Date: Fri Dec  2 08:14:10 2011
New Revision: 1209384

URL: http://svn.apache.org/viewvc?rev=1209384&view=rev
Log:
HIVE-2523. Add a new builtins subproject (John Sichi via cws)

Added:
    hive/trunk/builtins/
    hive/trunk/builtins/build-plugin.xml
    hive/trunk/builtins/build.xml
    hive/trunk/builtins/ivy.xml
    hive/trunk/builtins/src/
    hive/trunk/builtins/src/org/
    hive/trunk/builtins/src/org/apache/
    hive/trunk/builtins/src/org/apache/hive/
    hive/trunk/builtins/src/org/apache/hive/builtins/
    hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java
    hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java
    hive/trunk/builtins/test/
    hive/trunk/builtins/test/cleanup.sql
    hive/trunk/builtins/test/iris.txt
    hive/trunk/builtins/test/onerow.txt
    hive/trunk/builtins/test/setup.sql
Modified:
    hive/trunk/bin/hive
    hive/trunk/build.xml
    hive/trunk/eclipse-templates/.classpath
    hive/trunk/pdk/scripts/build-plugin.xml
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out

Modified: hive/trunk/bin/hive
URL: http://svn.apache.org/viewvc/hive/trunk/bin/hive?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/bin/hive (original)
+++ hive/trunk/bin/hive Fri Dec  2 08:14:10 2011
@@ -74,6 +74,11 @@ if [ ! -f ${HIVE_LIB}/hive-exec-*.jar ];
   exit 1;
 fi
 
+if [ ! -f ${HIVE_LIB}/hive-builtins-*.jar ]; then
+  echo "Missing Hive Builtins Jar: ${HIVE_LIB}/hive-builtins-*.jar"
+  exit 1;
+fi
+
 if [ ! -f ${HIVE_LIB}/hive-metastore-*.jar ]; then
   echo "Missing Hive MetaStore Jar"
   exit 2;

Modified: hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build.xml?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/build.xml (original)
+++ hive/trunk/build.xml Fri Dec  2 08:14:10 2011
@@ -139,7 +139,7 @@
       <subant target="@{target}">
         <property name="build.dir.hive" location="${build.dir.hive}"/>
         <property name="is-offline" value="${is-offline}"/>
-        <filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,ant/build.xml,pdk/build.xml"/>
+        <filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,ant/build.xml,pdk/build.xml,builtins/build.xml"/>
       </subant>
     </sequential>
   </macrodef>
@@ -150,7 +150,7 @@
       <subant target="@{target}">
         <property name="build.dir.hive" location="${build.dir.hive}"/>
         <property name="is-offline" value="${is-offline}"/>
-        <filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,pdk/build.xml"/>
+        <filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,contrib/build.xml,service/build.xml,cli/build.xml,jdbc/build.xml,hwi/build.xml,hbase-handler/build.xml,pdk/build.xml,builtins/build.xml"/>
       </subant>
     </sequential>
   </macrodef>
@@ -519,6 +519,13 @@
     <symlink overwrite="true" link="${target.lib.dir}/libthrift.jar" resource="libthrift-${libthrift.version}.jar"/>
     <symlink overwrite="true" link="${target.lib.dir}/libfb303.jar" resource="libfb303-${libfb303.version}.jar"/>
     <symlink overwrite="true" link="${target.lib.dir}/hive_contrib.jar" resource="hive-contrib-${version}.jar"/>
+    <!-- special case because builtins compilation depends on packaging
+         up everything else first -->
+    <ant antfile="${hive.root}/builtins/build.xml" target="package"
+         inheritAll="false" />
+    <copy todir="${target.lib.dir}" preservelastmodified="true" flatten="true">
+      <fileset file="${build.dir.hive}/builtins/hive-builtins-${version}.jar"/>
+    </copy>
   </target>
 
 
@@ -665,6 +672,7 @@
       <packageset dir="${build.dir.hive}/ql/gen/antlr/gen-java"/>
       <packageset dir="shims/src/common/java"/>
       <packageset dir="pdk/src/java"/>
+      <packageset dir="builtins/src"/>
 
       <link href="${javadoc.link.java}"/>
 
@@ -902,6 +910,8 @@
           todir="${mvn.jar.dir}" />
     <copy file="${build.dir.hive}/pdk/hive-pdk-${version}.jar"
           todir="${mvn.jar.dir}" />
+    <copy file="${build.dir.hive}/pdk/hive-builtins-${version}.jar"
+          todir="${mvn.jar.dir}" />
 
     <!-- copy over maven pom files created using the make-pom target and rename to maven
convention -->
     <copy file="${build.dir.hive}/anttasks/pom.xml"
@@ -930,6 +940,8 @@
           tofile="${mvn.pom.dir}/hive-shims-${version}.pom" />
     <copy file="${build.dir.hive}/pdk/pom.xml"
           tofile="${mvn.pom.dir}/hive-pdk-${version}.pom" />
+    <copy file="${build.dir.hive}/builtins/pom.xml"
+          tofile="${mvn.pom.dir}/hive-builtins-${version}.pom" />
 
     <!-- copy over licence -->
     <copy file="${hive.root}/LICENSE" todir="${mvn.license.dir}" />
@@ -1019,6 +1031,9 @@
       <param name="hive.project" value="pdk" />
     </antcall>
     <antcall target="maven-publish-artifact">
+      <param name="hive.project" value="builtins" />
+    </antcall>
+    <antcall target="maven-publish-artifact">
       <param name="hive.project" value="jdbc" />
     </antcall>
     <antcall target="maven-publish-artifact">
@@ -1138,6 +1153,14 @@
         output.file="${mvn.pom.dir}/hive-pdk-${version}.pom.asc"
         gpg.passphrase="${gpg.passphrase}"/>
     <sign-artifact
+        input.file="${mvn.jar.dir}/hive-builtins-${version}.jar"
+        output.file="${mvn.jar.dir}/hive-builtins-${version}.jar.asc"
+        gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact
+        input.file="${mvn.pom.dir}/hive-builtins-${version}.pom"
+        output.file="${mvn.pom.dir}/hive-builtins-${version}.pom.asc"
+        gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact
         input.file="${mvn.jar.dir}/hive-serde-${version}.jar"
         output.file="${mvn.jar.dir}/hive-serde-${version}.jar.asc"
         gpg.passphrase="${gpg.passphrase}"/>

Added: hive/trunk/builtins/build-plugin.xml
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/build-plugin.xml?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/build-plugin.xml (added)
+++ hive/trunk/builtins/build-plugin.xml Fri Dec  2 08:14:10 2011
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="builtins-plugin" default="jar">
+  <dirname property="builtins.dir" file="${ant.file.builtins}"/>
+  <property name="hive.root" location="${builtins.dir}/.."/>
+  <property file="${hive.root}/build.properties"/>
+  <property name="plugin.libname" value="hive-builtins"/>
+  <property name="plugin.title" value="Hive Builtin Function Library"/>
+  <property name="plugin.version" value="${version}"/>
+  <property name="plugin.vendor" value="Apache Software Foundation"/>
+  <property name="function.sql.prefix" value=""/>
+  <property name="build.dir" location="${hive.root}/build/builtins"/>
+  <property name="install.dir" location="${hive.root}/build/dist"/>
+  <import file="${hive.root}/pdk/scripts/build-plugin.xml"/>
+</project>

Added: hive/trunk/builtins/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/build.xml?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/build.xml (added)
+++ hive/trunk/builtins/build.xml Fri Dec  2 08:14:10 2011
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="builtins" default="jar">
+
+  <property name="src.dir"  location="${basedir}/src"/>
+  <import file="../build-common.xml"/>
+
+  <target name="compile" depends="init, setup">
+    <echo message="Project: ${ant.project.name}"/>
+    <!-- defer compilation until package phase -->
+  </target>
+
+  <target name="jar" depends="init">
+    <echo message="Project: ${ant.project.name}"/>
+    <!-- defer compilation until package phase -->
+  </target>
+
+  <target name="package">
+    <echo message="Project: ${ant.project.name}"/>
+    <ant antfile="build-plugin.xml" target="package" inheritAll="false">
+      <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
+      <property name="hadoop.home" value="${hadoop.root}"/>
+    </ant>
+  </target>
+
+  <target name="test" unless="testcase">
+    <echo message="Project: ${ant.project.name}"/>
+    <ant antfile="build-plugin.xml" target="test" inheritAll="false">
+      <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
+      <property name="hadoop.home" value="${hadoop.root}"/>
+    </ant>
+  </target>
+
+  <target name="clean">
+    <echo message="Project: ${ant.project.name}"/>
+    <ant antfile="build-plugin.xml" target="clean" inheritAll="false">
+      <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
+      <property name="hadoop.home" value="${hadoop.root}"/>
+    </ant>
+  </target>
+</project>

Added: hive/trunk/builtins/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/ivy.xml?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/ivy.xml (added)
+++ hive/trunk/builtins/ivy.xml Fri Dec  2 08:14:10 2011
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<ivy-module version="2.0">
+    <info organisation="${hive.ivy.org}" module="hive-builtins" revision="${version}">
+      <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"
/>
+      <description homepage="http://hive.apache.org">
+        The Apache Hive (TM) data warehouse software facilitates querying and managing large
datasets residing in distributed storage.
+        https://cwiki.apache.org/confluence/display/Hive/Home
+      </description>
+    </info>
+    <dependencies>
+    </dependencies>
+</ivy-module>

Added: hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java (added)
+++ hive/trunk/builtins/src/org/apache/hive/builtins/BuiltinUtils.java Fri Dec  2 08:14:10
2011
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.builtins;
+
+/**
+ * Utility method class supporting builtin functions.  We also
+ * use this class as a well-known name for loading metadata from
+ * the builtins jar.
+ */
+public abstract class BuiltinUtils
+{
+}

Added: hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java (added)
+++ hive/trunk/builtins/src/org/apache/hive/builtins/UDAFUnionMap.java Fri Dec  2 08:14:10
2011
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.builtins;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hive.pdk.HivePdkUnitTest;
+import org.apache.hive.pdk.HivePdkUnitTests;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+
+/**
+ * Aggregate all maps into a single map. If there are multiple values for
+ * the same key, result can contain any of those values.
+ * Because the mappers must keep all of the data in memory, if your data is
+ * non-trivially large you should set hive.map.aggr=false to ensure that
+ * UNION_MAP is only executed in the reduce phase.
+ */
+@HivePdkUnitTests(
+  setup = "",
+  cleanup = "",
+  cases = {
+    @HivePdkUnitTest(
+      query = "SELECT size(UNION_MAP(MAP(sepal_width, sepal_length))) "
+      +"FROM iris",
+      result = "23")
+  })
+@Description(
+  name = "union_map",
+  value = "_FUNC_(col) - aggregate given maps into a single map",
+  extended = "Aggregate maps, returns as a HashMap.")
+public class UDAFUnionMap extends AbstractGenericUDAFResolver {
+  @Override
+  public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException
{
+
+    // Next two validation calls are dependent on HIVE-2524, so
+    // leave them commented out for now.
+    //
+    // new LengthEquals(1).check(parameters.length);
+    // new IsMap().check(parameters[0], 0);
+
+    return new Evaluator();
+  }
+
+  public static class State implements AggregationBuffer {
+    HashMap<Object, Object> map = new HashMap<Object, Object>();
+  }
+
+  public static class Evaluator extends GenericUDAFEvaluator {
+    ObjectInspector inputOI;
+    MapObjectInspector internalMergeOI;
+
+    @Override
+    public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException
{
+      super.init(m, parameters);
+      if (m == Mode.COMPLETE || m == Mode.PARTIAL1) {
+        inputOI = (MapObjectInspector) parameters[0];
+      } else {
+        internalMergeOI = (MapObjectInspector) parameters[0];
+      }
+      return ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
+    }
+
+    @Override
+    public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+      return new State();
+    }
+
+    @Override
+    public void iterate(AggregationBuffer agg, Object[] input) throws HiveException {
+      if (input[0] != null) {
+        State state = (State) agg;
+        state.map.putAll((Map<?,?>)ObjectInspectorUtils.copyToStandardObject(input[0],
inputOI));
+      }
+    }
+
+    @Override
+    public void merge(AggregationBuffer agg, Object partial) throws HiveException {
+      if (partial != null) {
+        State state = (State) agg;
+        Map<?,?> pset = (Map<?,?>)ObjectInspectorUtils.copyToStandardObject(partial,
internalMergeOI);
+        state.map.putAll(pset);
+      }
+    }
+
+    @Override
+    public void reset(AggregationBuffer agg) throws HiveException {
+      ((State) agg).map.clear();
+    }
+
+    @Override
+    public Object terminate(AggregationBuffer agg) throws HiveException {
+      return ((State) agg).map;
+    }
+
+    @Override
+    public Object terminatePartial(AggregationBuffer agg) throws HiveException {
+      return ((State) agg).map;
+    }
+  }
+}

Added: hive/trunk/builtins/test/cleanup.sql
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/test/cleanup.sql?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/cleanup.sql (added)
+++ hive/trunk/builtins/test/cleanup.sql Fri Dec  2 08:14:10 2011
@@ -0,0 +1,2 @@
+drop table if exists onerow;
+drop table if exists iris;

Added: hive/trunk/builtins/test/iris.txt
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/test/iris.txt?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/iris.txt (added)
+++ hive/trunk/builtins/test/iris.txt Fri Dec  2 08:14:10 2011
@@ -0,0 +1,150 @@
+6	2.9	4.5	1.5	versicolor
+6.9	3.1	5.1	2.3	virginica
+5.4	3.4	1.5	0.4	setosa
+5.5	3.5	1.3	0.2	setosa
+4.6	3.1	1.5	0.2	setosa
+4.4	2.9	1.4	0.2	setosa
+5.9	3	4.2	1.5	versicolor
+5.1	3.8	1.6	0.2	setosa
+5	3.6	1.4	0.2	setosa
+6.7	3.1	5.6	2.4	virginica
+4.9	3.6	1.4	0.1	setosa
+4.9	2.5	4.5	1.7	virginica
+5.8	2.8	5.1	2.4	virginica
+4.5	2.3	1.3	0.3	setosa
+6.7	3	5.2	2.3	virginica
+6	3	4.8	1.8	virginica
+6.4	3.2	4.5	1.5	versicolor
+7.7	3.8	6.7	2.2	virginica
+5.7	2.6	3.5	1	versicolor
+5.7	2.9	4.2	1.3	versicolor
+5.4	3.9	1.7	0.4	setosa
+5	3.5	1.3	0.3	setosa
+6	2.2	5	1.5	virginica
+6.6	2.9	4.6	1.3	versicolor
+5.9	3	5.1	1.8	virginica
+4.7	3.2	1.6	0.2	setosa
+5	3.4	1.5	0.2	setosa
+6.9	3.2	5.7	2.3	virginica
+6.5	3	5.8	2.2	virginica
+5.5	4.2	1.4	0.2	setosa
+7.1	3	5.9	2.1	virginica
+6.2	2.9	4.3	1.3	versicolor
+5.7	4.4	1.5	0.4	setosa
+4.9	2.4	3.3	1	versicolor
+5.4	3.9	1.3	0.4	setosa
+6.4	2.8	5.6	2.1	virginica
+6.9	3.1	4.9	1.5	versicolor
+5.9	3.2	4.8	1.8	versicolor
+6.1	3	4.9	1.8	virginica
+5.6	3	4.5	1.5	versicolor
+5.3	3.7	1.5	0.2	setosa
+6.4	2.9	4.3	1.3	versicolor
+6.6	3	4.4	1.4	versicolor
+6.4	3.1	5.5	1.8	virginica
+6.3	2.5	4.9	1.5	versicolor
+6.7	3.3	5.7	2.5	virginica
+6.4	2.7	5.3	1.9	virginica
+6.5	3	5.5	1.8	virginica
+6.7	3.1	4.4	1.4	versicolor
+4.9	3	1.4	0.2	setosa
+6.2	2.8	4.8	1.8	virginica
+7.6	3	6.6	2.1	virginica
+5.2	3.5	1.5	0.2	setosa
+7.7	2.6	6.9	2.3	virginica
+6.5	3.2	5.1	2	virginica
+5.8	2.6	4	1.2	versicolor
+6.3	2.9	5.6	1.8	virginica
+5.8	4	1.2	0.2	setosa
+6.5	2.8	4.6	1.5	versicolor
+5	3.2	1.2	0.2	setosa
+7.4	2.8	6.1	1.9	virginica
+7	3.2	4.7	1.4	versicolor
+4.8	3.4	1.6	0.2	setosa
+6.4	2.8	5.6	2.2	virginica
+5.4	3.7	1.5	0.2	setosa
+5.7	3.8	1.7	0.3	setosa
+5.2	2.7	3.9	1.4	versicolor
+6.3	3.3	6	2.5	virginica
+5.1	2.5	3	1.1	versicolor
+7.7	2.8	6.7	2	virginica
+5.5	2.4	3.7	1	versicolor
+7.3	2.9	6.3	1.8	virginica
+4.3	3	1.1	0.1	setosa
+5.6	2.7	4.2	1.3	versicolor
+7.9	3.8	6.4	2	virginica
+5.6	2.8	4.9	2	virginica
+5	3	1.6	0.2	setosa
+6.1	3	4.6	1.4	versicolor
+5.1	3.5	1.4	0.2	setosa
+5.1	3.4	1.5	0.2	setosa
+4.8	3.4	1.9	0.2	setosa
+4.6	3.4	1.4	0.3	setosa
+6.5	3	5.2	2	virginica
+5.1	3.8	1.9	0.4	setosa
+5.6	2.9	3.6	1.3	versicolor
+6.7	3	5	1.7	versicolor
+7.2	3.6	6.1	2.5	virginica
+4.9	3.1	1.5	0.1	setosa
+6.2	3.4	5.4	2.3	virginica
+5.8	2.7	5.1	1.9	virginica
+5.6	2.5	3.9	1.1	versicolor
+6.3	2.5	5	1.9	virginica
+4.6	3.2	1.4	0.2	setosa
+5.8	2.7	5.1	1.9	virginica
+5.5	2.3	4	1.3	versicolor
+6.7	3.3	5.7	2.1	virginica
+6	2.2	4	1	versicolor
+5.1	3.7	1.5	0.4	setosa
+5	3.5	1.6	0.6	setosa
+5.8	2.7	4.1	1	versicolor
+5.7	2.8	4.1	1.3	versicolor
+6	3.4	4.5	1.6	versicolor
+6.1	2.9	4.7	1.4	versicolor
+5.2	3.4	1.4	0.2	setosa
+5.6	3	4.1	1.3	versicolor
+6.8	2.8	4.8	1.4	versicolor
+5.8	2.7	3.9	1.2	versicolor
+4.8	3.1	1.6	0.2	setosa
+5.1	3.5	1.4	0.3	setosa
+5	3.3	1.4	0.2	setosa
+7.2	3.2	6	1.8	virginica
+6.7	3.1	4.7	1.5	versicolor
+6.8	3.2	5.9	2.3	virginica
+4.6	3.6	1	0.2	setosa
+5.4	3	4.5	1.5	versicolor
+6.4	3.2	5.3	2.3	virginica
+4.8	3	1.4	0.1	setosa
+5.5	2.5	4	1.3	versicolor
+6.9	3.1	5.4	2.1	virginica
+6.8	3	5.5	2.1	virginica
+4.8	3	1.4	0.3	setosa
+6.3	2.7	4.9	1.8	virginica
+6	2.7	5.1	1.6	versicolor
+6.1	2.6	5.6	1.4	virginica
+7.2	3	5.8	1.6	virginica
+5	2.3	3.3	1	versicolor
+6.3	3.3	4.7	1.6	versicolor
+6.3	2.8	5.1	1.5	virginica
+4.7	3.2	1.3	0.2	setosa
+6.3	3.4	5.6	2.4	virginica
+5.7	2.8	4.5	1.3	versicolor
+5.7	3	4.2	1.2	versicolor
+4.9	3.1	1.5	0.2	setosa
+5	2	3.5	1	versicolor
+7.7	3	6.1	2.3	virginica
+5.7	2.5	5	2	virginica
+5.5	2.4	3.8	1.1	versicolor
+5.1	3.3	1.7	0.5	setosa
+6.3	2.3	4.4	1.3	versicolor
+4.4	3	1.3	0.2	setosa
+6.1	2.8	4	1.3	versicolor
+6.1	2.8	4.7	1.2	versicolor
+5.5	2.6	4.4	1.2	versicolor
+5.1	3.8	1.5	0.3	setosa
+4.4	3.2	1.3	0.2	setosa
+5	3.4	1.6	0.4	setosa
+6.7	2.5	5.8	1.8	virginica
+5.2	4.1	1.5	0.1	setosa
+6.2	2.2	4.5	1.5	versicolor
+5.4	3.4	1.7	0.2	setosa

Added: hive/trunk/builtins/test/onerow.txt
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/test/onerow.txt?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/onerow.txt (added)
+++ hive/trunk/builtins/test/onerow.txt Fri Dec  2 08:14:10 2011
@@ -0,0 +1 @@
+plugh

Added: hive/trunk/builtins/test/setup.sql
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/test/setup.sql?rev=1209384&view=auto
==============================================================================
--- hive/trunk/builtins/test/setup.sql (added)
+++ hive/trunk/builtins/test/setup.sql Fri Dec  2 08:14:10 2011
@@ -0,0 +1,10 @@
+create table onerow(s string);
+load data local inpath '${env:HIVE_PLUGIN_ROOT_DIR}/test/onerow.txt'
+overwrite into table onerow;
+create table iris(
+sepal_length string, sepal_width string,
+petal_length string, petal_width string,
+species string)
+row format delimited fields terminated by '\t' stored as textfile;
+load data local inpath '${env:HIVE_PLUGIN_ROOT_DIR}/test/iris.txt'
+overwrite into table iris;

Modified: hive/trunk/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/trunk/eclipse-templates/.classpath?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/eclipse-templates/.classpath (original)
+++ hive/trunk/eclipse-templates/.classpath Fri Dec  2 08:14:10 2011
@@ -69,6 +69,7 @@
   <classpathentry excluding="queries/|results/|templates/" kind="src" path="hbase-handler/src/test"/>
   <classpathentry kind="src" path="pdk/src/java"/>
   <classpathentry kind="src" path="pdk/test-plugin/src"/>
+  <classpathentry kind="src" path="builtins/src/java"/>
   <classpathentry kind="src" path="cli/src/test"/>
   <classpathentry kind="output" path="build/eclipse-classes"/>
 </classpath>

Modified: hive/trunk/pdk/scripts/build-plugin.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pdk/scripts/build-plugin.xml?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/pdk/scripts/build-plugin.xml (original)
+++ hive/trunk/pdk/scripts/build-plugin.xml Fri Dec  2 08:14:10 2011
@@ -59,10 +59,14 @@
     </javac>
   </target>
 
-  <target name="jar" depends="compile">
+  <target name="jar" depends="extract-functions">
     <jar
       jarfile="${build.dir}/${plugin.jar.basename}"
       basedir="${build.classes}">
+      <metainf dir="${build.metadata}">
+        <include name="class-info.xml"/>
+        <include name="class-registration.sql"/>
+      </metainf>
       <manifest>
         <attribute name="Implementation-Title" value="${plugin.title}"/>
         <attribute name="Implementation-Version" value="${plugin.version}"/>
@@ -111,7 +115,7 @@
     <delete quiet="true" dir="${build.dir}"/>
   </target>
 
-  <target name="package" depends="jar,extract-functions"/>
+  <target name="package" depends="jar"/>
 
   <target name="test" depends="package" unless="testcase">
     <junit fork="true" printsummary="on" haltonfailure="true"

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Fri Dec  2
08:14:10 2011
@@ -18,7 +18,9 @@
 
 package org.apache.hadoop.hive.ql.exec;
 
+import java.io.InputStream;
 import java.lang.reflect.Method;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -30,6 +32,9 @@ import java.util.TreeSet;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -212,8 +217,13 @@ import org.apache.hadoop.hive.serde2.typ
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.ReflectionUtils;
 
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 
 /**
  * FunctionRegistry.
@@ -1130,6 +1140,82 @@ public final class FunctionRegistry {
     return UDFOPPositive.class == udfClass;
   }
 
+  /**
+   * Registers the appropriate kind of temporary function based on a class's
+   * type.
+   *
+   * @param functionName name under which to register function
+   *
+   * @param udfClass class implementing UD[A|T]F
+   *
+   * @return true if udfClass's type was recognized (so registration
+   * succeeded); false otherwise
+   */
+  public static boolean registerTemporaryFunction(
+    String functionName, Class<?> udfClass) {
+
+    if (UDF.class.isAssignableFrom(udfClass)) {
+      FunctionRegistry.registerTemporaryUDF(
+        functionName, (Class<? extends UDF>) udfClass, false);
+    } else if (GenericUDF.class.isAssignableFrom(udfClass)) {
+      FunctionRegistry.registerTemporaryGenericUDF(
+        functionName, (Class<? extends GenericUDF>) udfClass);
+    } else if (GenericUDTF.class.isAssignableFrom(udfClass)) {
+      FunctionRegistry.registerTemporaryGenericUDTF(
+        functionName, (Class<? extends GenericUDTF>) udfClass);
+    } else if (UDAF.class.isAssignableFrom(udfClass)) {
+      FunctionRegistry.registerTemporaryUDAF(
+        functionName, (Class<? extends UDAF>) udfClass);
+    } else if (GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
+      FunctionRegistry.registerTemporaryGenericUDAF(
+        functionName, (GenericUDAFResolver)
+        ReflectionUtils.newInstance(udfClass, null));
+    } else {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Registers Hive functions from a plugin jar, using metadata from
+   * the jar's META-INF/class-info.xml.
+   *
+   * @param jarLocation URL for reading jar file
+   *
+   * @param classLoader classloader to use for loading function classes
+   */
+  public static void registerFunctionsFromPluginJar(
+    URL jarLocation,
+    ClassLoader classLoader) throws Exception {
+
+    URL url = new URL("jar:" + jarLocation + "!/META-INF/class-info.xml");
+    InputStream inputStream = null;
+    try {
+      inputStream = url.openStream();
+      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilder docBuilder = dbf.newDocumentBuilder();
+      Document doc = docBuilder.parse(inputStream);
+      Element root = doc.getDocumentElement();
+      if (!root.getTagName().equals("ClassList")) {
+        return;
+      }
+      NodeList children = root.getElementsByTagName("Class");
+      for (int i = 0; i < children.getLength(); ++i) {
+        Element child = (Element) children.item(i);
+        String javaName = child.getAttribute("javaname");
+        String sqlName = child.getAttribute("sqlname");
+        Class<?> udfClass = Class.forName(javaName, true, classLoader);
+        boolean registered = registerTemporaryFunction(sqlName, udfClass);
+        if (!registered) {
+          throw new RuntimeException(
+            "Class " + udfClass + " is not a Hive function implementation");
+        }
+      }
+    } finally {
+      IOUtils.closeStream(inputStream);
+    }
+  }
+
   private FunctionRegistry() {
     // prevent instantiation
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Fri Dec  2 08:14:10
2011
@@ -73,32 +73,15 @@ public class FunctionTask extends Task<F
   private int createFunction(CreateFunctionDesc createFunctionDesc) {
     try {
       Class<?> udfClass = getUdfClass(createFunctionDesc);
-      if (UDF.class.isAssignableFrom(udfClass)) {
-        FunctionRegistry.registerTemporaryUDF(createFunctionDesc
-            .getFunctionName(), (Class<? extends UDF>) udfClass, false);
-        return 0;
-      } else if (GenericUDF.class.isAssignableFrom(udfClass)) {
-        FunctionRegistry.registerTemporaryGenericUDF(createFunctionDesc
-            .getFunctionName(), (Class<? extends GenericUDF>) udfClass);
-        return 0;
-      } else if (GenericUDTF.class.isAssignableFrom(udfClass)) {
-        FunctionRegistry.registerTemporaryGenericUDTF(createFunctionDesc
-            .getFunctionName(), (Class<? extends GenericUDTF>) udfClass);
-        return 0;
-      } else if (UDAF.class.isAssignableFrom(udfClass)) {
-        FunctionRegistry.registerTemporaryUDAF(createFunctionDesc
-            .getFunctionName(), (Class<? extends UDAF>) udfClass);
-        return 0;
-      } else if (GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
-        FunctionRegistry.registerTemporaryGenericUDAF(createFunctionDesc
-            .getFunctionName(), (GenericUDAFResolver) ReflectionUtils
-            .newInstance(udfClass, null));
+      boolean registered = FunctionRegistry.registerTemporaryFunction(
+        createFunctionDesc.getFunctionName(),
+        udfClass);
+      if (registered) {
         return 0;
       }
       console.printError("FAILED: Class " + createFunctionDesc.getClassName()
           + " does not implement UDF, GenericUDF, or UDAF");
       return 1;
-
     } catch (ClassNotFoundException e) {
       console.printError("FAILED: Class " + createFunctionDesc.getClassName() + " not found");
       LOG.info("create function: " + StringUtils.stringifyException(e));

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Dec  2 08:14:10
2011
@@ -2275,4 +2275,8 @@ public final class Utilities {
 
     return sb.toString();
   }
+
+  public static Class getBuiltinUtilsClass() throws ClassNotFoundException {
+    return Class.forName("org.apache.hive.builtins.BuiltinUtils");
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Fri Dec  2
08:14:10 2011
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.net.URI;
+import java.net.URL;
 import java.util.Calendar;
 import java.util.GregorianCalendar;
 import java.util.HashMap;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.MapRedStats;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.history.HiveHistory;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -174,6 +176,19 @@ public class SessionState {
     this.conf = conf;
     isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT);
     ls = new LineageState();
+
+    // Register the Hive builtins jar and all of its functions
+    try {
+      Class<?> pluginClass = Utilities.getBuiltinUtilsClass();
+      URL jarLocation = pluginClass.getProtectionDomain().getCodeSource()
+        .getLocation();
+      add_builtin_resource(
+        ResourceType.JAR, jarLocation.toString());
+      FunctionRegistry.registerFunctionsFromPluginJar(
+        jarLocation, pluginClass.getClassLoader());
+    } catch (Exception ex) {
+      throw new RuntimeException("Failed to load Hive builtin functions", ex);
+    }
   }
 
   public void setCmd(String cmdString) {
@@ -509,8 +524,8 @@ public class SessionState {
     return null;
   }
 
-  private final HashMap<ResourceType, HashSet<String>> resource_map =
-    new HashMap<ResourceType, HashSet<String>>();
+  private final HashMap<ResourceType, Set<String>> resource_map =
+    new HashMap<ResourceType, Set<String>>();
 
   public String add_resource(ResourceType t, String value) {
     // By default don't convert to unix
@@ -525,23 +540,34 @@ public class SessionState {
       return null;
     }
 
-    if (resource_map.get(t) == null) {
-      resource_map.put(t, new HashSet<String>());
-    }
+    Set<String> resourceMap = getResourceMap(t);
 
     String fnlVal = value;
     if (t.hook != null) {
-      fnlVal = t.hook.preHook(resource_map.get(t), value);
+      fnlVal = t.hook.preHook(resourceMap, value);
       if (fnlVal == null) {
         return fnlVal;
       }
     }
     getConsole().printInfo("Added resource: " + fnlVal);
-    resource_map.get(t).add(fnlVal);
+    resourceMap.add(fnlVal);
 
     return fnlVal;
   }
 
+  public void add_builtin_resource(ResourceType t, String value) {
+    getResourceMap(t).add(value);
+  }
+
+  private Set<String> getResourceMap(ResourceType t) {
+    Set<String> result = resource_map.get(t);
+    if (result == null) {
+      result = new HashSet<String>();
+      resource_map.put(t, result);
+    }
+    return result;
+  }
+
   /**
    * Returns the list of filesystem schemas as regex which
    * are permissible for download as a resource.

Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1209384&r1=1209383&r2=1209384&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Fri Dec  2 08:14:10
2011
@@ -155,6 +155,7 @@ to_utc_timestamp
 trim
 ucase
 unhex
+union_map
 unix_timestamp
 upper
 var_pop



Mime
View raw message