Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C3D98200C46 for ; Wed, 29 Mar 2017 18:41:46 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id C23E4160B5D; Wed, 29 Mar 2017 16:41:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6EC54160B8A for ; Wed, 29 Mar 2017 18:41:45 +0200 (CEST) Received: (qmail 95856 invoked by uid 500); 29 Mar 2017 16:41:44 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 95845 invoked by uid 99); 29 Mar 2017 16:41:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Mar 2017 16:41:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 705EADFEE9; Wed, 29 Mar 2017 16:41:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hashutosh@apache.org To: commits@hive.apache.org Message-Id: <31f5cb7cad0944ad9fe4047fef62e904@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-15434 : Add UDF to allow interrogation of uniontype values (David Maughan via Ashutosh Chauhan) Date: Wed, 29 Mar 2017 16:41:44 +0000 (UTC) archived-at: Wed, 29 Mar 2017 16:41:47 -0000 Repository: hive Updated Branches: refs/heads/branch-2 48f6e3008 -> 9c6e01e70 HIVE-15434 : Add UDF to allow interrogation of uniontype values (David Maughan via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9c6e01e7 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9c6e01e7 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9c6e01e7 Branch: refs/heads/branch-2 Commit: 9c6e01e707854714bee047b8513ca8e1ad7f4e58 Parents: 48f6e30 Author: David Maughan Authored: Fri Dec 16 01:07:00 2016 -0800 Committer: Ashutosh Chauhan Committed: Wed Mar 29 09:38:38 2017 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/exec/FunctionRegistry.java | 1 + .../ql/udf/generic/GenericUDFExtractUnion.java | 272 +++++++++++++++++++ .../udf/generic/TestGenericUDFExtractUnion.java | 175 ++++++++++++ ...UDFExtractUnionObjectInspectorConverter.java | 109 ++++++++ ...estGenericUDFExtractUnionValueConverter.java | 108 ++++++++ .../results/clientpositive/show_functions.q.out | 1 + 6 files changed, 666 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9c6e01e7/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 1fa1a18..9e781dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -457,6 +457,7 @@ public final class FunctionRegistry { system.registerGenericUDF("struct", GenericUDFStruct.class); system.registerGenericUDF("named_struct", GenericUDFNamedStruct.class); system.registerGenericUDF("create_union", GenericUDFUnion.class); + system.registerGenericUDF("extract_union", GenericUDFExtractUnion.class); system.registerGenericUDF("case", GenericUDFCase.class); system.registerGenericUDF("when", GenericUDFWhen.class); http://git-wip-us.apache.org/repos/asf/hive/blob/9c6e01e7/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java new file mode 100644 index 0000000..5e574c8 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantIntObjectInspector; + +import com.google.common.annotations.VisibleForTesting; + +@Description( + name = "extract_union", + value = "_FUNC_(union[, tag])" + " - Recursively explodes unions into structs or simply extracts the given tag.", + extended = " > SELECT _FUNC_({0:\"foo\"}).tag_0 FROM src;\n foo\n" + + " > SELECT _FUNC_({0:\"foo\"}).tag_1 FROM src;\n null\n" + + " > SELECT _FUNC_({0:\"foo\"}, 0) FROM src;\n foo\n" + + " > SELECT _FUNC_({0:\"foo\"}, 1) FROM src;\n null") +public class GenericUDFExtractUnion extends GenericUDF { + + private static final int ALL_TAGS = -1; + + private final ObjectInspectorConverter objectInspectorConverter; + private final ValueConverter valueConverter; + + private int tag = ALL_TAGS; + private UnionObjectInspector unionOI; + private ObjectInspector sourceOI; + + public GenericUDFExtractUnion() { + this(new ObjectInspectorConverter(), new ValueConverter()); + } + + @VisibleForTesting + GenericUDFExtractUnion(ObjectInspectorConverter objectInspectorConverter, ValueConverter valueConverter) { + this.objectInspectorConverter = objectInspectorConverter; + this.valueConverter = valueConverter; + } + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length == 1) { + sourceOI = arguments[0]; + return objectInspectorConverter.convert(sourceOI); + } + + if (arguments.length == 2 && (arguments[0] instanceof UnionObjectInspector) + && (arguments[1] instanceof WritableConstantIntObjectInspector)) { + tag = ((WritableConstantIntObjectInspector) arguments[1]).getWritableConstantValue().get(); + unionOI = (UnionObjectInspector) arguments[0]; + List fieldOIs = ((UnionObjectInspector) arguments[0]).getObjectInspectors(); + if (tag < 0 || tag >= fieldOIs.size()) { + throw new UDFArgumentException( + "int constant must be a valid union tag for " + unionOI.getTypeName() + ". Expected 0-" + + (fieldOIs.size() - 1) + " got: " + tag); + } + return fieldOIs.get(tag); + } + + String argumentTypes = "nothing"; + if (arguments.length > 0) { + List typeNames = new ArrayList<>(); + for (ObjectInspector oi : arguments) { + typeNames.add(oi.getTypeName()); + } + argumentTypes = typeNames.toString(); + } + throw new UDFArgumentException( + "Unsupported arguments. Expected a type containing a union or a union and an int constant, got: " + + argumentTypes); + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object value = arguments[0].get(); + if (tag == ALL_TAGS) { + return valueConverter.convert(value, sourceOI); + } else if (tag == unionOI.getTag(value)) { + return unionOI.getField(value); + } else { + return null; + } + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("extract_union("); + for (int i = 0; i < children.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(children[i]); + } + sb.append(')'); + return sb.toString(); + } + + static class ObjectInspectorConverter { + + private static final String TAG_FIELD_PREFIX = "tag_"; + + ObjectInspector convert(ObjectInspector inspector) { + AtomicBoolean foundUnion = new AtomicBoolean(false); + ObjectInspector result = convert(inspector, foundUnion); + if (!foundUnion.get()) { + throw new IllegalArgumentException("No unions found in " + inspector.getTypeName()); + } + return result; + } + + private ObjectInspector convert(ObjectInspector inspector, AtomicBoolean foundUnion) { + Category category = inspector.getCategory(); + switch (category) { + case PRIMITIVE: + return inspector; + case LIST: + return convertList(inspector, foundUnion); + case MAP: + return convertMap(inspector, foundUnion); + case STRUCT: + return convertStruct(inspector, foundUnion); + case UNION: + foundUnion.set(true); + return convertUnion(inspector, foundUnion); + default: + throw new IllegalStateException("Unknown category: " + category); + } + } + + private ObjectInspector convertList(ObjectInspector inspector, AtomicBoolean foundUnion) { + ListObjectInspector listOI = (ListObjectInspector) inspector; + ObjectInspector elementOI = convert(listOI.getListElementObjectInspector(), foundUnion); + return ObjectInspectorFactory.getStandardListObjectInspector(elementOI); + } + + private ObjectInspector convertMap(ObjectInspector inspector, AtomicBoolean foundUnion) { + MapObjectInspector mapOI = (MapObjectInspector) inspector; + ObjectInspector keyOI = convert(mapOI.getMapKeyObjectInspector(), foundUnion); + ObjectInspector valueOI = convert(mapOI.getMapValueObjectInspector(), foundUnion); + return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI); + } + + private ObjectInspector convertStruct(ObjectInspector inspector, AtomicBoolean foundUnion) { + StructObjectInspector structOI = (StructObjectInspector) inspector; + List fields = structOI.getAllStructFieldRefs(); + List names = new ArrayList<>(fields.size()); + List inspectors = new ArrayList<>(fields.size()); + for (StructField field : fields) { + names.add(field.getFieldName()); + inspectors.add(convert(field.getFieldObjectInspector(), foundUnion)); + } + return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); + } + + private ObjectInspector convertUnion(ObjectInspector inspector, AtomicBoolean foundUnion) { + UnionObjectInspector unionOI = (UnionObjectInspector) inspector; + List fieldOIs = unionOI.getObjectInspectors(); + int tags = fieldOIs.size(); + List names = new ArrayList<>(tags); + List inspectors = new ArrayList<>(tags); + for (int i = 0; i < tags; i++) { + names.add(TAG_FIELD_PREFIX + i); + inspectors.add(convert(fieldOIs.get(i), foundUnion)); + } + return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); + } + + } + + static class ValueConverter { + + Object convert(Object value, ObjectInspector inspector) { + Category category = inspector.getCategory(); + switch (category) { + case PRIMITIVE: + return value; + case LIST: + return convertList(value, inspector); + case MAP: + return convertMap(value, inspector); + case STRUCT: + return convertStruct(value, inspector); + case UNION: + return convertUnion(value, inspector); + default: + throw new IllegalStateException("Unknown category: " + category); + } + } + + private Object convertList(Object list, ObjectInspector inspector) { + SettableListObjectInspector listOI = (SettableListObjectInspector) inspector; + int size = listOI.getListLength(list); + Object result = listOI.create(size); + for (int i = 0; i < size; i++) { + listOI.set(result, i, convert(listOI.getListElement(list, i), listOI.getListElementObjectInspector())); + } + return result; + } + + private Object convertMap(Object map, ObjectInspector inspector) { + SettableMapObjectInspector mapOI = (SettableMapObjectInspector) inspector; + Object result = mapOI.create(); + for (Object key : mapOI.getMap(map).keySet()) { + Object value = mapOI.getMapValueElement(map, key); + mapOI.put( + result, + convert(key, mapOI.getMapKeyObjectInspector()), + convert(value, mapOI.getMapValueObjectInspector())); + } + return result; + } + + private Object convertStruct(Object struct, ObjectInspector inspector) { + SettableStructObjectInspector structOI = (SettableStructObjectInspector) inspector; + Object result = structOI.create(); + for (StructField field : structOI.getAllStructFieldRefs()) { + Object value = structOI.getStructFieldData(struct, field); + structOI.setStructFieldData(result, field, convert(value, field.getFieldObjectInspector())); + } + return result; + } + + private Object convertUnion(Object union, ObjectInspector inspector) { + UnionObjectInspector unionOI = (UnionObjectInspector) inspector; + List childOIs = unionOI.getObjectInspectors(); + byte tag = unionOI.getTag(union); + Object value = unionOI.getField(union); + List result = new ArrayList<>(childOIs.size()); + for (int i = 0; i < childOIs.size(); i++) { + if (i == tag) { + result.add(convert(value, childOIs.get(i))); + } else { + result.add(null); + } + } + return result; + } + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/9c6e01e7/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnion.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnion.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnion.java new file mode 100644 index 0000000..c47fe80 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnion.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFExtractUnion.ObjectInspectorConverter; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFExtractUnion.ValueConverter; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantIntObjectInspector; +import org.apache.hadoop.io.IntWritable; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import com.google.common.collect.ImmutableList; + +@RunWith(MockitoJUnitRunner.class) +public class TestGenericUDFExtractUnion { + + @Mock + private ObjectInspectorConverter objectInspectorConverter; + @Mock + private ValueConverter valueConverter; + @Mock + private UnionObjectInspector unionOI; + @Mock + private WritableConstantIntObjectInspector tagOI; + @Mock + private ObjectInspector initialiseResult; + @Mock + private DeferredObject deferredObject; + + private final Object value = new Object(); + private final Object converted = new Object(); + + private GenericUDF underTest; + + @Before + public void before() { + underTest = new GenericUDFExtractUnion(objectInspectorConverter, valueConverter); + } + + @Test + public void evaluate_SingleArgumentOnly() throws HiveException { + when(deferredObject.get()).thenReturn(value); + when(valueConverter.convert(value, unionOI)).thenReturn(converted); + + underTest.initialize(new ObjectInspector[] { unionOI }); + Object result = underTest.evaluate(new DeferredObject[] { deferredObject }); + + assertThat(result, is(converted)); + } + + @Test + public void evaluate_UnionAndTagArguments_MatchesTag() throws HiveException { + when(unionOI.getObjectInspectors()).thenReturn( + ImmutableList. of( + PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableLongObjectInspector)); + when(tagOI.getWritableConstantValue()).thenReturn(new IntWritable(0)); + when(deferredObject.get()).thenReturn(value); + when(unionOI.getTag(value)).thenReturn((byte) 0); + when(unionOI.getField(value)).thenReturn("foo"); + + underTest.initialize(new ObjectInspector[] { unionOI, tagOI }); + Object result = underTest.evaluate(new DeferredObject[] { deferredObject }); + + assertThat(result, is((Object) "foo")); + } + + @Test + public void evaluate_UnionAndTagArguments_NotMatchesTag() throws HiveException { + when(unionOI.getObjectInspectors()).thenReturn( + ImmutableList. of( + PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableLongObjectInspector)); + when(tagOI.getWritableConstantValue()).thenReturn(new IntWritable(0)); + when(deferredObject.get()).thenReturn(value); + when(unionOI.getTag(value)).thenReturn((byte) 1); + + underTest.initialize(new ObjectInspector[] { unionOI, tagOI }); + Object result = underTest.evaluate(new DeferredObject[] { deferredObject }); + + assertThat(result, is(nullValue())); + } + + @Test + public void initialize_SingleArgumentOnly() throws UDFArgumentException { + when(objectInspectorConverter.convert(unionOI)).thenReturn(initialiseResult); + + ObjectInspector result = underTest.initialize(new ObjectInspector[] { unionOI }); + + assertThat(result, is(initialiseResult)); + } + + @Test + public void initialize_UnionAndTagArguments() throws UDFArgumentException { + when(unionOI.getObjectInspectors()).thenReturn( + ImmutableList. of( + PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableLongObjectInspector)); + when(tagOI.getWritableConstantValue()).thenReturn(new IntWritable(0)); + + ObjectInspector result = underTest.initialize(new ObjectInspector[] { unionOI, tagOI }); + + assertThat(result, is((ObjectInspector) PrimitiveObjectInspectorFactory.writableStringObjectInspector)); + } + + @Test(expected = UDFArgumentException.class) + public void initialize_NegativeTagThrowsException() throws UDFArgumentException { + when(tagOI.getWritableConstantValue()).thenReturn(new IntWritable(-1)); + + underTest.initialize(new ObjectInspector[] { unionOI, tagOI }); + } + + @Test(expected = UDFArgumentException.class) + public void initialize_OutOfBoundsTagThrowsException() throws UDFArgumentException { + when(unionOI.getObjectInspectors()).thenReturn( + ImmutableList. of( + PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableLongObjectInspector)); + when(tagOI.getWritableConstantValue()).thenReturn(new IntWritable(2)); + + underTest.initialize(new ObjectInspector[] { unionOI, tagOI }); + } + + @Test(expected = UDFArgumentException.class) + public void initialize_TagNotAnIntThrowsException() throws UDFArgumentException { + underTest.initialize(new ObjectInspector[] { unionOI, PrimitiveObjectInspectorFactory.writableIntObjectInspector }); + } + + @Test(expected = UDFArgumentException.class) + public void initialize_UnionNotAUnionThrowsException() throws UDFArgumentException { + underTest.initialize(new ObjectInspector[] { PrimitiveObjectInspectorFactory.writableIntObjectInspector, tagOI }); + } + + @Test(expected = UDFArgumentException.class) + public void initialize_NoArgumentsThrowsException() throws UDFArgumentException { + underTest.initialize(new ObjectInspector[] {}); + } + + @Test(expected = UDFArgumentException.class) + public void initialize_TooManyArgumentsThrowsException() throws UDFArgumentException { + underTest.initialize( + new ObjectInspector[] { unionOI, tagOI, PrimitiveObjectInspectorFactory.writableIntObjectInspector }); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/9c6e01e7/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionObjectInspectorConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionObjectInspectorConverter.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionObjectInspectorConverter.java new file mode 100644 index 0000000..33897d4 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionObjectInspectorConverter.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFExtractUnion.ObjectInspectorConverter; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Test; + +public class TestGenericUDFExtractUnionObjectInspectorConverter { + + private final ObjectInspector unionObjectInspector = ObjectInspectorFactory.getStandardUnionObjectInspector( + Arrays. asList( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector)); + + private final ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector( + Arrays.asList("tag_0", "tag_1"), + Arrays. asList( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector)); + + private final ObjectInspectorConverter underTest = new ObjectInspectorConverter(); + + @Test(expected = IllegalArgumentException.class) + public void convertValue_NoUnionFound() { + ObjectInspector inspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + + underTest.convert(inspector); + } + + @Test + public void convertList() { + ObjectInspector inspector = ObjectInspectorFactory.getStandardListObjectInspector(unionObjectInspector); + + ObjectInspector result = underTest.convert(inspector); + + assertThat( + result.getTypeName(), + is(ObjectInspectorFactory.getStandardListObjectInspector(structObjectInspector).getTypeName())); + } + + @Test + public void convertMap() { + ObjectInspector inspector = ObjectInspectorFactory + .getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, unionObjectInspector); + + ObjectInspector result = underTest.convert(inspector); + + assertThat(result.getTypeName(), is( + ObjectInspectorFactory + .getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + structObjectInspector) + .getTypeName())); + } + + @Test + public void convertStruct() { + List names = Arrays.asList("foo"); + ObjectInspector inspector = ObjectInspectorFactory + .getStandardStructObjectInspector(names, Arrays. asList(unionObjectInspector)); + + ObjectInspector result = underTest.convert(inspector); + + assertThat(result.getTypeName(), is( + ObjectInspectorFactory + .getStandardStructObjectInspector(names, Arrays. asList(structObjectInspector)) + .getTypeName())); + } + + @Test + public void convertUnion() { + ObjectInspector result = underTest.convert(unionObjectInspector); + + assertThat(result, is(structObjectInspector)); + + assertThat(result.getTypeName(), is(structObjectInspector.getTypeName())); + } + + @Test(expected = IllegalArgumentException.class) + public void noChildUnions() { + underTest.convert(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/9c6e01e7/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionValueConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionValueConverter.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionValueConverter.java new file mode 100644 index 0000000..2ec54ee --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFExtractUnionValueConverter.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertThat; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFExtractUnion.ValueConverter; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Test; + +public class TestGenericUDFExtractUnionValueConverter { + + private final StandardUnionObjectInspector unionObjectInspector = ObjectInspectorFactory + .getStandardUnionObjectInspector( + Arrays. asList( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector)); + + private final Object union = new StandardUnion((byte) 0, "foo"); + + private final ValueConverter underTest = new ValueConverter(); + + @Test + public void convertValue() { + ObjectInspector inspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + Object value = "foo"; + + Object result = underTest.convert(value, inspector); + + assertThat(result, is((Object) "foo")); + } + + @Test + public void convertList() { + ObjectInspector inspector = ObjectInspectorFactory.getStandardListObjectInspector(unionObjectInspector); + Object value = Arrays.asList(union); + + @SuppressWarnings("unchecked") + List result = (List) underTest.convert(value, inspector); + + assertThat(result.size(), is(1)); + assertThat(result.get(0), is((Object) Arrays.asList("foo", null))); + } + + @Test + public void convertMap() { + ObjectInspector inspector = ObjectInspectorFactory + .getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, unionObjectInspector); + Object value = Collections.singletonMap("bar", union); + + @SuppressWarnings("unchecked") + Map result = (Map) underTest.convert(value, inspector); + + assertThat(result.size(), is(1)); + assertThat(result.get("bar"), is((Object) Arrays.asList("foo", null))); + } + + @Test + public void convertStruct() { + ObjectInspector inspector = ObjectInspectorFactory + .getStandardStructObjectInspector(Arrays.asList("foo"), Arrays. asList(unionObjectInspector)); + Object value = Arrays.asList(union); + + @SuppressWarnings("unchecked") + List result = (List) underTest.convert(value, inspector); + + assertThat(result.size(), is(1)); + assertThat(result.get(0), is((Object) Arrays.asList("foo", null))); + } + + @Test + public void convertUnion() { + @SuppressWarnings("unchecked") + List result = (List) underTest.convert(union, unionObjectInspector); + + assertThat(result.size(), is(2)); + assertThat(result.get(0), is((Object) "foo")); + assertThat(result.get(1), is(nullValue())); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/9c6e01e7/ql/src/test/results/clientpositive/show_functions.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out index cf0cdb0..68e248a 100644 --- a/ql/src/test/results/clientpositive/show_functions.q.out +++ b/ql/src/test/results/clientpositive/show_functions.q.out @@ -87,6 +87,7 @@ ewah_bitmap_empty ewah_bitmap_or exp explode +extract_union factorial field find_in_set