drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sudhe...@apache.org
Subject [3/3] drill git commit: DRILL-4459: Resolve SchemaChangeException while querying hive json table
Date Wed, 20 Apr 2016 05:32:35 GMT
DRILL-4459: Resolve SchemaChangeException while querying hive json table

- Replace drill var16char to varchar datatype for hive string datatype
- Change testGenericUDF() and testUDF() to use VarChar instead of Var16Char
- Add unit test for hive GET_JSON_OBJECT UDF

closes #431


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3b056db0
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3b056db0
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3b056db0

Branch: refs/heads/master
Commit: 3b056db0f504d50fe11a6028b1a633ec74d478d2
Parents: 852b01a
Author: Vitalii Diravka <vitalii.diravka@gmail.com>
Authored: Thu Mar 10 14:52:28 2016 +0000
Committer: Sudheesh Katkam <skatkam@maprtech.com>
Committed: Tue Apr 19 21:58:16 2016 -0700

----------------------------------------------------------------------
 .../core/src/main/codegen/data/HiveTypes.tdd    |  4 +-
 .../templates/ObjectInspectorHelper.java        | 26 +++++++---
 .../codegen/templates/ObjectInspectors.java     | 20 ++++----
 .../drill/exec/expr/fn/HiveFuncHolder.java      |  5 +-
 .../exec/expr/fn/HiveFunctionRegistry.java      | 12 ++---
 .../apache/drill/exec/fn/hive/TestHiveUDFs.java | 51 ++++++++++----------
 .../drill/exec/fn/hive/TestInbuiltHiveUDFs.java | 14 ++++++
 .../exec/hive/TestInfoSchemaOnHiveStorage.java  |  1 +
 .../exec/store/hive/HiveTestDataGenerator.java  |  6 +++
 .../resources/functions/hive/GenericUDF.json    |  8 +--
 .../src/test/resources/functions/hive/UDF.json  |  2 +-
 .../core/src/test/resources/simple.json         | 19 ++++++++
 12 files changed, 110 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
index 73752a3..e13dc36 100644
--- a/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
+++ b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
@@ -80,10 +80,10 @@
       needOIForDrillType: false
     },
     {
-      hiveType: "STRING",l
+      hiveType: "STRING",
       hiveOI: "StringObjectInspector",
       javaType: "",
-      drillType: "Var16Char",
+      drillType: "VarChar",
       needOIForDrillType: true
     },
     {

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
index da83c40..d068868 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
@@ -40,30 +40,40 @@ import java.lang.UnsupportedOperationException;
 import java.sql.Timestamp;
 import java.util.HashMap;
 import java.util.Map;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.ArrayListMultimap;
 
 public class ObjectInspectorHelper {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ObjectInspectorHelper.class);
 
-  private static Map<MinorType, Class> OIMAP_REQUIRED = new HashMap<>();
-  private static Map<MinorType, Class> OIMAP_OPTIONAL = new HashMap<>();
+  private static Multimap<MinorType, Class> OIMAP_REQUIRED = ArrayListMultimap.create();
+  private static Multimap<MinorType, Class> OIMAP_OPTIONAL = ArrayListMultimap.create();
   static {
 <#list drillOI.map as entry>
     <#if entry.needOIForDrillType == true>
-    OIMAP_REQUIRED.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}ObjectInspector.Required.class);
-    OIMAP_OPTIONAL.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}ObjectInspector.Optional.class);
+    OIMAP_REQUIRED.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}${entry.hiveOI}.Required.class);
+    OIMAP_OPTIONAL.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}${entry.hiveOI}.Optional.class);
     </#if>
 </#list>
   }
 
-  public static ObjectInspector getDrillObjectInspector(DataMode mode, MinorType minorType)
{
+  public static ObjectInspector getDrillObjectInspector(DataMode mode, MinorType minorType,
boolean varCharToStringReplacement) {
     try {
       if (mode == DataMode.REQUIRED) {
         if (OIMAP_REQUIRED.containsKey(minorType)) {
-          return (ObjectInspector) OIMAP_REQUIRED.get(minorType).newInstance();
+          if (varCharToStringReplacement && minorType == MinorType.VARCHAR) {
+            return (ObjectInspector) ((Class) OIMAP_REQUIRED.get(minorType).toArray()[1]).newInstance();
+          } else {
+            return (ObjectInspector) ((Class) OIMAP_REQUIRED.get(minorType).toArray()[0]).newInstance();
+          }
         }
       } else if (mode == DataMode.OPTIONAL) {
         if (OIMAP_OPTIONAL.containsKey(minorType)) {
-          return (ObjectInspector) OIMAP_OPTIONAL.get(minorType).newInstance();
+          if (varCharToStringReplacement && minorType == MinorType.VARCHAR) {
+            return (ObjectInspector) ((Class) OIMAP_OPTIONAL.get(minorType).toArray()[1]).newInstance();
+          } else {
+            return (ObjectInspector) ((Class) OIMAP_OPTIONAL.get(minorType).toArray()[0]).newInstance();
+          }
         }
       } else {
         throw new UnsupportedOperationException("Repeated types are not supported as arguement
to Hive UDFs");
@@ -191,7 +201,7 @@ public class ObjectInspectorHelper {
           <#elseif entry.hiveType == "STRING">
             JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data",
               castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)
-                      .invoke("getBytes").arg(DirectExpression.direct("com.google.common.base.Charsets.UTF_16")));
+                      .invoke("getBytes"));
             jc._else().add(returnValueHolder.ref("buffer")
               .invoke("setBytes").arg(JExpr.lit(0)).arg(data));
             jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
index 5c696f9..ffd3a56 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
@@ -19,7 +19,7 @@
 
 <#list drillOI.map as entry>
 <#if entry.needOIForDrillType == true>
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.drillType}ObjectInspector.java"
/>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.drillType}${entry.hiveOI}.java"
/>
 
 <#include "/@includes/license.ftl" />
 
@@ -47,7 +47,7 @@ import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 
-public class Drill${entry.drillType}ObjectInspector {
+public class Drill${entry.drillType}${entry.hiveOI} {
 <#assign seq = ["Required", "Optional"]>
 <#list seq as mode>
 
@@ -56,7 +56,7 @@ public class Drill${entry.drillType}ObjectInspector {
       super(TypeInfoFactory.${entry.hiveType?lower_case}TypeInfo);
     }
 
-<#if entry.drillType == "VarChar">
+<#if entry.drillType == "VarChar" && entry.hiveType == "VARCHAR">
     @Override
     public HiveVarcharWritable getPrimitiveWritableObject(Object o) {
     <#if mode == "Optional">
@@ -85,18 +85,18 @@ public class Drill${entry.drillType}ObjectInspector {
       final String s = StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer);
       return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH);
     }
-<#elseif entry.drillType == "Var16Char">
+<#elseif entry.drillType == "VarChar" && entry.hiveType == "STRING">
     @Override
     public Text getPrimitiveWritableObject(Object o) {
     <#if mode == "Optional">
       if (o == null) {
         return null;
       }
-      final NullableVar16CharHolder h = (NullableVar16CharHolder)o;
+      final NullableVarCharHolder h = (NullableVarCharHolder)o;
     <#else>
-      final Var16CharHolder h = (Var16CharHolder)o;
+      final VarCharHolder h = (VarCharHolder)o;
     </#if>
-      return new Text(StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer));
+      return new Text(StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer));
     }
 
     @Override
@@ -105,11 +105,11 @@ public class Drill${entry.drillType}ObjectInspector {
       if (o == null) {
         return null;
       }
-      final NullableVar16CharHolder h = (NullableVar16CharHolder)o;
+      final NullableVarCharHolder h = (NullableVarCharHolder)o;
     <#else>
-      final Var16CharHolder h = (Var16CharHolder)o;
+      final VarCharHolder h = (VarCharHolder)o;
     </#if>
-      return StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer);
+      return StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer);
     }
 <#elseif entry.drillType == "VarBinary">
     @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
index a6e76b0..0a3cf18 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
@@ -45,6 +45,7 @@ import com.sun.codemodel.JExpr;
 import com.sun.codemodel.JInvocation;
 import com.sun.codemodel.JTryBlock;
 import com.sun.codemodel.JVar;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 
 public class HiveFuncHolder extends AbstractFuncHolder {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class);
@@ -188,7 +189,9 @@ public class HiveFuncHolder extends AbstractFuncHolder {
         oiArray.component(JExpr.lit(i)),
         oih.staticInvoke("getDrillObjectInspector")
           .arg(mode.staticInvoke("valueOf").arg(JExpr.lit(argTypes[i].getMode().getNumber())))
-          .arg(mt.staticInvoke("valueOf").arg(JExpr.lit(argTypes[i].getMinorType().getNumber()))));
+          .arg(mt.staticInvoke("valueOf").arg(JExpr.lit(argTypes[i].getMinorType().getNumber())))
+          .arg((((PrimitiveObjectInspector) returnOI).getPrimitiveCategory() ==
+              PrimitiveObjectInspector.PrimitiveCategory.STRING) ? JExpr.lit(true) : JExpr.lit(false)));
     }
 
     // declare and instantiate DeferredObject array

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
index 5e74f6f..c716e9e 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
@@ -126,23 +126,21 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry{
 
   /**
    * Helper method which resolves the given function call to a Hive UDF. It takes an argument
-   * <i>convertVarCharToVar16Char</i> which tells to implicitly cast input arguments
of type VARCHAR to VAR16CHAR
-   * and search Hive UDF registry using implicitly casted argument types.
+   * <i>varCharToStringReplacement</i> which tells to use hive STRING(true) or
VARCHAR(false) type for drill VARCHAR type
+   * and search Hive UDF registry using this replacement.
    *
    * TODO: This is a rudimentary function resolver. Need to include more implicit casting
such as DECIMAL28 to
    * DECIMAL38 as Hive UDFs can accept only DECIMAL38 type.
    */
-  private HiveFuncHolder resolveFunction(FunctionCall call, boolean convertVarCharToVar16Char)
{
+  private HiveFuncHolder resolveFunction(FunctionCall call, boolean varCharToStringReplacement)
{
     HiveFuncHolder holder;
     MajorType[] argTypes = new MajorType[call.args.size()];
     ObjectInspector[] argOIs = new ObjectInspector[call.args.size()];
     for (int i=0; i<call.args.size(); i++) {
       try {
         argTypes[i] = call.args.get(i).getMajorType();
-        if (convertVarCharToVar16Char && argTypes[i].getMinorType() == MinorType.VARCHAR)
{
-          argTypes[i] = Types.withMode(MinorType.VAR16CHAR, argTypes[i].getMode());
-        }
-        argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMode(),
argTypes[i].getMinorType());
+        argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMode(),
argTypes[i].getMinorType(),
+            varCharToStringReplacement);
       } catch(Exception e) {
         // Hive throws errors if there are unsupported types. Consider there is no hive UDF
supporting the
         // given argument types

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
index 3ce9a6d..2ad8edc 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
@@ -25,11 +25,12 @@ import java.util.List;
 import org.apache.drill.BaseTestQuery;
 import org.apache.drill.exec.record.RecordBatchLoader;
 import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.exec.vector.BigIntVector;
 import org.apache.drill.exec.vector.Float4Vector;
+import org.apache.drill.exec.vector.IntVector;
 import org.apache.drill.exec.vector.NullableFloat8Vector;
-import org.apache.drill.exec.vector.NullableIntVector;
-import org.apache.drill.exec.vector.NullableVar16CharVector;
-import org.apache.drill.exec.vector.Var16CharVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.VarCharVector;
 import org.junit.Test;
 
 import com.google.common.base.Charsets;
@@ -53,42 +54,42 @@ public class TestHiveUDFs extends BaseTestQuery {
         continue;
       }
       // Output columns and types
-      //  1. str1 : Var16Char
-      //  2. upperStr1 : NullableVar16Char
+      //  1. str1 : VarChar
+      //  2. upperStr1 : NullableVarChar
       //  3. concat : NullableVarChar
       //  4. flt1 : Float4
       //  5. format_number : NullableFloat8
-      //  6. nullableStr1 : NullableVar16Char
-      //  7. upperNullableStr1 : NullableVar16Char
-      Var16CharVector str1V = (Var16CharVector) batchLoader.getValueAccessorById(Var16CharVector.class,
0).getValueVector();
-      NullableVar16CharVector upperStr1V = (NullableVar16CharVector) batchLoader.getValueAccessorById(NullableVar16CharVector.class,
1).getValueVector();
-      NullableVar16CharVector concatV = (NullableVar16CharVector) batchLoader.getValueAccessorById(NullableVar16CharVector.class,
2).getValueVector();
+      //  6. nullableStr1 : NullableVarChar
+      //  7. upperNullableStr1 : NullableVarChar
+      VarCharVector str1V = (VarCharVector) batchLoader.getValueAccessorById(VarCharVector.class,
0).getValueVector();
+      NullableVarCharVector upperStr1V = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class,
1).getValueVector();
+      NullableVarCharVector concatV = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class,
2).getValueVector();
       Float4Vector flt1V = (Float4Vector) batchLoader.getValueAccessorById(Float4Vector.class,
3).getValueVector();
-      NullableVar16CharVector format_numberV = (NullableVar16CharVector) batchLoader.getValueAccessorById(NullableVar16CharVector.class,
4).getValueVector();
-      NullableVar16CharVector nullableStr1V = (NullableVar16CharVector) batchLoader.getValueAccessorById(NullableVar16CharVector.class,
5).getValueVector();
-      NullableVar16CharVector upperNullableStr1V = (NullableVar16CharVector) batchLoader.getValueAccessorById(NullableVar16CharVector.class,
6).getValueVector();
+      NullableVarCharVector format_numberV = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class,
4).getValueVector();
+      NullableVarCharVector nullableStr1V = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class,
5).getValueVector();
+      NullableVarCharVector upperNullableStr1V = (NullableVarCharVector) batchLoader.getValueAccessorById(NullableVarCharVector.class,
6).getValueVector();
 
       for (int i=0; i<batchLoader.getRecordCount(); i++) {
-        String in = new String(str1V.getAccessor().get(i), Charsets.UTF_16);
-        String upper = new String(upperStr1V.getAccessor().get(i), Charsets.UTF_16);
+        String in = new String(str1V.getAccessor().get(i), Charsets.UTF_8);
+        String upper = new String(upperStr1V.getAccessor().get(i), Charsets.UTF_8);
         assertTrue(in.toUpperCase().equals(upper));
 
 
-        String concat = new String(concatV.getAccessor().get(i), Charsets.UTF_16);
+        String concat = new String(concatV.getAccessor().get(i), Charsets.UTF_8);
         assertTrue(concat.equals(in+"-"+in));
 
         float flt1 = flt1V.getAccessor().get(i);
-        String format_number = new String(format_numberV.getAccessor().get(i), Charsets.UTF_16);
+        String format_number = new String(format_numberV.getAccessor().get(i), Charsets.UTF_8);
 
 
         String nullableStr1 = null;
         if (!nullableStr1V.getAccessor().isNull(i)) {
-          nullableStr1 = new String(nullableStr1V.getAccessor().get(i), Charsets.UTF_16);
+          nullableStr1 = new String(nullableStr1V.getAccessor().get(i), Charsets.UTF_8);
         }
 
         String upperNullableStr1 = null;
         if (!upperNullableStr1V.getAccessor().isNull(i)) {
-          upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i), Charsets.UTF_16);
+          upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i), Charsets.UTF_8);
         }
 
         assertEquals(nullableStr1 != null, upperNullableStr1 != null);
@@ -125,20 +126,20 @@ public class TestHiveUDFs extends BaseTestQuery {
       }
 
       // Output columns and types
-      // 1. str1 : Var16Char
+      // 1. str1 : VarChar
       // 2. str1Length : Int
       // 3. str1Ascii : Int
       // 4. flt1 : Float4
       // 5. pow : Float8
-      Var16CharVector str1V = (Var16CharVector) batchLoader.getValueAccessorById(Var16CharVector.class,
0).getValueVector();
-      NullableIntVector str1LengthV = (NullableIntVector) batchLoader.getValueAccessorById(NullableIntVector.class,
1).getValueVector();
-      NullableIntVector str1AsciiV = (NullableIntVector) batchLoader.getValueAccessorById(NullableIntVector.class,
2).getValueVector();
+      VarCharVector str1V = (VarCharVector) batchLoader.getValueAccessorById(VarCharVector.class,
0).getValueVector();
+      BigIntVector str1LengthV = (BigIntVector) batchLoader.getValueAccessorById(BigIntVector.class,
1).getValueVector();
+      IntVector str1AsciiV = (IntVector) batchLoader.getValueAccessorById(IntVector.class,
2).getValueVector();
       Float4Vector flt1V = (Float4Vector) batchLoader.getValueAccessorById(Float4Vector.class,
3).getValueVector();
       NullableFloat8Vector powV = (NullableFloat8Vector) batchLoader.getValueAccessorById(NullableFloat8Vector.class,
4).getValueVector();
 
       for (int i=0; i<batchLoader.getRecordCount(); i++) {
-        String str1 = new String(str1V.getAccessor().get(i), Charsets.UTF_16);
-        int str1Length = str1LengthV.getAccessor().get(i);
+        String str1 = new String(str1V.getAccessor().get(i), Charsets.UTF_8);
+        long str1Length = str1LengthV.getAccessor().get(i);
         assertTrue(str1.length() == str1Length);
 
         int str1Ascii = str1AsciiV.getAccessor().get(i);

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
index a126aaa..93c9374 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
@@ -19,9 +19,11 @@ package org.apache.drill.exec.fn.hive;
 
 import com.google.common.collect.Lists;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.drill.TestBuilder;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.exec.hive.HiveTestBase;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
 import org.junit.Test;
 
 import java.util.List;
@@ -70,4 +72,16 @@ public class TestInbuiltHiveUDFs extends HiveTestBase {
         .build()
         .run();
   }
+
+  @Test // DRILL-4459
+  public void testGetJsonObject() throws Exception {
+    testBuilder()
+        .sqlQuery("select convert_from(json, 'json') as json from hive.simple_json " +
+            "where GET_JSON_OBJECT(simple_json.json, '$.employee_id') like 'Emp2'")
+        .ordered()
+        .baselineColumns("json")
+        .baselineValues(TestBuilder.mapOf("employee_id","Emp2","full_name","Kamesh",
+            "first_name","Bh","last_name","Venkata","position","Store"))
+        .go();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
index 03acd22..a8c6e68 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
@@ -42,6 +42,7 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
         .baselineValues("hive.default", "kv_parquet")
         .baselineValues("hive.default", "kv_sh")
         .baselineValues("hive.default", "countstar_parquet")
+        .baselineValues("hive.default", "simple_json")
         .go();
 
     testBuilder()

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 9c2fe1c..7a5b72d 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -24,6 +24,7 @@ import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.Map;
 
+import com.google.common.io.Resources;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.drill.BaseTestQuery;
@@ -504,6 +505,11 @@ public class HiveTestDataGenerator {
     executeQuery(hiveDriver, createTableWithHeaderFooterProperties("skipper.kv_sequencefile_large",
"sequencefile", "1", "1"));
     executeQuery(hiveDriver, "insert into table skipper.kv_sequencefile_large select * from
skipper.kv_text_large");
 
+      // Create a table based on json file
+      executeQuery(hiveDriver, "create table default.simple_json(json string)");
+      final String loadData = String.format("load data local inpath '" +
+          Resources.getResource("simple.json") + "' into table default.simple_json");
+      executeQuery(hiveDriver, loadData);
     ss.close();
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/test/resources/functions/hive/GenericUDF.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/resources/functions/hive/GenericUDF.json b/contrib/storage-hive/core/src/test/resources/functions/hive/GenericUDF.json
index f5ae299..5dc076e 100644
--- a/contrib/storage-hive/core/src/test/resources/functions/hive/GenericUDF.json
+++ b/contrib/storage-hive/core/src/test/resources/functions/hive/GenericUDF.json
@@ -13,10 +13,10 @@
             url: "http://apache.org",
             entries:[
                 {records: 100, types: [
-                   {name: "str1", type: "VAR16CHAR", mode: "REQUIRED"},
-                   {name: "str2", type: "VAR16CHAR", mode: "REQUIRED"},
-                   {name: "str3", type: "VAR16CHAR", mode: "REQUIRED"},
-                   {name: "nullableStr1", type: "VAR16CHAR", mode: "OPTIONAL"},
+                   {name: "str1", type: "VARCHAR", mode: "REQUIRED"},
+                   {name: "str2", type: "VARCHAR", mode: "REQUIRED"},
+                   {name: "str3", type: "VARCHAR", mode: "REQUIRED"},
+                   {name: "nullableStr1", type: "VARCHAR", mode: "OPTIONAL"},
                    {name: "flt1", type: "FLOAT4", mode: "REQUIRED"}
                 ]}
             ]

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/test/resources/functions/hive/UDF.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/resources/functions/hive/UDF.json b/contrib/storage-hive/core/src/test/resources/functions/hive/UDF.json
index c2c7169..fecabec 100644
--- a/contrib/storage-hive/core/src/test/resources/functions/hive/UDF.json
+++ b/contrib/storage-hive/core/src/test/resources/functions/hive/UDF.json
@@ -13,7 +13,7 @@
             url: "http://apache.org",
             entries:[
                 {records: 100, types: [
-                   {name: "str1", type: "VAR16CHAR", mode: "REQUIRED"},
+                   {name: "str1", type: "VARCHAR", mode: "REQUIRED"},
                    {name: "flt1", type: "FLOAT4", mode: "REQUIRED"}
                 ]}
             ]

http://git-wip-us.apache.org/repos/asf/drill/blob/3b056db0/contrib/storage-hive/core/src/test/resources/simple.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/resources/simple.json b/contrib/storage-hive/core/src/test/resources/simple.json
new file mode 100644
index 0000000..0bf69ce
--- /dev/null
+++ b/contrib/storage-hive/core/src/test/resources/simple.json
@@ -0,0 +1,19 @@
+[ {"employee_id":"Emp1101","full_name":"Steve Eurich","first_name":"Steve","last_name":"Eurich","position":"Store
T"},
+  {"employee_id":"Emp1102","full_name":"Mary Pierson","first_name":"Mary","last_name":"Pierson","position":"Store
T"},
+  {"employee_id":"Emp1103","full_name":"Leo Jones","first_name":"Leo","last_name":"Jones","position":"Store
Tem"},
+  {"employee_id":"Emp1104","full_name":"Nancy Beatty","first_name":"Nancy","last_name":"Beatty","position":"Store
T"},
+  {"employee_id":"Emp1105","full_name":"Clara McNight","first_name":"Clara","last_name":"McNight","position":"Store"},
+  {"employee_id":"Emp1106","first_name":"Marcella","last_name":"Isaacs","position_id":17,"position":"Stor"},
+  {"employee_id":"Emp1107","full_name":"Charlotte Yonce","first_name":"Charlotte","last_name":"Yonce","position":"Stor"},
+  {"employee_id":"Emp1108","full_name":"Benjamin Foster","first_name":"Benjamin","last_name":"Foster","position":"Stor"},
+  {"employee_id":"Emp1109","full_name":"John Reed","first_name":"John","last_name":"Reed","position":"Store
Per"},
+  {"employee_id":"Emp1110","full_name":"Lynn Kwiatkowski","first_name":"Lynn","last_name":"Kwiatkowski","position":"St"},
+  {"employee_id":"Emp1111","full_name":"Donald Vann","first_name":"Donald","last_name":"Vann","position":"Store
Per"},
+  {"employee_id":"Emp1112","first_name":"William","last_name":"Smith","position":"St"},
+  {"employee_id":"Emp1113","full_name":"Amy Hensley","first_name":"Amy","last_name":"Hensley","position":"Store
Pe"},
+  {"employee_id":"Emp1114","full_name":"Judy Owens","first_name":"Judy","last_name":"Owens","position":"Store
Per"},
+  {"employee_id":"Emp1115","full_name":"Frederick Castillo","first_name":"Frederick","last_name":"Castillo","position":"S"},
+  {"employee_id":"Emp1116","full_name":"Phil Munoz","first_name":"Phil","last_name":"Munoz","position":"Store
Per"},
+  {"employee_id":"Emp1117","full_name":"Lori Lightfoot","first_name":"Lori","last_name":"Lightfoot","position":"Store"},
+  {"employee_id":"Emp1","full_name":"Kumar","first_name":"Anil","last_name":"B","position":"Store"},
+  {"employee_id":"Emp2","full_name":"Kamesh","first_name":"Bh","last_name":"Venkata","position":"Store"}
]


Mime
View raw message