drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject drill git commit: DRILL-1874: Support casting empty string to null numeric based on configuration parameter
Date Sat, 10 Jan 2015 01:12:57 GMT
Repository: drill
Updated Branches:
  refs/heads/master f82064973 -> 487d98e58


DRILL-1874: Support casting empty string to null numeric based on configuration parameter


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/487d98e5
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/487d98e5
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/487d98e5

Branch: refs/heads/master
Commit: 487d98e58380b888fb11f168459e65a0def9d77f
Parents: f820649
Author: Hsuan-Yi Chu <hsuanyi@usc.edu>
Authored: Thu Dec 18 14:22:13 2014 -0800
Committer: Hsuan-Yi Chu <hsuanyi@usc.edu>
Committed: Fri Jan 9 15:37:58 2015 -0800

----------------------------------------------------------------------
 .../common/expression/fn/CastFunctions.java     | 84 ++++++++++++++++++-
 exec/java-exec/src/main/codegen/data/Casts.tdd  | 22 ++++-
 .../templates/CastVarCharToNullableNumeric.java | 85 +++++++++++++++++++
 .../templates/Decimal/CastVarCharDecimal.java   | 65 +++++++++++++--
 .../org/apache/drill/exec/ExecConstants.java    |  3 +
 .../exec/expr/ExpressionTreeMaterializer.java   |  5 +-
 .../expr/fn/FunctionImplementationRegistry.java | 22 ++++-
 .../drill/exec/server/DrillbitContext.java      |  2 +-
 .../server/options/SystemOptionManager.java     |  1 +
 .../exec/store/text/DrillTextRecordReader.java  | 17 +++-
 .../exec/fn/impl/TestCastEmptyStrings.java      | 87 ++++++++++++++++++++
 .../src/test/resources/emptyStrings.csv         |  3 +
 12 files changed, 380 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
index 61d149d..f1a3b37 100644
--- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
+++ b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
@@ -18,13 +18,18 @@
 package org.apache.drill.common.expression.fn;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
-
+import java.util.Set;
+import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 
 public class CastFunctions {
 
   private static Map<MinorType, String> TYPE2FUNC = new HashMap<>();
+  private static Set<String> CAST_FUNC_REPLACEMENT_NEEDED = new HashSet<>();
 // The cast fucntions which are needed to be replaced (if "drill.exec.functions.cast_empty_string_to_null""
is set as true)
+  private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE = new HashMap<>();
// Map from the replaced functions to the new ones (for non-nullable varchar)
+  private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NULLABLE = new HashMap<>();
   // Map from the replaced functions to the new ones (for nullable varchar)
 
   static {
     TYPE2FUNC.put(MinorType.BIGINT, "castBIGINT");
@@ -49,8 +54,40 @@ public class CastFunctions {
     TYPE2FUNC.put(MinorType.DECIMAL28DENSE, "castDECIMAL28DENSE");
     TYPE2FUNC.put(MinorType.DECIMAL38SPARSE, "castDECIMAL38SPARSE");
     TYPE2FUNC.put(MinorType.DECIMAL38DENSE, "castDECIMAL38DENSE");
+
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.INT));
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.BIGINT));
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT4));
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT8));
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL9));
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL18));
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE));
+    CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE));
+
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringVarCharToNullableINT");
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringVarCharToNullableBIGINT");
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringVarCharToNullableFLOAT4");
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringVarCharToNullableFLOAT8");
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringVarCharToNullableDECIMAL9");
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringVarCharToNullableDECIMAL18");
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE),
"castEmptyStringVarCharToNullableDECIMAL28SPARSE");
+    CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE),
"castEmptyStringVarCharToNullableDECIMAL38SPARSE");
+
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringNullableVarCharToNullableINT");
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringNullableVarCharToNullableBIGINT");
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringNullableVarCharToNullableFLOAT4");
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringNullableVarCharToNullableFLOAT8");
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringNullableVarCharToNullableDECIMAL9");
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringNullableVarCharToNullableDECIMAL18");
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL28SPARSE");
+    CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL38SPARSE");
   }
 
+  /**
+  * Given the target type, get the appropriate cast function
+  * @param targetMinorType the target data type
+  * @return
+  */
   public static String getCastFunc(MinorType targetMinorType) {
     String func = TYPE2FUNC.get(targetMinorType);
     if (func != null) {
@@ -61,4 +98,49 @@ public class CastFunctions {
       String.format("cast function for type %s is not defined", targetMinorType.name()));
   }
 
+  /**
+  * Get a replacing cast function for the original function, based on the specified data
mode
+  * @param originalCastFunction original cast function
+  * @param dataMode data mode of the input data
+  * @return
+  */
+  public static String getReplacingCastFunction(String originalCastFunction, org.apache.drill.common.types.TypeProtos.DataMode
dataMode) {
+    if(dataMode == TypeProtos.DataMode.OPTIONAL) {
+      return getReplacingCastFunctionFromNullable(originalCastFunction);
+    }
+
+    if(dataMode == TypeProtos.DataMode.REQUIRED) {
+      return getReplacingCastFunctionFromNonNullable(originalCastFunction);
+    }
+
+    throw new RuntimeException(
+       String.format("replacing cast function for datatype %s is not defined", dataMode));
+  }
+
+  /**
+  * Check if a replacing cast function is available for the the original function
+  * @param originalfunction original cast function
+  * @return
+  */
+  public static boolean isReplacementNeeded(MinorType inputType, String originalfunction)
{
+    return inputType == MinorType.VARCHAR && CAST_FUNC_REPLACEMENT_NEEDED.contains(originalfunction);
+  }
+
+  private static String getReplacingCastFunctionFromNonNullable(String originalCastFunction)
{
+    if(CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.containsKey(originalCastFunction)) {
+      return CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.get(originalCastFunction);
+    }
+
+    throw new RuntimeException(
+      String.format("replacing cast function for %s is not defined", originalCastFunction));
+  }
+
+  private static String getReplacingCastFunctionFromNullable(String originalCastFunction)
{
+    if(CAST_FUNC_REPLACEMENT_FROM_NULLABLE.containsKey(originalCastFunction)) {
+      return CAST_FUNC_REPLACEMENT_FROM_NULLABLE.get(originalCastFunction);
+    }
+
+    throw new RuntimeException(
+      String.format("replacing cast function for %s is not defined", originalCastFunction));
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/codegen/data/Casts.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd
index 36c90c9..6fe7404 100644
--- a/exec/java-exec/src/main/codegen/data/Casts.tdd
+++ b/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -172,7 +172,27 @@
 
     {from: "Decimal38Sparse", to: "Decimal28Sparse", major: "DownwardDecimalComplexDecimalComplex",
arraySize: "5"},
     {from: "Decimal38Sparse", to: "Decimal18", major: "DownwardDecimalComplexDecimalSimple",
javatype: "long"},
-    {from: "Decimal38Sparse", to: "Decimal9", major: "DownwardDecimalComplexDecimalSimple",
javatype: "int"}
+    {from: "Decimal38Sparse", to: "Decimal9", major: "DownwardDecimalComplexDecimalSimple",
javatype: "int"},
+
+    {from: "VarChar", to: "Int", major: "EmptyString", javaType:"Integer", primeType:"int"},
+    {from: "VarChar", to: "BigInt", major: "EmptyString", javaType: "Long", primeType: "long"},
+    {from: "VarChar", to: "Float4", major: "EmptyString", javaType:"Float", parse:"Float"},
+    {from: "VarChar", to: "Float8", major: "EmptyString", javaType:"Double", parse:"Double"},
+
+    {from: "VarChar", to: "Decimal9", major: "EmptyStringVarCharDecimalSimple", javatype:
"int"},
+    {from: "VarChar", to: "Decimal18", major: "EmptyStringVarCharDecimalSimple", javatype:
"long"},
+    {from: "VarChar", to: "Decimal28Sparse", major: "EmptyStringVarCharDecimalComplex", arraySize:
"5"},
+    {from: "VarChar", to: "Decimal38Sparse", major: "EmptyStringVarCharDecimalComplex", arraySize:
"6"},
+
+    {from: "NullableVarChar", to: "Int", major: "EmptyString", javaType:"Integer", primeType:"int"},
+    {from: "NullableVarChar", to: "BigInt", major: "EmptyString", javaType: "Long", primeType:
"long"},
+    {from: "NullableVarChar", to: "Float4", major: "EmptyString", javaType:"Float", parse:"Float"},
+    {from: "NullableVarChar", to: "Float8", major: "EmptyString", javaType:"Double", parse:"Double"},
+
+    {from: "NullableVarChar", to: "Decimal9", major: "EmptyStringVarCharDecimalSimple", javatype:
"int"},
+    {from: "NullableVarChar", to: "Decimal18", major: "EmptyStringVarCharDecimalSimple",
javatype: "long"},
+    {from: "NullableVarChar", to: "Decimal28Sparse", major: "EmptyStringVarCharDecimalComplex",
arraySize: "5"},
+    {from: "NullableVarChar", to: "Decimal38Sparse", major: "EmptyStringVarCharDecimalComplex",
arraySize: "6"},
 
   ]
 } 

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/codegen/templates/CastVarCharToNullableNumeric.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharToNullableNumeric.java b/exec/java-exec/src/main/codegen/templates/CastVarCharToNullableNumeric.java
new file mode 100644
index 0000000..e417f26
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/CastVarCharToNullableNumeric.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+<#macro doError>
+{
+  byte[] buf = new byte[in.end - in.start];
+  in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
+  throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
+}
+</#macro>
+
+<#list cast.types as type>
+<#if type.major == "EmptyString">
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/CastEmptyString${type.from}ToNullable${type.to}.java"
/>
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl.gcast;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
+
+@SuppressWarnings("unused")
+@FunctionTemplate(name = "castEmptyString${type.from}ToNullable${type.to?upper_case}", scope
= FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.INTERNAL)
+public class CastEmptyString${type.from}ToNullable${type.to} implements DrillSimpleFunc{
+
+  @Param ${type.from}Holder in;
+  @Output Nullable${type.to}Holder out;
+
+  public void setup(RecordBatch incoming) {}
+
+  public void eval() {
+  <#if type.to == "Float4" || type.to == "Float8">
+    if(<#if type.from == "NullableVarChar"> in.isSet == 0 || </#if> in.end ==
in.start) {
+      out.isSet = 0;
+    } else{
+      out.isSet = 1;
+      byte[]buf=new byte[in.end-in.start];
+      in.buffer.getBytes(in.start,buf,0,in.end-in.start);
+      out.value=${type.javaType}.parse${type.parse}(new String(buf,com.google.common.base.Charsets.UTF_8));
+    }
+  <#elseif type.to=="Int">
+    if(<#if type.from == "NullableVarChar"> in.isSet == 0 || </#if> in.end ==
in.start) {
+      out.isSet = 0;
+    } else {
+      out.isSet = 1;
+      out.value = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.varCharToInt(in.start,
in.end, in.buffer);
+    }
+  <#elseif type.to == "BigInt">
+    if(<#if type.from == "NullableVarChar"> in.isSet == 0 || </#if> in.end ==
in.start) {
+      out.isSet = 0;
+    } else {
+      out.isSet = 1;
+      out.value = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.varCharToLong(in.start,
in.end, in.buffer);
+    }
+  </#if>
+  }
+}
+
+</#if> <#-- type.major -->
+</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
index acf2f1d..960368a 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
@@ -19,8 +19,13 @@
 
 <#list cast.types as type>
 
-<#if type.major == "VarCharDecimalSimple">  <#-- Cast function template for conversion
from VarChar to Decimal9, Decimal18 -->
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}${type.to}.java"
/>
+<#if type.major == "VarCharDecimalSimple" || type.major == "EmptyStringVarCharDecimalSimple">
 <#-- Cast function template for conversion from VarChar to Decimal9, Decimal18 -->
+
+<#if type.major == "VarCharDecimalSimple">
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}${type.to}.java"/>
+<#elseif type.major == "EmptyStringVarCharDecimalSimple">
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/CastEmptyString${type.from}ToNullable${type.to}.java"/>
+</#if>
 
 <#include "/@includes/license.ftl" />
 
@@ -44,18 +49,35 @@ import io.netty.buffer.DrillBuf;
 import java.nio.ByteBuffer;
 
 @SuppressWarnings("unused")
-@FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.DECIMAL_CAST,
nulls=NullHandling.NULL_IF_NULL)
+<#if type.major == "VarCharDecimalSimple">
+@FunctionTemplate(name ="cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.DECIMAL_CAST,
nulls=NullHandling.NULL_IF_NULL)
 public class Cast${type.from}${type.to} implements DrillSimpleFunc {
-
+<#elseif type.major == "EmptyStringVarCharDecimalSimple">
+@FunctionTemplate(name ="castEmptyString${type.from}ToNullable${type.to?upper_case}", scope
= FunctionTemplate.FunctionScope.DECIMAL_CAST, nulls=NullHandling.INTERNAL)
+public class CastEmptyString${type.from}ToNullable${type.to} implements DrillSimpleFunc {
+</#if>
     @Param ${type.from}Holder in;
     @Param BigIntHolder precision;
     @Param BigIntHolder scale;
+
+    <#if type.major == "VarCharDecimalSimple">
     @Output ${type.to}Holder out;
+    <#elseif type.major == "EmptyStringVarCharDecimalSimple">
+    @Output Nullable${type.to}Holder out;
+    </#if>
 
     public void setup(RecordBatch incoming) {
     }
 
     public void eval() {
+        <#if type.major == "EmptyStringVarCharDecimalSimple">
+        // Check if the input is null or empty string
+        if(<#if type.from == "NullableVarChar"> in.isSet == 0 || </#if> in.end
== in.start) {
+            out.isSet = 0;
+            return;
+        }
+        out.isSet = 1;
+        </#if>
 
         // Assign the scale and precision
         out.scale = (int) scale.value;
@@ -64,10 +86,13 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
         int readIndex = in.start;
         int endIndex  = in.end;
 
+        <#if type.major == "VarCharDecimalSimple">
         // Check if its an empty string
         if (endIndex - readIndex == 0) {
             throw new org.apache.drill.common.exceptions.DrillRuntimeException("Empty String,
cannot cast to Decimal");
         }
+        </#if>
+
         // Starting position of fractional part
         int scaleIndex = -1;
         // true if we have a negative sign at the beginning
@@ -167,8 +192,13 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
     }
 }
 
-<#elseif type.major == "VarCharDecimalComplex">  <#-- Cast function template for
conversion from VarChar to Decimal28, Decimal38 -->
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}${type.to}.java"
/>
+<#elseif type.major == "VarCharDecimalComplex" || type.major == "EmptyStringVarCharDecimalComplex">
 <#-- Cast function template for conversion from VarChar to Decimal28, Decimal38 -->
+
+<#if type.major == "VarCharDecimalComplex">
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}${type.to}.java"/>
+<#elseif type.major == "EmptyStringVarCharDecimalComplex">
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/CastEmptyString${type.from}ToNullable${type.to}.java"/>
+</#if>
 
 <#include "/@includes/license.ftl" />
 
@@ -191,14 +221,23 @@ import io.netty.buffer.ByteBuf;
 import java.nio.ByteBuffer;
 
 @SuppressWarnings("unused")
+<#if type.major == "VarCharDecimalComplex">
 @FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.DECIMAL_CAST,
nulls=NullHandling.NULL_IF_NULL)
 public class Cast${type.from}${type.to} implements DrillSimpleFunc {
-
+<#elseif type.major == "EmptyStringVarCharDecimalComplex">
+@FunctionTemplate(name = "castEmptyString${type.from}ToNullable${type.to?upper_case}", scope
= FunctionTemplate.FunctionScope.DECIMAL_CAST, nulls=NullHandling.INTERNAL)
+public class CastEmptyString${type.from}ToNullable${type.to} implements DrillSimpleFunc {
+</#if>
     @Param ${type.from}Holder in;
     @Inject DrillBuf buffer;
     @Param BigIntHolder precision;
     @Param BigIntHolder scale;
+
+    <#if type.major == "VarCharDecimalComplex">
     @Output ${type.to}Holder out;
+    <#elseif type.major == "EmptyStringVarCharDecimalComplex">
+    @Output Nullable${type.to}Holder out;
+    </#if>
 
     public void setup(RecordBatch incoming) {
         int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize);
@@ -206,6 +245,15 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
     }
 
     public void eval() {
+        <#if type.major == "EmptyStringVarCharDecimalComplex">
+        // Check if the input is null or empty string
+        if(<#if type.from == "NullableVarChar"> in.isSet == 0 || </#if> in.end
== in.start) {
+            out.isSet = 0;
+            return;
+        }
+        out.isSet = 1;
+        </#if>
+
         out.buffer = buffer;
         out.start  = 0;
 
@@ -241,9 +289,12 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
             scaleIndex = readIndex; // Fractional part starts at the first position
         }
 
+        <#if type.major == "VarCharDecimalComplex">
+        // Check if its an empty string
         if (in.end - readIndex == 0) {
             throw new org.apache.drill.common.exceptions.DrillRuntimeException("Empty String,
cannot cast to Decimal");
         }
+        </#if>
 
         // Store start index for the second pass
         startIndex = readIndex;

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index b390cd5..35b86d1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -104,6 +104,9 @@ public interface ExecConstants {
   public static final String SLICE_TARGET = "planner.slice_target";
   public static final OptionValidator SLICE_TARGET_OPTION = new PositiveLongValidator(SLICE_TARGET,
Long.MAX_VALUE, 100000);
 
+  public static final String CAST_TO_NULLABLE_NUMERIC = "drill.exec.functions.cast_empty_string_to_null";
+  public static final OptionValidator CAST_TO_NULLABLE_NUMERIC_OPTION = new BooleanValidator(CAST_TO_NULLABLE_NUMERIC,
false);
+
   /**
    * HashTable runtime settings
    */

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 0f40958..8557765 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -520,8 +520,8 @@ public class ExpressionTreeMaterializer {
       // if the cast is pointless, remove it.
       LogicalExpression input = e.getInput().accept(this,  value);
 
-      MajorType newMajor = e.getMajorType();
-      MinorType newMinor = input.getMajorType().getMinorType();
+      MajorType newMajor = e.getMajorType(); // Output type
+      MinorType newMinor = input.getMajorType().getMinorType(); // Input type
 
       if (castEqual(e.getPosition(), input.getMajorType(), newMajor)) {
         return input; // don't do pointless cast.
@@ -618,5 +618,4 @@ public class ExpressionTreeMaterializer {
       }
     }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index 055fdfa..25dcbbc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -24,6 +24,8 @@ import java.util.Set;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.fn.CastFunctions;
+import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.util.PathScanner;
 import org.apache.drill.exec.ExecConstants;
@@ -31,12 +33,14 @@ import org.apache.drill.exec.planner.sql.DrillOperatorTable;
 import org.apache.drill.exec.resolver.FunctionResolver;
 
 import com.google.common.collect.Lists;
+import org.apache.drill.exec.server.options.OptionManager;
 
 public class FunctionImplementationRegistry {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class);
 
   private DrillFunctionRegistry drillFuncRegistry;
   private List<PluggableFunctionRegistry> pluggableFuncRegistries = Lists.newArrayList();
+  private OptionManager optionManager = null;
 
   public FunctionImplementationRegistry(DrillConfig config){
     drillFuncRegistry = new DrillFunctionRegistry(config);
@@ -65,6 +69,11 @@ public class FunctionImplementationRegistry {
     }
   }
 
+  public FunctionImplementationRegistry(DrillConfig config, OptionManager optionManager)
{
+    this(config);
+    this.optionManager = optionManager;
+  }
+
   /**
    * Register functions in given operator table.
    * @param operatorTable
@@ -87,7 +96,18 @@ public class FunctionImplementationRegistry {
    * @return
    */
   public DrillFuncHolder findDrillFunction(FunctionResolver functionResolver, FunctionCall
functionCall) {
-    return functionResolver.getBestMatch(drillFuncRegistry.getMethods(functionCall.getName()),
functionCall);
+    return functionResolver.getBestMatch(drillFuncRegistry.getMethods(functionReplacement(functionCall)),
functionCall);
+  }
+
+  // Check if this Function Replacement is Needed; if yes, return a new name. otherwise,
return the original name
+  private String functionReplacement(FunctionCall functionCall) {
+    String funcName = functionCall.getName();
+    if(optionManager != null && optionManager.getOption(ExecConstants.CAST_TO_NULLABLE_NUMERIC).bool_val
&& CastFunctions.isReplacementNeeded(functionCall.args.get(0).getMajorType().getMinorType(),
funcName)) {
+      org.apache.drill.common.types.TypeProtos.DataMode dataMode = functionCall.args.get(0).getMajorType().getMode();
+      funcName = CastFunctions.getReplacingCastFunction(funcName, dataMode);
+    }
+
+    return funcName;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 165d5f3..83a89df 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -74,8 +74,8 @@ public class DrillbitContext {
     this.storagePlugins = new StoragePluginRegistry(this);
     this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(),
endpoint, storagePlugins);
     this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig());
-    this.functionRegistry = new FunctionImplementationRegistry(context.getConfig());
     this.systemOptions = new SystemOptionManager(context.getConfig(), provider);
+    this.functionRegistry = new FunctionImplementationRegistry(context.getConfig(), systemOptions);
     this.compiler = new CodeCompiler(context.getConfig(), systemOptions);
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 34729e2..f20627d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -54,6 +54,7 @@ public class SystemOptionManager implements OptionManager {
       PlannerSettings.PRODUCER_CONSUMER_QUEUE_SIZE,
       PlannerSettings.HASH_SINGLE_KEY,
       PlannerSettings.IDENTIFIER_MAX_LENGTH,
+      ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
       ExecConstants.OUTPUT_FORMAT_VALIDATOR,
       ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
       ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR,

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
index e0cce8b..495e3e2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
@@ -145,11 +145,16 @@ public class DrillTextRecordReader extends AbstractRecordReader {
       int recordCount = 0;
       while (redoRecord || (batchSize < 200*1000 && reader.next(key, value)))
{
         redoRecord = false;
+
+        // start and end together are used to split fields
         int start;
         int end = -1;
+
+        // index of the scanned field
         int p = 0;
         int i = 0;
         vector.getMutator().startNewGroup(recordCount);
+        // Process each field in this line
         while (end < value.getLength() - 1) {
           if(numCols > 0 && p >= numCols) {
             break;
@@ -195,7 +200,15 @@ public class DrillTextRecordReader extends AbstractRecordReader {
     }
   }
 
-  public int find(Text text, byte what, int start) {
+  /**
+   * Returns the index within the text of the first occurrence of delimiter, starting the
search at the specified index.
+   *
+   * @param  text  the text being searched
+   * @param  delimiter the delimiter
+   * @param  start the index to start searching
+   * @return      the first occurrence of delimiter, starting the search at the specified
index
+   */
+  public int find(Text text, byte delimiter, int start) {
     int len = text.getLength();
     int p = start;
     byte[] bytes = text.getBytes();
@@ -204,7 +217,7 @@ public class DrillTextRecordReader extends AbstractRecordReader {
       if ('\"' == bytes[p]) {
         inQuotes = !inQuotes;
       }
-      if (!inQuotes && bytes[p] == what) {
+      if (!inQuotes && bytes[p] == delimiter) {
         return p;
       }
       p++;

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
new file mode 100644
index 0000000..3e05c0e
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.fn.impl;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.FileUtils;
+import org.junit.Test;
+
+public class TestCastEmptyStrings extends BaseTestQuery {
+    @Test // see DRILL-1874
+    public void testCastInputTypeNullableVarCharToNumeric() throws Exception {
+        String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
+
+        // Enable the new cast functions (cast empty string "" to null)
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+
+        // Test Optional VarChar
+        test(String.format("select cast(columns[0] as int) from dfs_test.`%s`;", root));
+        test(String.format("select cast(columns[0] as bigint) from dfs_test.`%s`;", root));
+        test(String.format("select cast(columns[0] as float) from dfs_test.`%s`;", root));
+        test(String.format("select cast(columns[0] as double) from dfs_test.`%s`;", root));
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+    }
+
+    @Test // see DRILL-1874
+    public void testCastInputTypeNonNullableVarCharToNumeric() throws Exception {
+        String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
+
+        // Enable the new cast functions (cast empty string "" to null)
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+        // Test Required VarChar
+        test(String.format("select cast('' as int) from dfs_test.`%s`;", root));
+        test(String.format("select cast('' as bigint) from dfs_test.`%s`;", root));
+        test(String.format("select cast('' as float) from dfs_test.`%s`;", root));
+        test(String.format("select cast('' as double) from dfs_test.`%s`;", root));
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+    }
+
+    @Test // see DRILL-1874
+    public void testCastInputTypeNullableVarCharToDecimal() throws Exception {
+        String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
+
+        // Enable the new cast functions (cast empty string "" to null)
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+
+        // Test Optional VarChar
+        test(String.format("select cast(columns[0] as decimal) from dfs_test.`%s` where cast(columns[0]
as decimal) is null;", root));
+        test(String.format("select cast(columns[0] as decimal(9)) from dfs_test.`%s`;", root));
+        test(String.format("select cast(columns[0] as decimal(18)) from dfs_test.`%s`;",
root));
+        test(String.format("select cast(columns[0] as decimal(28)) from dfs_test.`%s`;",
root));
+        test(String.format("select cast(columns[0] as decimal(38)) from dfs_test.`%s`;",
root));
+
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+    }
+
+    @Test // see DRILL-1874
+    public void testCastInputTypeNonNullableVarCharToDecimal() throws Exception {
+        String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
+
+        // Enable the new cast functions (cast empty string "" to null)
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+
+        // Test Required VarChar
+        test(String.format("select cast('' as decimal) from dfs_test.`%s` where cast('' as
decimal) is null;", root));
+        test(String.format("select cast('' as decimal(18)) from dfs_test.`%s`;", root));
+        test(String.format("select cast('' as decimal(28)) from dfs_test.`%s`;", root));
+        test(String.format("select cast('' as decimal(38)) from dfs_test.`%s`;", root));
+
+        test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+    }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/487d98e5/exec/java-exec/src/test/resources/emptyStrings.csv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/emptyStrings.csv b/exec/java-exec/src/test/resources/emptyStrings.csv
new file mode 100644
index 0000000..93ebd40
--- /dev/null
+++ b/exec/java-exec/src/test/resources/emptyStrings.csv
@@ -0,0 +1,3 @@
+,0,0,0
+1,,1,1
+2,2,,2
\ No newline at end of file


Mime
View raw message