orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [4/7] orc git commit: HIVE-14000 Changing a numeric type column causes values other than NULL.
Date Thu, 30 Jun 2016 22:00:17 GMT
HIVE-14000 Changing a numeric type column causes values other than NULL.


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/17e14f62
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/17e14f62
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/17e14f62

Branch: refs/heads/master
Commit: 17e14f62d2315c5149e4e491ccf922913b13cc7e
Parents: 68cdbd4
Author: Owen O'Malley <omalley@apache.org>
Authored: Thu Jun 30 10:21:24 2016 -0700
Committer: Owen O'Malley <omalley@apache.org>
Committed: Thu Jun 30 14:33:57 2016 -0700

----------------------------------------------------------------------
 .../orc/impl/ConvertTreeReaderFactory.java      | 305 ++++++++++---------
 .../hadoop/hive/ql/util/TimestampUtils.java     |  70 +++--
 2 files changed, 206 insertions(+), 169 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/orc/blob/17e14f62/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index 3ba56f7..753e5bc 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -35,8 +35,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
 import org.apache.hadoop.hive.ql.util.TimestampUtils;
 import org.apache.hadoop.hive.serde2.io.DateWritable;
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.FloatWritable;
 import org.apache.orc.OrcProto;
 import org.apache.orc.TypeDescription;
 import org.apache.orc.TypeDescription.Category;
@@ -263,6 +261,22 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       return string;
     }
 
+    private static final double MIN_LONG_AS_DOUBLE = -0x1p63;
+    /*
+     * We cannot store Long.MAX_VALUE as a double without losing precision. Instead, we store
+     * Long.MAX_VALUE + 1 == -Long.MIN_VALUE, and then offset all comparisons by 1.
+     */
+    private static final double MAX_LONG_AS_DOUBLE_PLUS_ONE = 0x1p63;
+
+    public boolean doubleCanFitInLong(double doubleValue) {
+
+      // Borrowed from Guava DoubleMath.roundToLong except do not want dependency on Guava
and we
+      // don't want to catch an exception.
+
+      return ((MIN_LONG_AS_DOUBLE - doubleValue < 1.0) &&
+              (doubleValue < MAX_LONG_AS_DOUBLE_PLUS_ONE));
+    }
+
     @Override
     void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
       // Pass-thru.
@@ -336,20 +350,44 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       }
     }
 
-    public long downCastAnyInteger(long input, TypeDescription readerType) {
-      switch (readerType.getCategory()) {
+    public void downCastAnyInteger(LongColumnVector longColVector, int elementNum,
+        TypeDescription readerType) {
+      downCastAnyInteger(longColVector, elementNum, longColVector.vector[elementNum], readerType);
+    }
+
+    public void downCastAnyInteger(LongColumnVector longColVector, int elementNum, long inputLong,
+        TypeDescription readerType) {
+      long[] vector = longColVector.vector;
+      long outputLong;
+      Category readerCategory = readerType.getCategory();
+      switch (readerCategory) {
       case BOOLEAN:
-        return input == 0 ? 0 : 1;
+        // No data loss for boolean.
+        vector[elementNum] = inputLong == 0 ? 0 : 1;
+        return;
       case BYTE:
-        return (byte) input;
+        outputLong = (byte) inputLong;
+        break;
       case SHORT:
-        return (short) input;
+        outputLong = (short) inputLong;
+        break;
       case INT:
-        return (int) input;
+        outputLong = (int) inputLong;
+        break;
       case LONG:
-        return input;
+        // No data loss for long.
+        vector[elementNum] = inputLong;
+        return;
       default:
-        throw new RuntimeException("Unexpected type kind " + readerType.getCategory().name());
+        throw new RuntimeException("Unexpected type kind " + readerCategory.name());
+      }
+
+      if (outputLong != inputLong) {
+        // Data loss.
+        longColVector.isNull[elementNum] = true;
+        longColVector.noNulls = false;
+      } else {
+        vector[elementNum] = outputLong;
       }
     }
 
@@ -439,25 +477,22 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       anyIntegerAsLongTreeReader.nextVector(previousVector, isNull, batchSize);
       LongColumnVector resultColVector = (LongColumnVector) previousVector;
       if (downCastNeeded) {
-        long[] resultVector = resultColVector.vector;
         if (resultColVector.isRepeating) {
           if (resultColVector.noNulls || !resultColVector.isNull[0]) {
-            resultVector[0] = downCastAnyInteger(resultVector[0], readerType);
+            downCastAnyInteger(resultColVector, 0, readerType);
           } else {
-            resultColVector.noNulls = false;
-            resultColVector.isNull[0] = true;
+            // Result remains null.
           }
         } else if (resultColVector.noNulls){
           for (int i = 0; i < batchSize; i++) {
-            resultVector[i] = downCastAnyInteger(resultVector[i], readerType);
+            downCastAnyInteger(resultColVector, i, readerType);
           }
         } else {
           for (int i = 0; i < batchSize; i++) {
             if (!resultColVector.isNull[i]) {
-              resultVector[i] = downCastAnyInteger(resultVector[i], readerType);
+              downCastAnyInteger(resultColVector, i, readerType);
             } else {
-              resultColVector.noNulls = false;
-              resultColVector.isNull[i] = true;
+              // Result remains null.
             }
           }
         }
@@ -470,7 +505,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     private FloatTreeReader floatTreeReader;
 
     private final TypeDescription readerType;
-    private FloatWritable floatResult;
     private DoubleColumnVector doubleColVector;
     private LongColumnVector longColVector;
 
@@ -480,15 +514,19 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       this.readerType = readerType;
       floatTreeReader = new FloatTreeReader(columnId);
       setConvertTreeReader(floatTreeReader);
-      floatResult = new FloatWritable();
     }
 
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
-      float floatValue = (float) doubleColVector.vector[elementNum];
-      longColVector.vector[elementNum] =
-          downCastAnyInteger(
-              (long) floatValue, readerType);
+      double doubleValue = doubleColVector.vector[elementNum];
+      if (!doubleCanFitInLong(doubleValue)) {
+        longColVector.isNull[elementNum] = true;
+        longColVector.noNulls = false;
+      } else {
+        // UNDONE: Does the overflow check above using double really work here for float?
+        float floatValue = (float) doubleValue;
+        downCastAnyInteger(longColVector, elementNum, (long) floatValue, readerType);
+      }
     }
 
     @Override
@@ -525,9 +563,13 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
-      longColVector.vector[elementNum] =
-          downCastAnyInteger(
-              (long) doubleColVector.vector[elementNum], readerType);
+      double doubleValue = doubleColVector.vector[elementNum];
+      if (!doubleCanFitInLong(doubleValue)) {
+        longColVector.isNull[elementNum] = true;
+        longColVector.noNulls = false;
+      } else {
+        downCastAnyInteger(longColVector, elementNum, (long) doubleValue, readerType);
+      }
     }
 
     @Override
@@ -553,7 +595,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     private final int precision;
     private final int scale;
     private final TypeDescription readerType;
-    private HiveDecimalWritable hiveDecimalResult;
     private DecimalColumnVector decimalColVector;
     private LongColumnVector longColVector;
 
@@ -565,15 +606,21 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       this.readerType = readerType;
       decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
       setConvertTreeReader(decimalTreeReader);
-      hiveDecimalResult = new HiveDecimalWritable();
     }
 
+    private static HiveDecimal DECIMAL_MAX_LONG = HiveDecimal.create(Long.MAX_VALUE);
+    private static HiveDecimal DECIMAL_MIN_LONG = HiveDecimal.create(Long.MIN_VALUE);
+
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
-      longColVector.vector[elementNum] =
-          downCastAnyInteger(
-              decimalColVector.vector[elementNum].getHiveDecimal().longValue(),
-              readerType);
+      HiveDecimal decimalValue = decimalColVector.vector[elementNum].getHiveDecimal();
+      if (decimalValue.compareTo(DECIMAL_MAX_LONG) > 0 ||
+          decimalValue.compareTo(DECIMAL_MIN_LONG) < 0) {
+        longColVector.isNull[elementNum] = true;
+        longColVector.noNulls = false;
+      } else {
+        downCastAnyInteger(longColVector, elementNum, decimalValue.longValue(), readerType);
+      }
     }
 
     @Override
@@ -596,7 +643,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
     private final TypeDescription readerType;
     private BytesColumnVector bytesColVector;
     private LongColumnVector longColVector;
@@ -604,7 +650,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     AnyIntegerFromStringGroupTreeReader(int columnId, TypeDescription fileType,
         TypeDescription readerType) throws IOException {
       super(columnId);
-      this.fileType = fileType;
       this.readerType = readerType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
@@ -615,8 +660,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       String string = stringFromBytesColumnVectorEntry(bytesColVector, elementNum);
       long longValue = parseLongFromString(string);
       if (!getIsParseError()) {
-        longColVector.vector[elementNum] =
-            downCastAnyInteger(longValue, readerType);
+        downCastAnyInteger(longColVector, elementNum, longValue, readerType);
       } else {
         longColVector.noNulls = false;
         longColVector.isNull[elementNum] = true;
@@ -660,8 +704,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       // Use TimestampWritable's getSeconds.
       long longValue = TimestampUtils.millisToSeconds(
           timestampColVector.asScratchTimestamp(elementNum).getTime());
-      longColVector.vector[elementNum] =
-          downCastAnyInteger(longValue, readerType);
+      downCastAnyInteger(longColVector, elementNum, longValue, readerType);
     }
 
     @Override
@@ -745,8 +788,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
         if (resultColVector.noNulls || !resultColVector.isNull[0]) {
           resultVector[0] = (float) resultVector[0];
         } else {
-          resultColVector.noNulls = false;
-          resultColVector.isNull[0] = true;
+          // Remains null.
         }
       } else if (resultColVector.noNulls){
         for (int i = 0; i < batchSize; i++) {
@@ -757,8 +799,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
           if (!resultColVector.isNull[i]) {
             resultVector[i] = (float) resultVector[i];
           } else {
-            resultColVector.noNulls = false;
-            resultColVector.isNull[i] = true;
+            // Remains null.
           }
         }
       }
@@ -771,8 +812,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private final int precision;
     private final int scale;
-    private final TypeDescription readerType;
-    private HiveDecimalWritable hiveDecimalResult;
     private DecimalColumnVector decimalColVector;
     private DoubleColumnVector doubleColVector;
 
@@ -781,10 +820,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       super(columnId);
       this.precision = fileType.getPrecision();
       this.scale = fileType.getScale();
-      this.readerType = readerType;
       decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
       setConvertTreeReader(decimalTreeReader);
-      hiveDecimalResult = new HiveDecimalWritable();
     }
 
     @Override
@@ -813,14 +850,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
     private BytesColumnVector bytesColVector;
     private DoubleColumnVector doubleColVector;
 
     FloatFromStringGroupTreeReader(int columnId, TypeDescription fileType)
         throws IOException {
       super(columnId);
-      this.fileType = fileType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
     }
@@ -858,14 +893,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TimestampTreeReader timestampTreeReader;
 
-    private final TypeDescription readerType;
     private TimestampColumnVector timestampColVector;
     private DoubleColumnVector doubleColVector;
 
-    FloatFromTimestampTreeReader(int columnId, TypeDescription readerType,
-        boolean skipCorrupt) throws IOException {
+    FloatFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
       super(columnId);
-      this.readerType = readerType;
       timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
       setConvertTreeReader(timestampTreeReader);
     }
@@ -940,13 +972,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private FloatTreeReader floatTreeReader;
 
-    private FloatWritable floatResult;
-
     DoubleFromFloatTreeReader(int columnId) throws IOException {
       super(columnId);
       floatTreeReader = new FloatTreeReader(columnId);
       setConvertTreeReader(floatTreeReader);
-      floatResult = new FloatWritable();
     }
 
     @Override
@@ -964,20 +993,15 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private final int precision;
     private final int scale;
-    private final TypeDescription readerType;
-    private HiveDecimalWritable hiveDecimalResult;
     private DecimalColumnVector decimalColVector;
     private DoubleColumnVector doubleColVector;
 
-    DoubleFromDecimalTreeReader(int columnId, TypeDescription fileType,
-        TypeDescription readerType) throws IOException {
+    DoubleFromDecimalTreeReader(int columnId, TypeDescription fileType) throws IOException
{
       super(columnId);
       this.precision = fileType.getPrecision();
       this.scale = fileType.getScale();
-      this.readerType = readerType;
       decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
       setConvertTreeReader(decimalTreeReader);
-      hiveDecimalResult = new HiveDecimalWritable();
     }
 
     @Override
@@ -1006,14 +1030,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
     private BytesColumnVector bytesColVector;
     private DoubleColumnVector doubleColVector;
 
     DoubleFromStringGroupTreeReader(int columnId, TypeDescription fileType)
         throws IOException {
       super(columnId);
-      this.fileType = fileType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
     }
@@ -1050,14 +1072,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TimestampTreeReader timestampTreeReader;
 
-    private final TypeDescription readerType;
     private TimestampColumnVector timestampColVector;
     private DoubleColumnVector doubleColVector;
 
-    DoubleFromTimestampTreeReader(int columnId, TypeDescription readerType,
-        boolean skipCorrupt) throws IOException {
+    DoubleFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
       super(columnId);
-      this.readerType = readerType;
       timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
       setConvertTreeReader(timestampTreeReader);
     }
@@ -1088,16 +1107,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private AnyIntegerTreeReader anyIntegerAsLongTreeReader;
 
-    private int precision;
-    private int scale;
     private LongColumnVector longColVector;
     private DecimalColumnVector decimalColVector;
 
-    DecimalFromAnyIntegerTreeReader(int columnId, TypeDescription fileType,
-        TypeDescription readerType, boolean skipCorrupt) throws IOException {
+    DecimalFromAnyIntegerTreeReader(int columnId, TypeDescription fileType, boolean skipCorrupt)
+        throws IOException {
       super(columnId);
-      this.precision = readerType.getPrecision();
-      this.scale = readerType.getScale();
       anyIntegerAsLongTreeReader =
           new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
       setConvertTreeReader(anyIntegerAsLongTreeReader);
@@ -1106,8 +1121,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     @Override
     public void setConvertVectorElement(int elementNum) {
       long longValue = longColVector.vector[elementNum];
-      HiveDecimalWritable hiveDecimalWritable =
-          new HiveDecimalWritable(longValue);
+      HiveDecimalWritable hiveDecimalWritable = new HiveDecimalWritable(longValue);
+      // The DecimalColumnVector will enforce precision and scale and set the entry to null
when out of bounds.
       decimalColVector.set(elementNum, hiveDecimalWritable);
     }
 
@@ -1131,30 +1146,25 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private FloatTreeReader floatTreeReader;
 
-    private int precision;
-    private int scale;
-    private FloatWritable floatResult;
     private DoubleColumnVector doubleColVector;
     private DecimalColumnVector decimalColVector;
 
     DecimalFromFloatTreeReader(int columnId, TypeDescription readerType)
         throws IOException {
       super(columnId);
-      this.precision = readerType.getPrecision();
-      this.scale = readerType.getScale();
       floatTreeReader = new FloatTreeReader(columnId);
       setConvertTreeReader(floatTreeReader);
-      floatResult = new FloatWritable();
     }
 
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
       float floatValue = (float) doubleColVector.vector[elementNum];
       if (!Float.isNaN(floatValue)) {
-        HiveDecimal value =
+        HiveDecimal decimalValue =
             HiveDecimal.create(Float.toString(floatValue));
-        if (value != null) {
-          decimalColVector.set(elementNum, value);
+        if (decimalValue != null) {
+          // The DecimalColumnVector will enforce precision and scale and set the entry to
null when out of bounds.
+          decimalColVector.set(elementNum, decimalValue);
         } else {
           decimalColVector.noNulls = false;
           decimalColVector.isNull[elementNum] = true;
@@ -1227,14 +1237,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
     private BytesColumnVector bytesColVector;
     private DecimalColumnVector decimalColVector;
 
     DecimalFromStringGroupTreeReader(int columnId, TypeDescription fileType,
         TypeDescription readerType) throws IOException {
       super(columnId);
-      this.fileType = fileType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
     }
@@ -1244,6 +1252,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       String string = stringFromBytesColumnVectorEntry(bytesColVector, elementNum);
       HiveDecimal value = parseDecimalFromString(string);
       if (value != null) {
+        // The DecimalColumnVector will enforce precision and scale and set the entry to
null when out of bounds.
         decimalColVector.set(elementNum, value);
       } else {
         decimalColVector.noNulls = false;
@@ -1271,18 +1280,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TimestampTreeReader timestampTreeReader;
 
-    private final TypeDescription readerType;
     private TimestampColumnVector timestampColVector;
-    private int precision;
-    private int scale;
     private DecimalColumnVector decimalColVector;
 
-    DecimalFromTimestampTreeReader(int columnId, TypeDescription readerType,
-        boolean skipCorrupt) throws IOException {
+    DecimalFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException
{
       super(columnId);
-      this.readerType = readerType;
-      this.precision = readerType.getPrecision();
-      this.scale = readerType.getScale();
       timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
       setConvertTreeReader(timestampTreeReader);
     }
@@ -1293,6 +1295,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
           timestampColVector.asScratchTimestamp(elementNum));
       HiveDecimal value = HiveDecimal.create(Double.toString(doubleValue));
       if (value != null) {
+        // The DecimalColumnVector will enforce precision and scale and set the entry to
null when out of bounds.
         decimalColVector.set(elementNum, value);
       } else {
         decimalColVector.noNulls = false;
@@ -1316,11 +1319,61 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     }
   }
 
+  public static class DecimalFromDecimalTreeReader extends ConvertTreeReader {
+
+    private DecimalTreeReader decimalTreeReader;
+
+    private DecimalColumnVector fileDecimalColVector;
+    private int filePrecision;
+    private int fileScale;
+    private int readerPrecision;
+    private int readerScale;
+    private DecimalColumnVector decimalColVector;
+
+    DecimalFromDecimalTreeReader(int columnId, TypeDescription fileType, TypeDescription
readerType)
+        throws IOException {
+      super(columnId);
+      filePrecision = fileType.getPrecision();
+      fileScale = fileType.getScale();
+      readerPrecision = readerType.getPrecision();
+      readerScale = readerType.getScale();
+      decimalTreeReader = new DecimalTreeReader(columnId, filePrecision, fileScale);
+      setConvertTreeReader(decimalTreeReader);
+    }
+
+    @Override
+    public void setConvertVectorElement(int elementNum) throws IOException {
+
+      HiveDecimalWritable valueWritable = HiveDecimalWritable.enforcePrecisionScale(
+          fileDecimalColVector.vector[elementNum], readerPrecision, readerScale);
+      if (valueWritable != null) {
+        decimalColVector.set(elementNum, valueWritable);
+      } else {
+        decimalColVector.noNulls = false;
+        decimalColVector.isNull[elementNum] = true;
+      }
+    }
+
+    @Override
+    public void nextVector(ColumnVector previousVector,
+                           boolean[] isNull,
+                           final int batchSize) throws IOException {
+      if (fileDecimalColVector == null) {
+        // Allocate column vector for file; cast column vector for reader.
+        fileDecimalColVector = new DecimalColumnVector(filePrecision, fileScale);
+        decimalColVector = (DecimalColumnVector) previousVector;
+      }
+      // Read present/isNull stream
+      decimalTreeReader.nextVector(fileDecimalColVector, isNull, batchSize);
+
+      convertVector(fileDecimalColVector, decimalColVector, batchSize);
+    }
+  }
+
   public static class StringGroupFromAnyIntegerTreeReader extends ConvertTreeReader {
 
     private AnyIntegerTreeReader anyIntegerAsLongTreeReader;
 
-    private final TypeDescription fileType;
     private final TypeDescription readerType;
     private LongColumnVector longColVector;
     private BytesColumnVector bytesColVector;
@@ -1328,7 +1381,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     StringGroupFromAnyIntegerTreeReader(int columnId, TypeDescription fileType,
         TypeDescription readerType, boolean skipCorrupt) throws IOException {
       super(columnId);
-      this.fileType = fileType;
       this.readerType = readerType;
       anyIntegerAsLongTreeReader =
           new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
@@ -1364,7 +1416,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     private FloatTreeReader floatTreeReader;
 
     private final TypeDescription readerType;
-    private FloatWritable floatResult;
     private DoubleColumnVector doubleColVector;
     private BytesColumnVector bytesColVector;
 
@@ -1375,7 +1426,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       this.readerType = readerType;
       floatTreeReader = new FloatTreeReader(columnId);
       setConvertTreeReader(floatTreeReader);
-      floatResult = new FloatWritable();
     }
 
     @Override
@@ -1544,7 +1594,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     private final TypeDescription readerType;
     private LongColumnVector longColVector;
     private BytesColumnVector bytesColVector;
-    private DateWritable dateWritableResult;
     private Date date;
 
     StringGroupFromDateTreeReader(int columnId, TypeDescription readerType,
@@ -1553,7 +1602,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       this.readerType = readerType;
       dateTreeReader = new DateTreeReader(columnId);
       setConvertTreeReader(dateTreeReader);
-      dateWritableResult = new DateWritable();
       date = new Date(0);
     }
 
@@ -1585,13 +1633,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
     private final TypeDescription readerType;
 
     StringGroupFromStringGroupTreeReader(int columnId, TypeDescription fileType,
         TypeDescription readerType) throws IOException {
       super(columnId);
-      this.fileType = fileType;
       this.readerType = readerType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
@@ -1609,8 +1655,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
         if (resultColVector.noNulls || !resultColVector.isNull[0]) {
           convertStringGroupVectorElement(resultColVector, 0, readerType);
         } else {
-          resultColVector.noNulls = false;
-          resultColVector.isNull[0] = true;
+          // Remains null.
         }
       } else if (resultColVector.noNulls){
         for (int i = 0; i < batchSize; i++) {
@@ -1621,8 +1666,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
           if (!resultColVector.isNull[i]) {
             convertStringGroupVectorElement(resultColVector, i, readerType);
           } else {
-            resultColVector.noNulls = false;
-            resultColVector.isNull[i] = true;
+            // Remains null.
           }
         }
       }
@@ -1634,7 +1678,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     private BinaryTreeReader binaryTreeReader;
 
     private final TypeDescription readerType;
-    private BytesWritable binaryWritableResult;
     private BytesColumnVector inBytesColVector;
     private BytesColumnVector outBytesColVector;
 
@@ -1644,7 +1687,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       this.readerType = readerType;
       binaryTreeReader = new BinaryTreeReader(columnId);
       setConvertTreeReader(binaryTreeReader);
-      binaryWritableResult = new BytesWritable();
     }
 
     @Override
@@ -1725,7 +1767,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private FloatTreeReader floatTreeReader;
 
-    private FloatWritable floatResult;
     private DoubleColumnVector doubleColVector;
     private TimestampColumnVector timestampColVector;
 
@@ -1734,14 +1775,14 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       super(columnId);
       floatTreeReader = new FloatTreeReader(columnId);
       setConvertTreeReader(floatTreeReader);
-      floatResult = new FloatWritable();
     }
 
     @Override
     public void setConvertVectorElement(int elementNum) {
       float floatValue = (float) doubleColVector.vector[elementNum];
-      timestampColVector.set(elementNum,
-          TimestampUtils.doubleToTimestamp(floatValue));
+      Timestamp timestampValue = TimestampUtils.doubleToTimestamp(floatValue);
+      // The TimestampColumnVector will set the entry to null when a null timestamp is passed
in.
+      timestampColVector.set(elementNum, timestampValue);
     }
 
     @Override
@@ -1777,8 +1818,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     @Override
     public void setConvertVectorElement(int elementNum) {
       double doubleValue = doubleColVector.vector[elementNum];
-      timestampColVector.set(elementNum,
-          TimestampUtils.doubleToTimestamp(doubleValue));
+      Timestamp timestampValue = TimestampUtils.doubleToTimestamp(doubleValue);
+      // The TimestampColumnVector will set the entry to null when a null timestamp is passed
in.
+      timestampColVector.set(elementNum, timestampValue);
     }
 
     @Override
@@ -1803,7 +1845,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private final int precision;
     private final int scale;
-    private HiveDecimalWritable hiveDecimalResult;
     private DecimalColumnVector decimalColVector;
     private TimestampColumnVector timestampColVector;
 
@@ -1814,14 +1855,14 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       this.scale = fileType.getScale();
       decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
       setConvertTreeReader(decimalTreeReader);
-      hiveDecimalResult = new HiveDecimalWritable();
     }
 
     @Override
     public void setConvertVectorElement(int elementNum) {
       Timestamp timestampValue =
-          TimestampUtils.decimalToTimestamp(
-              decimalColVector.vector[elementNum].getHiveDecimal());
+            TimestampUtils.decimalToTimestamp(
+                decimalColVector.vector[elementNum].getHiveDecimal());
+      // The TimestampColumnVector will set the entry to null when a null timestamp is passed
in.
       timestampColVector.set(elementNum, timestampValue);
     }
 
@@ -1845,14 +1886,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
     private BytesColumnVector bytesColVector;
     private TimestampColumnVector timestampColVector;
 
     TimestampFromStringGroupTreeReader(int columnId, TypeDescription fileType)
         throws IOException {
       super(columnId);
-      this.fileType = fileType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
     }
@@ -1890,7 +1929,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private DateTreeReader dateTreeReader;
 
-    private DateWritable doubleResult;
     private LongColumnVector longColVector;
     private TimestampColumnVector timestampColVector;
 
@@ -1899,7 +1937,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       super(columnId);
       dateTreeReader = new DateTreeReader(columnId);
       setConvertTreeReader(dateTreeReader);
-      doubleResult = new DateWritable();
     }
 
     @Override
@@ -1929,14 +1966,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
     private BytesColumnVector bytesColVector;
     private LongColumnVector longColVector;
 
     DateFromStringGroupTreeReader(int columnId, TypeDescription fileType)
         throws IOException {
       super(columnId);
-      this.fileType = fileType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
     }
@@ -1974,14 +2009,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TimestampTreeReader timestampTreeReader;
 
-    private final TypeDescription readerType;
     private TimestampColumnVector timestampColVector;
     private LongColumnVector longColVector;
 
-    DateFromTimestampTreeReader(int columnId, TypeDescription readerType,
-        boolean skipCorrupt) throws IOException {
+    DateFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
       super(columnId);
-      this.readerType = readerType;
       timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
       setConvertTreeReader(timestampTreeReader);
     }
@@ -2014,12 +2046,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private TreeReader stringGroupTreeReader;
 
-    private final TypeDescription fileType;
-
     BinaryFromStringGroupTreeReader(int columnId, TypeDescription fileType)
         throws IOException {
       super(columnId);
-      this.fileType = fileType;
       stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
       setConvertTreeReader(stringGroupTreeReader);
     }
@@ -2064,7 +2093,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
           skipCorrupt);
 
     case DECIMAL:
-      return new DecimalFromAnyIntegerTreeReader(columnId, fileType, readerType, skipCorrupt);
+      return new DecimalFromAnyIntegerTreeReader(columnId, fileType, skipCorrupt);
 
     case STRING:
     case CHAR:
@@ -2208,7 +2237,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       return new FloatFromDecimalTreeReader(columnId, fileType, readerType);
 
     case DOUBLE:
-      return new DoubleFromDecimalTreeReader(columnId, fileType, readerType);
+      return new DoubleFromDecimalTreeReader(columnId, fileType);
 
     case STRING:
     case CHAR:
@@ -2424,13 +2453,13 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       return new AnyIntegerFromTimestampTreeReader(columnId, readerType, skipCorrupt);
 
     case FLOAT:
-      return new FloatFromTimestampTreeReader(columnId, readerType, skipCorrupt);
+      return new FloatFromTimestampTreeReader(columnId, skipCorrupt);
 
     case DOUBLE:
-      return new DoubleFromTimestampTreeReader(columnId, readerType, skipCorrupt);
+      return new DoubleFromTimestampTreeReader(columnId, skipCorrupt);
 
     case DECIMAL:
-      return new DecimalFromTimestampTreeReader(columnId, readerType, skipCorrupt);
+      return new DecimalFromTimestampTreeReader(columnId, skipCorrupt);
 
     case STRING:
     case CHAR:
@@ -2442,7 +2471,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
           readerType.getCategory() + " to self needed");
 
     case DATE:
-      return new DateFromTimestampTreeReader(columnId, readerType, skipCorrupt);
+      return new DateFromTimestampTreeReader(columnId, skipCorrupt);
 
     // Not currently supported conversion(s):
     case BINARY:

http://git-wip-us.apache.org/repos/asf/orc/blob/17e14f62/java/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
----------------------------------------------------------------------
diff --git a/java/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
b/java/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
index 189ead5..41db9ca 100644
--- a/java/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
+++ b/java/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
@@ -39,45 +39,53 @@ public class TimestampUtils {
   }
 
   public static Timestamp doubleToTimestamp(double f) {
-    long seconds = (long) f;
-
-    // We must ensure the exactness of the double's fractional portion.
-    // 0.6 as the fraction part will be converted to 0.59999... and
-    // significantly reduce the savings from binary serialization
-    BigDecimal bd;
     try {
-      bd = new BigDecimal(String.valueOf(f));
+      long seconds = (long) f;
+
+      // We must ensure the exactness of the double's fractional portion.
+      // 0.6 as the fraction part will be converted to 0.59999... and
+      // significantly reduce the savings from binary serialization
+      BigDecimal bd = new BigDecimal(String.valueOf(f));
+
+      bd = bd.subtract(new BigDecimal(seconds)).multiply(new BigDecimal(1000000000));
+      int nanos = bd.intValue();
+
+      // Convert to millis
+      long millis = seconds * 1000;
+      if (nanos < 0) {
+        millis -= 1000;
+        nanos += 1000000000;
+      }
+      Timestamp t = new Timestamp(millis);
+
+      // Set remaining fractional portion to nanos
+      t.setNanos(nanos);
+      return t;
     } catch (NumberFormatException nfe) {
       return null;
+    } catch (IllegalArgumentException iae) {
+      return null;
     }
-    bd = bd.subtract(new BigDecimal(seconds)).multiply(new BigDecimal(1000000000));
-    int nanos = bd.intValue();
-
-    // Convert to millis
-    long millis = seconds * 1000;
-    if (nanos < 0) {
-      millis -= 1000;
-      nanos += 1000000000;
-    }
-    Timestamp t = new Timestamp(millis);
-
-    // Set remaining fractional portion to nanos
-    t.setNanos(nanos);
-    return t;
   }
 
   public static Timestamp decimalToTimestamp(HiveDecimal d) {
-    BigDecimal nanoInstant = d.bigDecimalValue().multiply(BILLION_BIG_DECIMAL);
-    int nanos = nanoInstant.remainder(BILLION_BIG_DECIMAL).intValue();
-    if (nanos < 0) {
-      nanos += 1000000000;
-    }
-    long seconds =
-        nanoInstant.subtract(new BigDecimal(nanos)).divide(BILLION_BIG_DECIMAL).longValue();
-    Timestamp t = new Timestamp(seconds * 1000);
-    t.setNanos(nanos);
+    try {
+      BigDecimal nanoInstant = d.bigDecimalValue().multiply(BILLION_BIG_DECIMAL);
+      int nanos = nanoInstant.remainder(BILLION_BIG_DECIMAL).intValue();
+      if (nanos < 0) {
+        nanos += 1000000000;
+      }
+      long seconds =
+          nanoInstant.subtract(new BigDecimal(nanos)).divide(BILLION_BIG_DECIMAL).longValue();
+      Timestamp t = new Timestamp(seconds * 1000);
+      t.setNanos(nanos);
 
-    return t;
+      return t;
+    } catch (NumberFormatException nfe) {
+      return null;
+    } catch (IllegalArgumentException iae) {
+      return null;
+    }
   }
 
   /**


Mime
View raw message