hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmccl...@apache.org
Subject [10/10] hive git commit: HIVE-15335: Fast Decimal (Matt McCline, reviewed by Sergey Shelukhin, Prasanth Jayachandran, Owen O'Malley)
Date Thu, 22 Dec 2016 08:32:39 GMT
HIVE-15335: Fast Decimal (Matt McCline, reviewed by Sergey Shelukhin, Prasanth Jayachandran, Owen O'Malley)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ba713cc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ba713cc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ba713cc

Branch: refs/heads/master
Commit: 4ba713ccd85c3706d195aeef9476e6e6363f1c21
Parents: 597ca1b
Author: Matt McCline <mmccline@hortonworks.com>
Authored: Thu Dec 22 00:32:08 2016 -0800
Committer: Matt McCline <mmccline@hortonworks.com>
Committed: Thu Dec 22 00:32:08 2016 -0800

----------------------------------------------------------------------
 .../hive/common/type/HiveDecimalTestBase.java   |  561 ++
 .../hive/common/type/TestHiveDecimal.java       |  216 -
 .../TestHiveDecimalOrcSerializationUtils.java   |  378 +
 .../apache/orc/impl/ColumnStatisticsImpl.java   |   50 +-
 .../orc/impl/ConvertTreeReaderFactory.java      |   81 +-
 .../org/apache/orc/impl/TreeReaderFactory.java  |   30 +-
 .../java/org/apache/orc/impl/WriterImpl.java    |   33 +-
 .../org/apache/orc/TestColumnStatistics.java    |   13 +-
 .../UDAFTemplates/VectorUDAFMinMaxDecimal.txt   |   45 +-
 .../ql/exec/vector/VectorDeserializeRow.java    |    3 +-
 .../hive/ql/exec/vector/VectorExtractRow.java   |    3 +-
 .../ql/exec/vector/VectorHashKeyWrapper.java    |    3 +-
 .../exec/vector/VectorHashKeyWrapperBatch.java  |    2 +-
 .../hive/ql/exec/vector/VectorSerializeRow.java |    7 +-
 .../ql/exec/vector/VectorizationContext.java    |  174 +-
 .../expressions/CastDecimalToBoolean.java       |    7 +-
 .../vector/expressions/CastDecimalToChar.java   |    4 +-
 .../vector/expressions/CastDecimalToDouble.java |    2 +-
 .../vector/expressions/CastDecimalToLong.java   |   44 +-
 .../vector/expressions/CastDecimalToString.java |   23 +-
 .../expressions/CastDecimalToTimestamp.java     |   11 +-
 .../expressions/CastDecimalToVarChar.java       |    4 +-
 .../vector/expressions/CastDoubleToDecimal.java |   10 +-
 .../vector/expressions/DecimalColumnInList.java |   21 +-
 .../ql/exec/vector/expressions/DecimalUtil.java |  281 +-
 .../expressions/FilterDecimalColumnInList.java  |   18 +-
 .../expressions/FilterStructColumnInList.java   |    2 +-
 .../vector/expressions/FuncDecimalToLong.java   |   19 +-
 .../vector/expressions/StructColumnInList.java  |    2 +-
 .../aggregates/VectorUDAFAvgDecimal.java        |   95 +-
 .../aggregates/VectorUDAFSumDecimal.java        |  148 +-
 .../mapjoin/VectorMapJoinCommonOperator.java    |   11 +-
 .../ql/io/parquet/write/DataWritableWriter.java |    3 +-
 .../hive/ql/io/sarg/ConvertAstToSearchArg.java  |    6 +-
 .../org/apache/hadoop/hive/ql/udf/UDFLog.java   |    4 +-
 .../org/apache/hadoop/hive/ql/udf/UDFMath.java  |    2 +-
 .../org/apache/hadoop/hive/ql/udf/UDFSign.java  |    6 +-
 .../apache/hadoop/hive/ql/udf/UDFToBoolean.java |    4 +-
 .../apache/hadoop/hive/ql/udf/UDFToByte.java    |    4 +-
 .../apache/hadoop/hive/ql/udf/UDFToDouble.java  |    4 +-
 .../apache/hadoop/hive/ql/udf/UDFToFloat.java   |    4 +-
 .../apache/hadoop/hive/ql/udf/UDFToInteger.java |    4 +-
 .../apache/hadoop/hive/ql/udf/UDFToLong.java    |    4 +-
 .../apache/hadoop/hive/ql/udf/UDFToShort.java   |    4 +-
 .../hive/ql/udf/generic/GenericUDAFSum.java     |   44 +-
 .../hive/ql/udf/generic/GenericUDFAbs.java      |    3 +-
 .../hive/ql/udf/generic/GenericUDFBRound.java   |    7 +-
 .../hive/ql/udf/generic/GenericUDFCeil.java     |    4 +-
 .../hive/ql/udf/generic/GenericUDFFloor.java    |    4 +-
 .../ql/udf/generic/GenericUDFOPNegative.java    |    4 +-
 .../udf/generic/GenericUDFOPNumericMinus.java   |    7 +-
 .../ql/udf/generic/GenericUDFOPNumericPlus.java |    9 +-
 .../hive/ql/udf/generic/GenericUDFRound.java    |   16 +-
 .../exec/vector/TestVectorGroupByOperator.java  |   12 +-
 .../hive/ql/exec/vector/TestVectorSerDeRow.java |  226 +-
 .../ql/exec/vector/VectorRandomRowSource.java   |   26 +-
 .../vector/expressions/TestDecimalUtil.java     |    1 -
 .../vector/expressions/TestVectorTypeCasts.java |   28 +-
 .../mapjoin/fast/CheckFastRowHashMap.java       |   27 +-
 .../exec/vector/mapjoin/fast/VerifyFastRow.java |    5 +-
 .../vector/util/VectorizedRowGroupGenUtil.java  |    6 +-
 .../test/results/clientpositive/decimal_2.q.out |    4 +-
 ...ema_evol_orc_nonvec_part_all_primitive.q.out |    4 +-
 ...schema_evol_orc_vec_part_all_primitive.q.out |    4 +-
 ...ma_evol_text_nonvec_part_all_primitive.q.out |    8 +-
 ...chema_evol_text_vec_part_all_primitive.q.out |    8 +-
 ...ma_evol_text_vecrow_part_all_primitive.q.out |    8 +-
 .../clientpositive/llap/vector_decimal_2.q.out  |    4 +-
 .../llap/vector_decimal_expressions.q.out       |   20 +-
 .../vector_decimal_expressions.q.out            |   20 +-
 .../hadoop/hive/serde2/avro/AvroSerdeUtils.java |    5 +-
 .../binarysortable/BinarySortableSerDe.java     |  138 +-
 .../fast/BinarySortableDeserializeRead.java     |   35 +-
 .../fast/BinarySortableSerializeWrite.java      |   26 +-
 .../hadoop/hive/serde2/fast/SerializeWrite.java |    5 +
 .../hive/serde2/lazy/LazyHiveDecimal.java       |   72 +-
 .../lazy/fast/LazySimpleDeserializeRead.java    |   23 +-
 .../lazy/fast/LazySimpleSerializeWrite.java     |   26 +-
 .../LazyHiveDecimalObjectInspector.java         |    9 +-
 .../lazybinary/LazyBinaryHiveDecimal.java       |    6 +-
 .../hive/serde2/lazybinary/LazyBinarySerDe.java |   74 +-
 .../fast/LazyBinaryDeserializeRead.java         |   28 +-
 .../fast/LazyBinarySerializeWrite.java          |   56 +-
 .../objectinspector/ObjectInspectorUtils.java   |    2 +
 .../PrimitiveObjectInspectorUtils.java          |   58 +-
 ...tableConstantHiveDecimalObjectInspector.java |   17 +-
 .../hive/serde2/typeinfo/HiveDecimalUtils.java  |    5 +-
 .../hive/serde2/SerdeRandomRowSource.java       |   28 +-
 .../apache/hadoop/hive/serde2/VerifyFast.java   |    2 +-
 .../hive/serde2/avro/TestAvroSerializer.java    |    8 +-
 .../binarysortable/MyTestPrimitiveClass.java    |   16 +-
 .../binarysortable/TestBinarySortableFast.java  |   43 +-
 .../hive/serde2/io/TestHiveDecimalWritable.java |  250 -
 .../hive/common/type/FastHiveDecimal.java       |  741 ++
 .../hive/common/type/FastHiveDecimalImpl.java   | 9149 ++++++++++++++++++
 .../hadoop/hive/common/type/HiveDecimal.java    | 1487 ++-
 .../hadoop/hive/common/type/HiveDecimalV1.java  |  386 +
 .../hive/common/type/HiveDecimalVersionV1.java  |   33 +
 .../hive/common/type/HiveDecimalVersionV2.java  |   33 +
 .../hadoop/hive/common/type/RandomTypeUtil.java |   27 +-
 .../ql/exec/vector/DecimalColumnVector.java     |   35 +-
 .../hadoop/hive/ql/util/TimestampUtils.java     |   83 +-
 .../hive/serde2/io/HiveDecimalWritable.java     |  956 +-
 .../hive/serde2/io/HiveDecimalWritableV1.java   |  191 +
 .../serde2/io/HiveDecimalWritableVersionV1.java |   33 +
 .../serde2/io/HiveDecimalWritableVersionV2.java |   33 +
 .../org/apache/hadoop/hive/VersionTestBase.java |  322 +
 .../hive/common/type/HiveDecimalTestBase.java   |  558 ++
 .../hive/common/type/TestHiveDecimal.java       | 3404 +++++++
 .../common/type/TestHiveDecimalVersion.java     |   67 +
 .../hive/serde2/io/TestHiveDecimalWritable.java |   52 +
 .../io/TestHiveDecimalWritableVersion.java      |   71 +
 112 files changed, 19789 insertions(+), 1663 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/common/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java b/common/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java
new file mode 100644
index 0000000..dee49f9
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java
@@ -0,0 +1,561 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.util.Random;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
+
+// A COPY of the one in storage-api since currently sharing test classes isn't supported in
+// our build.
+
+public class HiveDecimalTestBase {
+
+  public static int POUND_FACTOR = 1000;
+
+  public static enum BigDecimalFlavor {
+    NORMAL_RANGE,
+    FRACTIONS_ONLY,
+    NEGATIVE_SCALE,
+    LONG_TAIL
+  }
+
+  public static enum BigDecimalPairFlavor {
+    RANDOM,
+    NEAR,
+    INVERSE
+  }
+
+  public BigDecimal randHiveBigDecimal(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) {
+    switch (bigDecimalFlavor) {
+    case NORMAL_RANGE:
+      return randHiveBigDecimalNormalRange(r, digitAlphabet);
+    case FRACTIONS_ONLY:
+      return randHiveBigDecimalFractionsOnly(r, digitAlphabet);
+    case NEGATIVE_SCALE:
+      return randHiveBigDecimalNegativeScale(r, digitAlphabet);
+    case LONG_TAIL:
+      return randHiveBigDecimalLongTail(r, digitAlphabet);
+    default:
+      throw new RuntimeException("Unexpected big decimal flavor " + bigDecimalFlavor);
+    }
+  }
+
+  public BigDecimal[] randHiveBigDecimalPair(Random r, String digitAlphabet,
+      BigDecimalFlavor bigDecimalFlavor, BigDecimalPairFlavor bigDecimalPairFlavor) {
+    BigDecimal[] pair = new BigDecimal[2];
+    BigDecimal bigDecimal1 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor);
+    pair[0] = bigDecimal1;
+
+    BigDecimal bigDecimal2;
+    switch (bigDecimalPairFlavor) {
+    case RANDOM:
+      bigDecimal2 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor);
+      break;
+    case NEAR:
+      bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1);
+      break;
+    case INVERSE:
+      bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1);
+      break;
+    default:
+      throw new RuntimeException("Unexpected big decimal pair flavor " + bigDecimalPairFlavor);
+    }
+    pair[1] = bigDecimal2;
+    return pair;
+  }
+
+  public BigDecimal randHiveBigDecimalNormalRange(Random r, String digitAlphabet) {
+    String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38));
+    BigInteger bigInteger = new BigInteger(digits);
+    boolean negated = false;
+    if (r.nextBoolean()) {
+      bigInteger = bigInteger.negate();
+      negated = true;
+    }
+    int scale = 0 + r.nextInt(38 + 1);
+    return new BigDecimal(bigInteger, scale);
+  }
+
+  public BigDecimal randHiveBigDecimalNegativeScale(Random r, String digitAlphabet) {
+    String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38));
+    BigInteger bigInteger = new BigInteger(digits);
+    boolean negated = false;
+    if (r.nextBoolean()) {
+      bigInteger = bigInteger.negate();
+      negated = true;
+    }
+    int scale = 0 + (r.nextBoolean() ? 0 : r.nextInt(38 + 1));
+    if (r.nextBoolean()) {
+      scale = -scale;
+    }
+    return new BigDecimal(bigInteger, scale);
+  }
+
+  public BigDecimal randHiveBigDecimalLongTail(Random r, String digitAlphabet) {
+    int scale = 0 + r.nextInt(38 + 20);
+    final int maxDigits = 38 + (scale == 0 ? 0 : 20);
+    String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(maxDigits));
+    BigInteger bigInteger = new BigInteger(digits);
+    boolean negated = false;
+    if (r.nextBoolean()) {
+      bigInteger = bigInteger.negate();
+      negated = true;
+    }
+    return new BigDecimal(bigInteger, scale);
+  }
+
+  public BigDecimal randHiveBigDecimalFractionsOnly(Random r, String digitAlphabet) {
+    int scale = 1 + r.nextInt(38 + 1);
+    String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(scale));
+    BigInteger bigInteger = new BigInteger(digits);
+    boolean negated = false;
+    if (r.nextBoolean()) {
+      bigInteger = bigInteger.negate();
+      negated = true;
+    }
+    return new BigDecimal(bigInteger, scale);
+  }
+
+  public BigDecimal randHiveBigDecimalNear(Random r, BigDecimal bigDecimal) {
+
+    int scale = bigDecimal.scale();
+    int delta = r.nextInt(10);
+    if (r.nextBoolean()) {
+      return bigDecimal.add(new BigDecimal(BigInteger.valueOf(delta), scale));
+    } else {
+      return bigDecimal.subtract(new BigDecimal(BigInteger.valueOf(delta), scale));
+    }
+  }
+
+  public BigDecimal randHiveBigDecimalInverse(Random r, BigDecimal bigDecimal) {
+    if (bigDecimal.signum() == 0) {
+      return bigDecimal;
+    }
+    return BigDecimal.ONE.divide(bigDecimal);
+  }
+
+  public BigInteger randHiveBigInteger(Random r, String digitAlphabet) {
+    String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38));
+    BigInteger bigInteger = new BigInteger(digits);
+    boolean negated = false;
+    if (r.nextBoolean()) {
+      bigInteger = bigInteger.negate();
+      negated = true;
+    }
+    return bigInteger;
+  }
+
+  public boolean isTenPowerBug(String string) {
+    // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string);
+    if (string.charAt(0) == '-') {
+      string = string.substring(1);
+    }
+    int index = string.indexOf('.');
+    if (index != -1) {
+      if (index == 0) {
+        string = string.substring(1);
+      } else {
+        string = string.substring(0, index) + string.substring(index + 1);
+      }
+    }
+    // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string);
+    return string.equals("100000000000000000000000000000000000000");
+  }
+
+  //------------------------------------------------------------------------------------------------
+
+  public static String[] specialDecimalStrings = new String[] {
+    "0",
+    "1",
+    "-1",
+    "10",
+    "-10",
+    "100",
+    "-100",
+    "127",                                          // Byte.MAX_VALUE
+    "127.1",
+    "127.0008",
+    "127.49",
+    "127.5",
+    "127.9999999999999999999",
+    "-127",
+    "-127.1",
+    "-127.0008",
+    "-127.49",
+    "-127.5",
+    "-127.999999",
+    "128",
+    "128.1",
+    "128.0008",
+    "128.49",
+    "128.5",
+    "128.9999999999999999999",
+    "-128",                                         // Byte.MIN_VALUE
+    "-128.1",
+    "-128.0008",
+    "-128.49",
+    "-128.5",
+    "-128.999",
+    "129",
+    "129.1",
+    "-129",
+    "-129.1",
+    "1000",
+    "-1000",
+    "10000",
+    "-10000",
+    "32767",                                        // Short.MAX_VALUE
+    "32767.1",
+    "32767.0008",
+    "32767.49",
+    "32767.5",
+    "32767.99999999999",
+    "-32767",
+    "-32767.1",
+    "-32767.0008",
+    "-32767.49",
+    "-32767.5",
+    "-32767.9",
+    "32768",
+    "32768.1",
+    "32768.0008",
+    "32768.49",
+    "32768.5",
+    "32768.9999999999",
+    "-32768",                                       // Short.MIN_VALUE
+    "-32768.1",
+    "-32768.0008",
+    "-32768.49",
+    "-32768.5",
+    "-32768.9999999",
+    "32769",
+    "32769.1",
+    "-32769",
+    "-32769.1",
+    "100000",
+    "-100000",
+    "1000000",
+    "-1000000",
+    "10000000",
+    "-10000000",
+    "100000000",
+    "99999999",                                     // 10^8 - 1
+    "-99999999",
+    "-100000000",
+    "1000000000",
+    "-1000000000",
+    "2147483647",                                  // Integer.MAX_VALUE
+    "2147483647.1",
+    "2147483647.0008",
+    "2147483647.49",
+    "2147483647.5",
+    "2147483647.9999999999",
+    "-2147483647",
+    "-2147483647.1",
+    "-2147483647.0008",
+    "-2147483647.49",
+    "-2147483647.5",
+    "-2147483647.9999999999999999999",
+    "2147483648",
+    "2147483648.1",
+    "2147483648.0008",
+    "2147483648.49",
+    "2147483648.5",
+    "2147483648.9",
+    "-2147483648",                                 // Integer.MIN_VALUE
+    "-2147483648.1",
+    "-2147483648.0008",
+    "-2147483648.49",
+    "-2147483648.5",
+    "-2147483648.999",
+    "2147483649",
+    "2147483649.1",
+    "-2147483649",
+    "-2147483649.1",
+    "10000000000",
+    "-10000000000",
+    "100000000000",
+    "-100000000000",
+    "1000000000000",
+    "-1000000000000",
+    "10000000000000",
+    "-10000000000000",
+    "100000000000000",
+    "-100000000000000",
+    "999999999999999",
+    "-999999999999999",
+    "1000000000000000",                            // 10^15
+    "-1000000000000000",
+    "9999999999999999",                            // 10^16 - 1
+    "-9999999999999999",
+    "10000000000000000",                           // 10^16
+    "-10000000000000000",
+    "100000000000000000",
+    "-100000000000000000",
+    "1000000000000000000",
+    "-1000000000000000000",
+    "9223372036854775807",                         // Long.MAX_VALUE
+    "9223372036854775807.1",
+    "9223372036854775807.0008",
+    "9223372036854775807.49",
+    "9223372036854775807.5",
+    "9223372036854775807.9",
+    "-9223372036854775807",
+    "-9223372036854775807.1",
+    "-9223372036854775807.0008",
+    "-9223372036854775807.49",
+    "-9223372036854775807.5",
+    "-9223372036854775807.9999999999999999999",
+    "-9223372036854775808",
+    "-9223372036854775808.1",
+    "9223372036854775808",
+    "9223372036854775808.1",
+    "9223372036854775808.0008",
+    "9223372036854775808.49",
+    "9223372036854775808.5",
+    "9223372036854775808.9",
+    "9223372036854775809",
+    "9223372036854775809.1",
+    "-9223372036854775808",                        // Long.MIN_VALUE
+    "-9223372036854775808.1",
+    "-9223372036854775808.0008",
+    "-9223372036854775808.49",
+    "-9223372036854775808.5",
+    "-9223372036854775808.9999999",
+    "9223372036854775809",
+    "9223372036854775809.1",
+    "-9223372036854775809",
+    "-9223372036854775809.1",
+    "10000000000000000000000000000000",            // 10^31
+    "-10000000000000000000000000000000",
+    "99999999999999999999999999999999",            // 10^32 - 1
+    "-99999999999999999999999999999999", 
+    "100000000000000000000000000000000",           // 10^32
+    "-100000000000000000000000000000000",
+    "10000000000000000000000000000000000000",      // 10^37
+    "-10000000000000000000000000000000000000",
+    "99999999999999999999999999999999999999",      // 10^38 - 1
+    "-99999999999999999999999999999999999999",
+    "100000000000000000000000000000000000000",     // 10^38
+    "-100000000000000000000000000000000000000", 
+    "1000000000000000000000000000000000000000",    // 10^39
+    "-1000000000000000000000000000000000000000",
+
+    "18446744073709551616",                        // Unsigned 64 max.
+    "-18446744073709551616",
+    "340282366920938463463374607431768211455",     // 2^128 - 1
+    "-340282366920938463463374607431768211455",
+
+    "0.999999999999999",
+    "-0.999999999999999",
+    "0.0000000000000001",                          // 10^-15
+    "-0.0000000000000001",
+    "0.9999999999999999",
+    "-0.9999999999999999",
+    "0.00000000000000001",                         // 10^-16
+    "-0.00000000000000001",
+    "0.00000000000000000000000000000001",          // 10^-31
+    "-0.00000000000000000000000000000001",
+    "0.99999999999999999999999999999999",          // 10^-32 + 1
+    "-0.99999999999999999999999999999999",
+    "0.000000000000000000000000000000001",         // 10^-32
+    "-0.000000000000000000000000000000001",
+    "0.00000000000000000000000000000000000001",    // 10^-37
+    "-0.00000000000000000000000000000000000001",
+    "0.99999999999999999999999999999999999999",    // 10^-38 + 1
+    "-0.99999999999999999999999999999999999999",
+    "0.000000000000000000000000000000000000001",   // 10^-38
+    "-0.000000000000000000000000000000000000001",
+    "0.0000000000000000000000000000000000000001",  // 10^-39
+    "-0.0000000000000000000000000000000000000001",
+    "0.0000000000000000000000000000000000000005",  // 10^-39  (rounds)
+    "-0.0000000000000000000000000000000000000005",
+    "0.340282366920938463463374607431768211455",   // (2^128 - 1) * 10^-39
+    "-0.340282366920938463463374607431768211455",
+    "0.000000000000000000000000000000000000001",   // 10^-38
+    "-0.000000000000000000000000000000000000001",
+    "0.000000000000000000000000000000000000005",   // 10^-38
+    "-0.000000000000000000000000000000000000005",
+
+    "234.79",
+    "342348.343",
+    "12.25",
+    "-12.25",
+    "72057594037927935",                           // 2^56 - 1
+    "-72057594037927935",
+    "72057594037927936",                           // 2^56
+    "-72057594037927936",
+    "5192296858534827628530496329220095",          // 2^56 * 2^56 - 1
+    "-5192296858534827628530496329220095",
+    "5192296858534827628530496329220096",          // 2^56 * 2^56
+    "-5192296858534827628530496329220096",
+
+    "54216721532321902598.70",
+    "-906.62545207002374150309544832320",
+    "-0.0709351061072",
+    "1460849063411925.53",
+    "8.809130E-33",
+    "-4.0786300706013636202E-20",
+    "-3.8823936518E-1",
+    "-3.8823936518E-28",
+    "-3.8823936518E-29",
+    "598575157855521918987423259.94094",
+    "299999448432.001342152474197",
+    "1786135888657847525803324040144343378.09799306448796128931113691624",  // More than 38 digits.
+    "-1786135888657847525803324040144343378.09799306448796128931113691624",
+    "57847525803324040144343378.09799306448796128931113691624",
+    "0.999999999999999999990000",
+    "005.34000",
+    "1E-90",
+
+    "0.4",
+    "-0.4",
+    "0.5",
+    "-0.5",
+    "0.6",
+    "-0.6",
+    "1.4",
+    "-1.4",
+    "1.5",
+    "-1.5",
+    "1.6",
+    "-1.6",
+    "2.4",
+    "-2.4",
+    "2.49",
+    "-2.49",
+    "2.5",
+    "-2.5",
+    "2.51",
+    "-2.51",
+    "-2.5",
+    "2.6",
+    "-2.6",
+    "3.00001415926",
+    "0.00",
+    "-12.25",
+    "234.79"
+  };
+
+  public static BigDecimal[] specialBigDecimals = stringArrayToBigDecimals(specialDecimalStrings);
+
+  // decimal_1_1.txt
+  public static String[] decimal_1_1_txt = {
+    "0.0",
+    "0.0000",
+    ".0",
+    "0.1",
+    "0.15",
+    "0.9",
+    "0.94",
+    "0.99",
+    "0.345",
+    "1.0",
+    "1",
+    "0",
+    "00",
+    "22",
+    "1E-9",
+    "-0.0",
+    "-0.0000",
+    "-.0",
+    "-0.1",
+    "-0.15",
+    "-0.9",
+    "-0.94",
+    "-0.99",
+    "-0.345",
+    "-1.0",
+    "-1",
+    "-0",
+    "-00",
+    "-22",
+    "-1E-9"
+  };
+
+  // kv7.txt KEYS
+  public static String[] kv7_txt_keys = {
+    "-4400",
+    "1E+99",
+    "1E-99",
+    "0",
+    "100",
+    "10",
+    "1",
+    "0.1",
+    "0.01",
+    "200",
+    "20",
+    "2",
+    "0",
+    "0.2",
+    "0.02",
+    "0.3",
+    "0.33",
+    "0.333",
+    "-0.3",
+    "-0.33",
+    "-0.333",
+    "1.0",
+    "2",
+    "3.14",
+    "-1.12",
+    "-1.12",
+    "-1.122",
+    "1.12",
+    "1.122",
+    "124.00",
+    "125.2",
+    "-1255.49",
+    "3.14",
+    "3.14",
+    "3.140",
+    "0.9999999999999999999999999",
+    "-1234567890.1234567890",
+    "1234567890.1234567800"
+  };
+
+  public static String standardAlphabet = "0123456789";
+
+  public static String[] sparseAlphabets = new String[] {
+
+    "0000000000000000000000000000000000000003",
+    "0000000000000000000000000000000000000009",
+    "0000000000000000000000000000000000000001",
+    "0000000000000000000003",
+    "0000000000000000000009",
+    "0000000000000000000001",
+    "0000000000091",
+    "000000000005",
+    "9",
+    "5555555555999999999000000000000001111111",
+    "24680",
+    "1"
+  };
+
+  public static BigDecimal[] stringArrayToBigDecimals(String[] strings) {
+    BigDecimal[] result = new BigDecimal[strings.length];
+    for (int i = 0; i < strings.length; i++) {
+      result[i] = new BigDecimal(strings[i]);
+    }
+    return result;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
deleted file mode 100644
index f68842c..0000000
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.type;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-
-import com.google.code.tempusfugit.concurrency.annotations.*;
-import com.google.code.tempusfugit.concurrency.*;
-import org.junit.*;
-import static org.junit.Assert.*;
-
-public class TestHiveDecimal {
-
-  @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
-  @Rule public RepeatingRule repeatingRule = new RepeatingRule();
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testPrecisionScaleEnforcement() {
-    String decStr = "1786135888657847525803324040144343378.09799306448796128931113691624";
-    HiveDecimal dec = HiveDecimal.create(decStr);
-    Assert.assertEquals("1786135888657847525803324040144343378.1", dec.toString());
-    Assert.assertTrue("Decimal precision should not go above maximum",
-        dec.precision() <= HiveDecimal.MAX_PRECISION);
-    Assert.assertTrue("Decimal scale should not go above maximum", dec.scale() <= HiveDecimal.MAX_SCALE);
-
-    decStr = "57847525803324040144343378.09799306448796128931113691624";
-    HiveDecimal bd = HiveDecimal.create(decStr);
-    HiveDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
-    Assert.assertNull(bd1);
-    bd1 = HiveDecimal.enforcePrecisionScale(bd, 35, 5);
-    Assert.assertEquals("57847525803324040144343378.09799", bd1.toString());
-    bd1 = HiveDecimal.enforcePrecisionScale(bd, 45, 20);
-    Assert.assertNull(bd1);
-
-    dec = HiveDecimal.create(new BigDecimal(decStr), false);
-    Assert.assertNull(dec);
-
-    dec = HiveDecimal.create("-1786135888657847525803324040144343378.09799306448796128931113691624");
-    Assert.assertEquals("-1786135888657847525803324040144343378.1", dec.toString());
-
-    dec = HiveDecimal.create("005.34000");
-    Assert.assertEquals(dec.precision(), 3);
-    Assert.assertEquals(dec.scale(), 2);
-
-    dec = HiveDecimal.create("178613588865784752580332404014434337809799306448796128931113691624");
-    Assert.assertNull(dec);
-
-    // Rounding numbers that increase int digits
-    Assert.assertEquals("10",
-        HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 2, 0).toString());
-    Assert.assertNull(HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 1, 0));
-    Assert.assertEquals("9",
-        HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.4"), 1, 0).toString());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testTrailingZeroRemovalAfterEnforcement() {
-    String decStr = "8.090000000000000000000000000000000000000123456";
-    HiveDecimal dec = HiveDecimal.create(decStr);
-    Assert.assertEquals("8.09", dec.toString());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testMultiply() {
-    HiveDecimal dec1 = HiveDecimal.create("0.00001786135888657847525803");
-    HiveDecimal dec2 = HiveDecimal.create("3.0000123456789");
-    Assert.assertNull(dec1.multiply(dec2));
-
-    dec1 = HiveDecimal.create("178613588865784752580323232232323444.4");
-    dec2 = HiveDecimal.create("178613588865784752580302323232.3");
-    Assert.assertNull(dec1.multiply(dec2));
-
-    dec1 = HiveDecimal.create("47.324");
-    dec2 = HiveDecimal.create("9232.309");
-    Assert.assertEquals("436909.791116", dec1.multiply(dec2).toString());
-
-    dec1 = HiveDecimal.create("3.140");
-    dec2 = HiveDecimal.create("1.00");
-    Assert.assertEquals("3.14", dec1.multiply(dec2).toString());
-
-    dec1 = HiveDecimal.create("43.010");
-    dec2 = HiveDecimal.create("2");
-    Assert.assertEquals("86.02", dec1.multiply(dec2).toString());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testPow() {
-    HiveDecimal dec = HiveDecimal.create("3.00001415926");
-    Assert.assertEquals(dec.pow(2), dec.multiply(dec));
-
-    HiveDecimal dec1 = HiveDecimal.create("0.000017861358882");
-    dec1 = dec1.pow(3);
-    Assert.assertNull(dec1);
-
-    dec1 = HiveDecimal.create("3.140");
-    Assert.assertEquals("9.8596", dec1.pow(2).toString());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testDivide() {
-    HiveDecimal dec1 = HiveDecimal.create("3.14");
-    HiveDecimal dec2 = HiveDecimal.create("3");
-    Assert.assertNotNull(dec1.divide(dec2));
-
-    dec1 = HiveDecimal.create("15");
-    dec2 = HiveDecimal.create("5");
-    Assert.assertEquals("3", dec1.divide(dec2).toString());
-
-    dec1 = HiveDecimal.create("3.140");
-    dec2 = HiveDecimal.create("1.00");
-    Assert.assertEquals("3.14", dec1.divide(dec2).toString());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testPlus() {
-    HiveDecimal dec1 = HiveDecimal.create("99999999999999999999999999999999999");
-    HiveDecimal dec2 = HiveDecimal.create("1");
-    Assert.assertNotNull(dec1.add(dec2));
-
-    dec1 = HiveDecimal.create("3.140");
-    dec2 = HiveDecimal.create("1.00");
-    Assert.assertEquals("4.14", dec1.add(dec2).toString());
-  }
-
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testSubtract() {
-      HiveDecimal dec1 = HiveDecimal.create("3.140");
-      HiveDecimal dec2 = HiveDecimal.create("1.00");
-      Assert.assertEquals("2.14", dec1.subtract(dec2).toString());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testPosMod() {
-    HiveDecimal hd1 = HiveDecimal.create("-100.91");
-    HiveDecimal hd2 = HiveDecimal.create("9.8");
-    HiveDecimal dec = hd1.remainder(hd2).add(hd2).remainder(hd2);
-    Assert.assertEquals("6.89", dec.toString());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testHashCode() {
-      Assert.assertEquals(HiveDecimal.create("9").hashCode(), HiveDecimal.create("9.00").hashCode());
-      Assert.assertEquals(HiveDecimal.create("0").hashCode(), HiveDecimal.create("0.00").hashCode());
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testException() {
-    HiveDecimal dec = HiveDecimal.create("3.1415.926");
-    Assert.assertNull(dec);
-    dec = HiveDecimal.create("3abc43");
-    Assert.assertNull(dec);
-  }
-
-  @Test
-  @Concurrent(count=4)
-  @Repeating(repetition=100)
-  public void testBinaryConversion() {
-    testBinaryConversion("0.00");
-    testBinaryConversion("-12.25");
-    testBinaryConversion("234.79");
-  }
-
-  private void testBinaryConversion(String num) {
-    HiveDecimal dec = HiveDecimal.create(num);
-    int scale = 2;
-    byte[] d = dec.setScale(2).unscaledValue().toByteArray();
-    Assert.assertEquals(dec, HiveDecimal.create(new BigInteger(d), scale));
-    int prec = 5;
-    int len =  (int)
-        Math.ceil((Math.log(Math.pow(10, prec) - 1) / Math.log(2) + 1) / 8);
-    byte[] res = new byte[len];
-    if ( dec.signum() == -1)
-      for (int i = 0; i < len; i++)
-        res[i] |= 0xFF;
-    System.arraycopy(d, 0, res, len-d.length, d.length); // Padding leading zeros.
-    Assert.assertEquals(dec, HiveDecimal.create(new BigInteger(res), scale));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalOrcSerializationUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalOrcSerializationUtils.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalOrcSerializationUtils.java
new file mode 100644
index 0000000..882ff86
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalOrcSerializationUtils.java
@@ -0,0 +1,378 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.sql.Timestamp;
+import java.util.Random;
+import java.util.Arrays;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritableV1;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.orc.impl.SerializationUtils;
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
+import org.apache.hadoop.hive.ql.util.TimestampUtils;
+
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+
+import org.junit.*;
+
+import static org.junit.Assert.*;
+
+public class TestHiveDecimalOrcSerializationUtils extends HiveDecimalTestBase {
+
+  //------------------------------------------------------------------------------------------------
+
+  @Test
+  @Concurrent(count=4)
+  public void testSerializationUtilsWriteRead() {
+    testSerializationUtilsWriteRead("0.00");
+    testSerializationUtilsWriteRead("1");
+    testSerializationUtilsWriteRead("234.79");
+    testSerializationUtilsWriteRead("-12.25");
+    testSerializationUtilsWriteRead("99999999999999999999999999999999");
+    testSerializationUtilsWriteRead("-99999999999999999999999999999999");
+    testSerializationUtilsWriteRead("99999999999999999999999999999999999999");
+    //                               12345678901234567890123456789012345678
+    testSerializationUtilsWriteRead("-99999999999999999999999999999999999999");
+    testSerializationUtilsWriteRead("999999999999.99999999999999999999");
+    testSerializationUtilsWriteRead("-999999.99999999999999999999999999");
+    testSerializationUtilsWriteRead("9999999999999999999999.9999999999999999");
+    testSerializationUtilsWriteRead("-9999999999999999999999999999999.9999999");
+
+    testSerializationUtilsWriteRead("4611686018427387903");  // 2^62 - 1
+    testSerializationUtilsWriteRead("-4611686018427387903");
+    testSerializationUtilsWriteRead("4611686018427387904");  // 2^62
+    testSerializationUtilsWriteRead("-4611686018427387904");
+
+    testSerializationUtilsWriteRead("42535295865117307932921825928971026431");  // 2^62*2^63 - 1
+    testSerializationUtilsWriteRead("-42535295865117307932921825928971026431");
+    testSerializationUtilsWriteRead("42535295865117307932921825928971026432");  // 2^62*2^63
+    testSerializationUtilsWriteRead("-42535295865117307932921825928971026432");
+
+    testSerializationUtilsWriteRead("54216721532321902598.70");
+    testSerializationUtilsWriteRead("-906.62545207002374150309544832320");
+  }
+
+  private void testSerializationUtilsWriteRead(String string) {
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ START ~~~~~~~~~~~~~~~~~");
+
+    HiveDecimal dec = HiveDecimal.create(string);
+    assertTrue(dec != null);
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER dec " + dec.toString());
+
+    BigInteger bigInteger = dec.unscaledValue();
+    int scale = dec.scale();
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER bigInteger " + bigInteger.toString());
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER scale " + scale);
+
+    //---------------------------------------------------
+    HiveDecimalV1 oldDec = HiveDecimalV1.create(string);
+    assertTrue(oldDec != null);
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER oldDec " + oldDec.toString());
+
+    BigInteger oldBigInteger = oldDec.unscaledValue();
+    int oldScale = oldDec.scale();
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER oldBigInteger " + oldBigInteger.toString());
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER oldScale " + oldScale);
+    //---------------------------------------------------
+
+    long[] scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN];
+
+    int which = 0;
+    try {
+      ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+      if (!dec.serializationUtilsWrite(
+          outputStream, scratchLongs)) {
+        // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER serializationUtilsWrite failed");
+        fail();
+      }
+      byte[] bytes = outputStream.toByteArray();
+  
+      ByteArrayOutputStream outputStreamExpected = new ByteArrayOutputStream();
+      SerializationUtils.writeBigInteger(outputStreamExpected, bigInteger);
+      byte[] bytesExpected = outputStreamExpected.toByteArray();
+  
+      // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER check streams");
+      // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER bytes1        " + displayBytes(bytes, 0, bytes.length));
+      if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) {
+        // Tailing zeroes difference ok.
+        // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER streams not equal");
+        // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER bytesExpected " + displayBytes(bytesExpected, 0, bytesExpected.length));
+      }
+      // Deserialize and check...
+      which = 1;
+      ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
+      BigInteger deserializedBigInteger = SerializationUtils.readBigInteger(byteArrayInputStream);
+
+      which = 2;
+      ByteArrayInputStream byteArrayInputStreamExpected = new ByteArrayInputStream(bytesExpected);
+      BigInteger deserializedBigIntegerExpected = SerializationUtils.readBigInteger(byteArrayInputStreamExpected);
+      // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER deserialized equals " +
+      //    deserializedBigInteger.equals(deserializedBigIntegerExpected));
+      if (!deserializedBigInteger.equals(deserializedBigIntegerExpected)) {
+        // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER deserializedBigInteger " + deserializedBigInteger.toString() +
+        //    " deserializedBigIntegerExpected " + deserializedBigIntegerExpected.toString());
+        fail();
+      }
+
+      which = 3;
+      ByteArrayInputStream byteArrayInputStreamRead = new ByteArrayInputStream(bytes);
+      byte[] scratchBytes = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ];
+      HiveDecimal readHiveDecimal =
+          HiveDecimal.serializationUtilsRead(byteArrayInputStreamRead, scale, scratchBytes);
+      assertTrue(readHiveDecimal != null);
+      // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER read readHiveDecimal " + readHiveDecimal.toString() +
+      //    " dec " + dec.toString() + " (scale parameter " + scale + ")");
+      // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER read toString equals " +
+      //    readHiveDecimal.toString().equals(dec.toString()));
+      assertEquals(readHiveDecimal.toString(), dec.toString());
+      // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER read equals " +
+      //    readHiveDecimal.equals(dec));
+      assertEquals(readHiveDecimal, dec);
+    } catch (IOException e) {
+      // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER " + e + " which " + which);
+      fail();
+    }
+    // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~  END  ~~~~~~~~~~~~~~~~~");
+
+  }
+
+  //------------------------------------------------------------------------------------------------
+
+  @Test
+  public void testRandomSerializationUtilsRead()
+      throws IOException {
+    doTestRandomSerializationUtilsRead(standardAlphabet);
+  }
+
+  @Test
+  public void testRandomSerializationUtilsReadSparse()
+      throws IOException {
+    for (String digitAlphabet : sparseAlphabets) {
+      doTestRandomSerializationUtilsRead(digitAlphabet);
+    }
+  }
+
+  private void doTestRandomSerializationUtilsRead(String digitAlphabet)
+      throws IOException {
+
+    Random r = new Random(2389);
+    for (int i = 0; i < POUND_FACTOR; i++) {
+      BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet);
+
+      doTestSerializationUtilsRead(r, bigInteger);
+    }
+  }
+
+  @Test
+  public void testSerializationUtilsReadSpecial()
+      throws IOException {
+    Random r = new Random(9923);
+    for (BigDecimal bigDecimal : specialBigDecimals) {
+      doTestSerializationUtilsRead(r, bigDecimal.unscaledValue());
+    }
+  }
+
+  private void doTestSerializationUtilsRead(Random r, BigInteger bigInteger)
+     throws IOException {
+
+    // System.out.println("TEST_SERIALIZATION_UTILS_READ bigInteger " + bigInteger);
+
+    HiveDecimalV1 oldDec = HiveDecimalV1.create(bigInteger);
+    if (oldDec != null && isTenPowerBug(oldDec.toString())) {
+      return;
+    }
+    HiveDecimal dec = HiveDecimal.create(bigInteger);
+    if (oldDec == null) {
+      assertTrue(dec == null);
+      return;
+    }
+    assertTrue(dec != null);
+    dec.validate();
+    // System.out.println("TEST_SERIALIZATION_UTILS_READ oldDec " + oldDec);
+    // System.out.println("TEST_SERIALIZATION_UTILS_READ dec " + dec);
+
+    Assert.assertEquals(bigInteger, oldDec.unscaledValue());
+    Assert.assertEquals(bigInteger, dec.unscaledValue());
+
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    SerializationUtils.writeBigInteger(outputStream, bigInteger);
+    byte[] bytes = outputStream.toByteArray();
+
+    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
+    BigInteger deserializedBigInteger =
+        SerializationUtils.readBigInteger(byteArrayInputStream);
+
+    // Verify SerializationUtils first.
+    Assert.assertEquals(bigInteger, deserializedBigInteger);
+
+    // Now HiveDecimal
+    byte[] scratchBytes = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ];
+
+    byteArrayInputStream = new ByteArrayInputStream(bytes);
+    HiveDecimal resultDec =
+        dec.serializationUtilsRead(
+            byteArrayInputStream, dec.scale(),
+            scratchBytes);
+    assertTrue(resultDec != null);
+    resultDec.validate();
+
+    Assert.assertEquals(dec.toString(), resultDec.toString());
+
+    //----------------------------------------------------------------------------------------------
+
+    // Add scale.
+
+    int scale = 0 + r.nextInt(38 + 1);
+    BigDecimal bigDecimal = new BigDecimal(bigInteger, scale);
+
+    oldDec = HiveDecimalV1.create(bigDecimal);
+    dec = HiveDecimal.create(bigDecimal);
+    if (oldDec == null) {
+      assertTrue(dec == null);
+      return;
+    }
+    assertTrue(dec != null);
+    dec.validate();
+    // System.out.println("TEST_SERIALIZATION_UTILS_READ with scale oldDec " + oldDec);
+    // System.out.println("TEST_SERIALIZATION_UTILS_READ with scale dec " + dec);
+
+    outputStream = new ByteArrayOutputStream();
+    SerializationUtils.writeBigInteger(outputStream, dec.unscaledValue());
+    bytes = outputStream.toByteArray();
+
+    // Now HiveDecimal
+    byteArrayInputStream = new ByteArrayInputStream(bytes);
+    resultDec =
+        dec.serializationUtilsRead(
+            byteArrayInputStream, dec.scale(),
+            scratchBytes);
+    assertTrue(resultDec != null);
+    resultDec.validate();
+
+    Assert.assertEquals(dec.toString(), resultDec.toString());
+  }
+
+  //------------------------------------------------------------------------------------------------
+
+  @Test
+  public void testRandomSerializationUtilsWrite()
+      throws IOException {
+    doTestRandomSerializationUtilsWrite(standardAlphabet, false);
+  }
+
+  @Test
+  public void testRandomSerializationUtilsWriteFractionsOnly()
+      throws IOException {
+    doTestRandomSerializationUtilsWrite(standardAlphabet, true);
+  }
+
+  @Test
+  public void testRandomSerializationUtilsWriteSparse()
+      throws IOException {
+    for (String digitAlphabet : sparseAlphabets) {
+      doTestRandomSerializationUtilsWrite(digitAlphabet, false);
+    }
+  }
+
+  private void doTestRandomSerializationUtilsWrite(String digitAlphabet, boolean fractionsOnly)
+      throws IOException {
+
+    Random r = new Random(823);
+    for (int i = 0; i < POUND_FACTOR; i++) {
+      BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet);
+
+      doTestSerializationUtilsWrite(r, bigInteger);
+    }
+  }
+
+  @Test
+  public void testSerializationUtilsWriteSpecial()
+      throws IOException {
+    Random r = new Random(998737);
+    for (BigDecimal bigDecimal : specialBigDecimals) {
+      doTestSerializationUtilsWrite(r, bigDecimal.unscaledValue());
+    }
+  }
+
+  private void doTestSerializationUtilsWrite(Random r, BigInteger bigInteger)
+     throws IOException {
+
+    // System.out.println("TEST_SERIALIZATION_UTILS_WRITE bigInteger " + bigInteger);
+
+    HiveDecimalV1 oldDec = HiveDecimalV1.create(bigInteger);
+    if (oldDec != null && isTenPowerBug(oldDec.toString())) {
+      return;
+    }
+    HiveDecimal dec = HiveDecimal.create(bigInteger);
+    if (oldDec == null) {
+      assertTrue(dec == null);
+      return;
+    }
+    assertTrue(dec != null);
+    dec.validate();
+    // System.out.println("TEST_SERIALIZATION_UTILS_WRITE oldDec " + oldDec);
+    // System.out.println("TEST_SERIALIZATION_UTILS_WRITE dec " + dec);
+
+    Assert.assertEquals(bigInteger, oldDec.unscaledValue());
+    Assert.assertEquals(bigInteger, dec.unscaledValue());
+
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    SerializationUtils.writeBigInteger(outputStream, bigInteger);
+    byte[] bytes = outputStream.toByteArray();
+
+    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
+    BigInteger deserializedBigInteger =
+        SerializationUtils.readBigInteger(byteArrayInputStream);
+
+    // Verify SerializationUtils first.
+    Assert.assertEquals(bigInteger, deserializedBigInteger);
+
+    ByteArrayOutputStream decOutputStream = new ByteArrayOutputStream();
+
+    long[] scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN];
+
+    boolean successful =
+        dec.serializationUtilsWrite(
+            decOutputStream, scratchLongs);
+    Assert.assertTrue(successful);
+    byte[] decBytes = decOutputStream.toByteArray();
+
+    if (!StringExpr.equal(bytes, 0, bytes.length, decBytes, 0, decBytes.length)) {
+      // Tailing zeroes difference ok...
+      // System.out.println("TEST_SERIALIZATION_UTILS_WRITE streams not equal");
+      // System.out.println("TEST_SERIALIZATION_UTILS_WRITE bytes " + displayBytes(bytes, 0, bytes.length));
+      // System.out.println("TEST_SERIALIZATION_UTILS_WRITE decBytes " + displayBytes(decBytes, 0, decBytes.length));
+    }
+
+    ByteArrayInputStream decByteArrayInputStream = new ByteArrayInputStream(decBytes);
+    BigInteger decDeserializedBigInteger =
+        SerializationUtils.readBigInteger(decByteArrayInputStream);
+
+    Assert.assertEquals(bigInteger, decDeserializedBigInteger);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java b/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
index 745ed9a..1118c5c 100644
--- a/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
+++ b/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
@@ -20,6 +20,7 @@ package org.apache.orc.impl;
 import java.sql.Date;
 import java.sql.Timestamp;
 
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.serde2.io.DateWritable;
 import org.apache.hadoop.io.BytesWritable;
@@ -573,9 +574,11 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
 
   private static final class DecimalStatisticsImpl extends ColumnStatisticsImpl
       implements DecimalColumnStatistics {
-    private HiveDecimal minimum = null;
-    private HiveDecimal maximum = null;
-    private HiveDecimal sum = HiveDecimal.ZERO;
+
+    // These objects are mutable for better performance.
+    private HiveDecimalWritable minimum = null;
+    private HiveDecimalWritable maximum = null;
+    private HiveDecimalWritable sum = new HiveDecimalWritable(0);
 
     DecimalStatisticsImpl() {
     }
@@ -584,13 +587,13 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
       super(stats);
       OrcProto.DecimalStatistics dec = stats.getDecimalStatistics();
       if (dec.hasMaximum()) {
-        maximum = HiveDecimal.create(dec.getMaximum());
+        maximum = new HiveDecimalWritable(dec.getMaximum());
       }
       if (dec.hasMinimum()) {
-        minimum = HiveDecimal.create(dec.getMinimum());
+        minimum = new HiveDecimalWritable(dec.getMinimum());
       }
       if (dec.hasSum()) {
-        sum = HiveDecimal.create(dec.getSum());
+        sum = new HiveDecimalWritable(dec.getSum());
       } else {
         sum = null;
       }
@@ -601,21 +604,21 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
       super.reset();
       minimum = null;
       maximum = null;
-      sum = HiveDecimal.ZERO;
+      sum = new HiveDecimalWritable(0);
     }
 
     @Override
-    public void updateDecimal(HiveDecimal value) {
+    public void updateDecimal(HiveDecimalWritable value) {
       if (minimum == null) {
-        minimum = value;
-        maximum = value;
+        minimum = new HiveDecimalWritable(value);
+        maximum = new HiveDecimalWritable(value);
       } else if (minimum.compareTo(value) > 0) {
-        minimum = value;
+        minimum.set(value);
       } else if (maximum.compareTo(value) < 0) {
-        maximum = value;
+        maximum.set(value);
       }
       if (sum != null) {
-        sum = sum.add(value);
+        sum.mutateAdd(value);
       }
     }
 
@@ -624,20 +627,20 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
       if (other instanceof DecimalStatisticsImpl) {
         DecimalStatisticsImpl dec = (DecimalStatisticsImpl) other;
         if (minimum == null) {
-          minimum = dec.minimum;
-          maximum = dec.maximum;
+          minimum = (dec.minimum != null ? new HiveDecimalWritable(dec.minimum) : null);
+          maximum = (dec.maximum != null ? new HiveDecimalWritable(dec.maximum) : null);
           sum = dec.sum;
         } else if (dec.minimum != null) {
           if (minimum.compareTo(dec.minimum) > 0) {
-            minimum = dec.minimum;
+            minimum.set(dec.minimum);
           }
           if (maximum.compareTo(dec.maximum) < 0) {
-            maximum = dec.maximum;
+            maximum.set(dec.maximum);
           }
           if (sum == null || dec.sum == null) {
             sum = null;
           } else {
-            sum = sum.add(dec.sum);
+            sum.mutateAdd(dec.sum);
           }
         }
       } else {
@@ -657,7 +660,8 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
         dec.setMinimum(minimum.toString());
         dec.setMaximum(maximum.toString());
       }
-      if (sum != null) {
+      // Check isSet for overflow.
+      if (sum != null && sum.isSet()) {
         dec.setSum(sum.toString());
       }
       result.setDecimalStatistics(dec);
@@ -666,17 +670,17 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
 
     @Override
     public HiveDecimal getMinimum() {
-      return minimum;
+      return minimum.getHiveDecimal();
     }
 
     @Override
     public HiveDecimal getMaximum() {
-      return maximum;
+      return maximum.getHiveDecimal();
     }
 
     @Override
     public HiveDecimal getSum() {
-      return sum;
+      return sum.getHiveDecimal();
     }
 
     @Override
@@ -987,7 +991,7 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
     throw new UnsupportedOperationException("Can't update string");
   }
 
-  public void updateDecimal(HiveDecimal value) {
+  public void updateDecimal(HiveDecimalWritable value) {
     throw new UnsupportedOperationException("Can't update decimal");
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index 5d5f991..e60075f 100644
--- a/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -608,19 +608,55 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       setConvertTreeReader(decimalTreeReader);
     }
 
-    private static HiveDecimal DECIMAL_MAX_LONG = HiveDecimal.create(Long.MAX_VALUE);
-    private static HiveDecimal DECIMAL_MIN_LONG = HiveDecimal.create(Long.MIN_VALUE);
-
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
-      HiveDecimal decimalValue = decimalColVector.vector[elementNum].getHiveDecimal();
-      if (decimalValue.compareTo(DECIMAL_MAX_LONG) > 0 ||
-          decimalValue.compareTo(DECIMAL_MIN_LONG) < 0) {
+      HiveDecimalWritable decWritable = decimalColVector.vector[elementNum];
+      long[] vector = longColVector.vector;
+      Category readerCategory = readerType.getCategory();
+
+      // Check to see if the decimal will fit in the Hive integer data type.
+      // If not, set the element to null.
+      boolean isInRange;
+      switch (readerCategory) {
+      case BOOLEAN:
+        // No data loss for boolean.
+        vector[elementNum] = decWritable.signum() == 0 ? 0 : 1;
+        return;
+      case BYTE:
+        isInRange = decWritable.isByte();
+        break;
+      case SHORT:
+        isInRange = decWritable.isShort();
+        break;
+      case INT:
+        isInRange = decWritable.isInt();
+        break;
+      case LONG:
+        isInRange = decWritable.isLong();
+        break;
+      default:
+        throw new RuntimeException("Unexpected type kind " + readerCategory.name());
+      }
+      if (!isInRange) {
         longColVector.isNull[elementNum] = true;
         longColVector.noNulls = false;
-      } else {
-        // TODO: lossy conversion!
-        downCastAnyInteger(longColVector, elementNum, decimalValue.longValue(), readerType);
+        return;
+      }
+      switch (readerCategory) {
+      case BYTE:
+        vector[elementNum] = decWritable.byteValue();
+        break;
+      case SHORT:
+        vector[elementNum] = decWritable.shortValue();
+        break;
+      case INT:
+        vector[elementNum] = decWritable.intValue();
+        break;
+      case LONG:
+        vector[elementNum] = decWritable.longValue();
+        break;
+      default:
+        throw new RuntimeException("Unexpected type kind " + readerCategory.name());
       }
     }
 
@@ -828,7 +864,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
       doubleColVector.vector[elementNum] =
-          (float) decimalColVector.vector[elementNum].getHiveDecimal().doubleValue();
+          (float) decimalColVector.vector[elementNum].doubleValue();
     }
 
     @Override
@@ -1034,7 +1070,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
       doubleColVector.vector[elementNum] =
-          decimalColVector.vector[elementNum].getHiveDecimal().doubleValue();
+          decimalColVector.vector[elementNum].doubleValue();
     }
 
     @Override
@@ -1371,14 +1407,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     @Override
     public void setConvertVectorElement(int elementNum) throws IOException {
 
-      HiveDecimalWritable valueWritable = HiveDecimalWritable.enforcePrecisionScale(
-          fileDecimalColVector.vector[elementNum], readerPrecision, readerScale);
-      if (valueWritable != null) {
-        decimalColVector.set(elementNum, valueWritable);
-      } else {
-        decimalColVector.noNulls = false;
-        decimalColVector.isNull[elementNum] = true;
-      }
+      decimalColVector.set(elementNum, fileDecimalColVector.vector[elementNum]);
+
     }
 
     @Override
@@ -1540,6 +1570,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     private final TypeDescription readerType;
     private DecimalColumnVector decimalColVector;
     private BytesColumnVector bytesColVector;
+    private byte[] scratchBuffer;
 
     StringGroupFromDecimalTreeReader(int columnId, TypeDescription fileType,
         TypeDescription readerType, boolean skipCorrupt) throws IOException {
@@ -1549,13 +1580,19 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       this.readerType = readerType;
       decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
       setConvertTreeReader(decimalTreeReader);
+      scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES];
     }
 
     @Override
     public void setConvertVectorElement(int elementNum) {
-      String string = decimalColVector.vector[elementNum].getHiveDecimal().toString();
-      byte[] bytes = string.getBytes();
-      assignStringGroupVectorEntry(bytesColVector, elementNum, readerType, bytes);
+      HiveDecimalWritable decWritable = decimalColVector.vector[elementNum];
+
+      // Convert decimal into bytes instead of a String for better performance.
+      final int byteIndex = decWritable.toBytes(scratchBuffer);
+
+      assignStringGroupVectorEntry(
+          bytesColVector, elementNum, readerType,
+          scratchBuffer, byteIndex, HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES - byteIndex);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java b/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
index 484209b..3ddafba 100644
--- a/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
+++ b/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.orc.TypeDescription;
 import org.apache.orc.OrcProto;
 
@@ -1044,6 +1045,7 @@ public class TreeReaderFactory {
     protected InStream valueStream;
     protected IntegerReader scaleReader = null;
     private int[] scratchScaleVector;
+    private byte[] scratchBytes;
 
     private final int precision;
     private final int scale;
@@ -1060,6 +1062,7 @@ public class TreeReaderFactory {
       this.scale = scale;
       this.scratchScaleVector = new int[VectorizedRowBatch.DEFAULT_SIZE];
       this.valueStream = valueStream;
+      this.scratchBytes = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ];
       if (scaleStream != null && encoding != null) {
         checkEncoding(encoding);
         this.scaleReader = createIntegerReader(encoding.getKind(), scaleStream, true, false);
@@ -1112,18 +1115,30 @@ public class TreeReaderFactory {
       // read the scales
       scaleReader.nextVector(result, scratchScaleVector, batchSize);
       // Read value entries based on isNull entries
+      // Use the fast ORC deserialization method that emulates SerializationUtils.readBigInteger
+      // provided by HiveDecimalWritable.
+      HiveDecimalWritable[] vector = result.vector;
+      HiveDecimalWritable decWritable;
       if (result.noNulls) {
         for (int r=0; r < batchSize; ++r) {
-          BigInteger bInt = SerializationUtils.readBigInteger(valueStream);
-          HiveDecimal dec = HiveDecimal.create(bInt, scratchScaleVector[r]);
-          result.set(r, dec);
+          decWritable = vector[r];
+          if (!decWritable.serializationUtilsRead(
+              valueStream, scratchScaleVector[r],
+              scratchBytes)) {
+            result.isNull[r] = true;
+            result.noNulls = false;
+          }
         }
       } else if (!result.isRepeating || !result.isNull[0]) {
         for (int r=0; r < batchSize; ++r) {
           if (!result.isNull[r]) {
-            BigInteger bInt = SerializationUtils.readBigInteger(valueStream);
-            HiveDecimal dec = HiveDecimal.create(bInt, scratchScaleVector[r]);
-            result.set(r, dec);
+            decWritable = vector[r];
+            if (!decWritable.serializationUtilsRead(
+                valueStream, scratchScaleVector[r],
+                scratchBytes)) {
+              result.isNull[r] = true;
+              result.noNulls = false;
+            }
           }
         }
       }
@@ -1132,8 +1147,9 @@ public class TreeReaderFactory {
     @Override
     void skipRows(long items) throws IOException {
       items = countNonNulls(items);
+      HiveDecimalWritable scratchDecWritable = new HiveDecimalWritable();
       for (int i = 0; i < items; i++) {
-        SerializationUtils.readBigInteger(valueStream);
+        scratchDecWritable.serializationUtilsRead(valueStream, 0, scratchBytes);
       }
       scaleReader.skip(items);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/orc/src/java/org/apache/orc/impl/WriterImpl.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/WriterImpl.java b/orc/src/java/org/apache/orc/impl/WriterImpl.java
index b17fb41..518a5f7 100644
--- a/orc/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/orc/src/java/org/apache/orc/impl/WriterImpl.java
@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.TimeZone;
 import java.util.TreeMap;
 
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.ql.util.JavaDataModel;
 import org.apache.orc.BinaryColumnStatistics;
 import org.apache.orc.BloomFilterIO;
@@ -1610,6 +1611,11 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
 
   private static class DecimalTreeWriter extends TreeWriter {
     private final PositionedOutputStream valueStream;
+
+    // These scratch buffers allow us to serialize decimals much faster.
+    private final long[] scratchLongs;
+    private final byte[] scratchBuffer;
+
     private final IntegerWriter scaleStream;
     private final boolean isDirectV2;
 
@@ -1620,6 +1626,8 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
       super(columnId, schema, writer, nullable);
       this.isDirectV2 = isNewWriteFormat(writer);
       valueStream = writer.createStream(id, OrcProto.Stream.Kind.DATA);
+      scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN];
+      scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES];
       this.scaleStream = createIntegerWriter(writer.createStream(id,
           OrcProto.Stream.Kind.SECONDARY), true, isDirectV2, writer);
       recordPosition(rowIndexPosition);
@@ -1642,27 +1650,36 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
       DecimalColumnVector vec = (DecimalColumnVector) vector;
       if (vector.isRepeating) {
         if (vector.noNulls || !vector.isNull[0]) {
-          HiveDecimal value = vec.vector[0].getHiveDecimal();
+          HiveDecimalWritable value = vec.vector[0];
           indexStatistics.updateDecimal(value);
           if (createBloomFilter) {
-            bloomFilter.addString(value.toString());
+
+            // The HiveDecimalWritable toString() method with a scratch buffer for good performance
+            // when creating the String.  We need to use a String hash code and not UTF-8 byte[]
+            // hash code in order to get the right hash code.
+            bloomFilter.addString(value.toString(scratchBuffer));
           }
           for(int i=0; i < length; ++i) {
-            SerializationUtils.writeBigInteger(valueStream,
-                value.unscaledValue());
+
+            // Use the fast ORC serialization method that emulates SerializationUtils.writeBigInteger
+            // provided by HiveDecimalWritable.
+            value.serializationUtilsWrite(
+                valueStream,
+                scratchLongs);
             scaleStream.write(value.scale());
           }
         }
       } else {
         for(int i=0; i < length; ++i) {
           if (vec.noNulls || !vec.isNull[i + offset]) {
-            HiveDecimal value = vec.vector[i + offset].getHiveDecimal();
-            SerializationUtils.writeBigInteger(valueStream,
-                value.unscaledValue());
+            HiveDecimalWritable value = vec.vector[i + offset];
+            value.serializationUtilsWrite(
+                valueStream,
+                scratchLongs);
             scaleStream.write(value.scale());
             indexStatistics.updateDecimal(value);
             if (createBloomFilter) {
-              bloomFilter.addString(value.toString());
+              bloomFilter.addString(value.toString(scratchBuffer));
             }
           }
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/orc/src/test/org/apache/orc/TestColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/TestColumnStatistics.java b/orc/src/test/org/apache/orc/TestColumnStatistics.java
index 1837dbb..93d4bdb 100644
--- a/orc/src/test/org/apache/orc/TestColumnStatistics.java
+++ b/orc/src/test/org/apache/orc/TestColumnStatistics.java
@@ -30,6 +30,7 @@ import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -170,17 +171,17 @@ public class TestColumnStatistics {
 
     ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(schema);
     ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(schema);
-    stats1.updateDecimal(HiveDecimal.create(10));
-    stats1.updateDecimal(HiveDecimal.create(100));
-    stats2.updateDecimal(HiveDecimal.create(1));
-    stats2.updateDecimal(HiveDecimal.create(1000));
+    stats1.updateDecimal(new HiveDecimalWritable(10));
+    stats1.updateDecimal(new HiveDecimalWritable(100));
+    stats2.updateDecimal(new HiveDecimalWritable(1));
+    stats2.updateDecimal(new HiveDecimalWritable(1000));
     stats1.merge(stats2);
     DecimalColumnStatistics typed = (DecimalColumnStatistics) stats1;
     assertEquals(1, typed.getMinimum().longValue());
     assertEquals(1000, typed.getMaximum().longValue());
     stats1.reset();
-    stats1.updateDecimal(HiveDecimal.create(-10));
-    stats1.updateDecimal(HiveDecimal.create(10000));
+    stats1.updateDecimal(new HiveDecimalWritable(-10));
+    stats1.updateDecimal(new HiveDecimalWritable(10000));
     stats1.merge(stats2);
     assertEquals(-10, typed.getMinimum().longValue());
     assertEquals(10000, typed.getMaximum().longValue());

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
index 9a48171..b532e2f 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
@@ -61,12 +61,11 @@ public class <ClassName> extends VectorAggregateExpression {
       }
 
       public void checkValue(HiveDecimalWritable writable, short scale) {
-        HiveDecimal value = writable.getHiveDecimal();
         if (isNull) {
           isNull = false;
-          this.value.set(value);
-        } else if (this.value.getHiveDecimal().compareTo(value) <OperatorSymbol> 0) {
-          this.value.set(value);
+          this.value.set(writable);
+        } else if (this.value.compareTo(writable) <OperatorSymbol> 0) {
+          this.value.set(writable);
         }
       }
 
@@ -321,8 +320,7 @@ public class <ClassName> extends VectorAggregateExpression {
           if (inputVector.noNulls &&
             (myagg.isNull || (myagg.value.compareTo(vector[0]) <OperatorSymbol> 0))) {
             myagg.isNull = false;
-            HiveDecimal value = vector[0].getHiveDecimal();
-            myagg.value.set(value);
+            myagg.value.set(vector[0]);
           }
           return;
         }
@@ -354,13 +352,13 @@ public class <ClassName> extends VectorAggregateExpression {
       for (int j=0; j< batchSize; ++j) {
         int i = selected[j];
         if (!isNull[i]) {
-          HiveDecimal value = vector[i].getHiveDecimal();
+          HiveDecimalWritable writable = vector[i];
           if (myagg.isNull) {
             myagg.isNull = false;
-            myagg.value.set(value);
+            myagg.value.set(writable);
           }
-          else if (myagg.value.getHiveDecimal().compareTo(value) <OperatorSymbol> 0) {
-            myagg.value.set(value);
+          else if (myagg.value.compareTo(writable) <OperatorSymbol> 0) {
+            myagg.value.set(writable);
           }
         }
       }
@@ -374,15 +372,14 @@ public class <ClassName> extends VectorAggregateExpression {
         int[] selected) {
 
       if (myagg.isNull) {
-        HiveDecimal value = vector[selected[0]].getHiveDecimal();
-        myagg.value.set(value);
+        myagg.value.set(vector[selected[0]]);
         myagg.isNull = false;
       }
 
       for (int i=0; i< batchSize; ++i) {
-        HiveDecimal value = vector[selected[i]].getHiveDecimal();
-        if (myagg.value.getHiveDecimal().compareTo(value) <OperatorSymbol> 0) {
-          myagg.value.set(value);
+        HiveDecimalWritable writable = vector[selected[i]];
+        if (myagg.value.compareTo(writable) <OperatorSymbol> 0) {
+          myagg.value.set(writable);
         }
       }
     }
@@ -396,13 +393,13 @@ public class <ClassName> extends VectorAggregateExpression {
 
       for(int i=0;i<batchSize;++i) {
         if (!isNull[i]) {
-          HiveDecimal value = vector[i].getHiveDecimal();
+          HiveDecimalWritable writable = vector[i];
           if (myagg.isNull) {
-            myagg.value.set(value);
+            myagg.value.set(writable);
             myagg.isNull = false;
           }
-          else if (myagg.value.getHiveDecimal().compareTo(value) <OperatorSymbol> 0) {
-            myagg.value.set(value);
+          else if (myagg.value.compareTo(writable) <OperatorSymbol> 0) {
+            myagg.value.set(writable);
           }
         }
       }
@@ -414,15 +411,14 @@ public class <ClassName> extends VectorAggregateExpression {
         short scale,
         int batchSize) {
       if (myagg.isNull) {
-        HiveDecimal value = vector[0].getHiveDecimal();
-        myagg.value.set(value);
+        myagg.value.set(vector[0]);
         myagg.isNull = false;
       }
 
       for (int i=0;i<batchSize;++i) {
-        HiveDecimal value = vector[i].getHiveDecimal();
-        if (myagg.value.getHiveDecimal().compareTo(value) <OperatorSymbol> 0) {
-          myagg.value.set(value);
+        HiveDecimalWritable writable = vector[i];
+        if (myagg.value.compareTo(writable) <OperatorSymbol> 0) {
+          myagg.value.set(writable);
         }
       }
     }
@@ -472,4 +468,3 @@ public class <ClassName> extends VectorAggregateExpression {
     this.inputExpression = inputExpression;
   }
 }
-

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
index d31d338..fc82cf7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
@@ -534,8 +534,9 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
           }
           break;
         case DECIMAL:
+          // The DecimalColumnVector set method will quickly copy the deserialized decimal writable fields.
           ((DecimalColumnVector) batch.cols[projectionColumnNum]).set(
-              batchIndex, deserializeRead.currentHiveDecimalWritable.getHiveDecimal());
+              batchIndex, deserializeRead.currentHiveDecimalWritable);
           break;
         case INTERVAL_YEAR_MONTH:
           ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] =

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
index e6dc9ec..631dcb2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
@@ -310,8 +310,9 @@ public class VectorExtractRow {
             return primitiveWritable;
           }
         case DECIMAL:
+          // The HiveDecimalWritable set method will quickly copy the deserialized decimal writable fields.
           ((HiveDecimalWritable) primitiveWritable).set(
-              ((DecimalColumnVector) batch.cols[projectionColumnNum]).vector[adjustedIndex].getHiveDecimal());
+              ((DecimalColumnVector) batch.cols[projectionColumnNum]).vector[adjustedIndex]);
           return primitiveWritable;
         case INTERVAL_YEAR_MONTH:
           ((HiveIntervalYearMonthWritable) primitiveWritable).set(

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
index 8a101a6..2bd1850 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
@@ -109,7 +109,8 @@ public class VectorHashKeyWrapper extends KeyWrapper {
         Arrays.hashCode(isNull);
 
     for (int i = 0; i < decimalValues.length; i++) {
-      hashcode ^= decimalValues[i].getHiveDecimal().hashCode();
+      // Use the new faster hash code since we are hashing memory objects.
+      hashcode ^= decimalValues[i].newFasterHashCode();
     }
 
     for (int i = 0; i < timestampValues.length; i++) {

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
index bfd26ae..b4708b5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
@@ -773,7 +773,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
     } else if (klh.decimalIndex >= 0) {
       return kw.getIsDecimalNull(klh.decimalIndex)? null :
           keyOutputWriter.writeValue(
-                kw.getDecimal(klh.decimalIndex).getHiveDecimal());
+                kw.getDecimal(klh.decimalIndex));
     } else if (klh.timestampIndex >= 0) {
       return kw.getIsTimestampNull(klh.timestampIndex)? null :
           keyOutputWriter.writeValue(

http://git-wip-us.apache.org/repos/asf/hive/blob/4ba713cc/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
index 6af3d99..a95098a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
@@ -497,7 +497,10 @@ public final class VectorSerializeRow<T extends SerializeWrite> {
 
       if (colVector.isRepeating) {
         if (colVector.noNulls || !colVector.isNull[0]) {
-          serializeWrite.writeHiveDecimal(colVector.vector[0].getHiveDecimal(), colVector.scale);
+          // We serialize specifying the HiveDecimalWritable but also the desired
+          // serialization scale that will be used by text serialization for adding
+          // trailing fractional zeroes.
+          serializeWrite.writeHiveDecimal(colVector.vector[0], colVector.scale);
           return true;
         } else {
           serializeWrite.writeNull();
@@ -505,7 +508,7 @@ public final class VectorSerializeRow<T extends SerializeWrite> {
         }
       } else {
         if (colVector.noNulls || !colVector.isNull[batchIndex]) {
-          serializeWrite.writeHiveDecimal(colVector.vector[batchIndex].getHiveDecimal(), colVector.scale);
+          serializeWrite.writeHiveDecimal(colVector.vector[batchIndex], colVector.scale);
           return true;
         } else {
           serializeWrite.writeNull();


Mime
View raw message