hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject hive git commit: HIVE-11835: Type decimal(1, 1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)
Date Mon, 05 Oct 2015 12:42:32 GMT
Repository: hive
Updated Branches:
  refs/heads/master c23841e55 -> c0c19d073


HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0c19d07
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0c19d07
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0c19d07

Branch: refs/heads/master
Commit: c0c19d0733a3a9fe785e120f618414212e4976be
Parents: c23841e
Author: Xuefu Zhang <xzhang@Cloudera.com>
Authored: Mon Oct 5 05:42:20 2015 -0700
Committer: Xuefu Zhang <xzhang@Cloudera.com>
Committed: Mon Oct 5 05:42:20 2015 -0700

----------------------------------------------------------------------
 .../hive/common/type/TestHiveDecimal.java       |  12 +--
 .../test/resources/testconfiguration.properties |   1 +
 .../test/queries/clientpositive/decimal_1_1.q   |   9 ++
 .../results/clientpositive/decimal_1_1.q.out    | 104 +++++++++++++++++++
 .../clientpositive/spark/decimal_1_1.q.out      | 104 +++++++++++++++++++
 .../hadoop/hive/common/type/HiveDecimal.java    |  10 +-
 6 files changed, 233 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
index ba5ef71..f68842c 100644
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
@@ -42,15 +42,15 @@ public class TestHiveDecimal {
     Assert.assertTrue("Decimal scale should not go above maximum", dec.scale() <= HiveDecimal.MAX_SCALE);
 
     decStr = "57847525803324040144343378.09799306448796128931113691624";
-    BigDecimal bd = new BigDecimal(decStr);
-    BigDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
+    HiveDecimal bd = HiveDecimal.create(decStr);
+    HiveDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
     Assert.assertNull(bd1);
     bd1 = HiveDecimal.enforcePrecisionScale(bd, 35, 5);
     Assert.assertEquals("57847525803324040144343378.09799", bd1.toString());
     bd1 = HiveDecimal.enforcePrecisionScale(bd, 45, 20);
     Assert.assertNull(bd1);
 
-    dec = HiveDecimal.create(bd, false);
+    dec = HiveDecimal.create(new BigDecimal(decStr), false);
     Assert.assertNull(dec);
 
     dec = HiveDecimal.create("-1786135888657847525803324040144343378.09799306448796128931113691624");
@@ -65,10 +65,10 @@ public class TestHiveDecimal {
 
     // Rounding numbers that increase int digits
     Assert.assertEquals("10",
-        HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 2, 0).toString());
-    Assert.assertNull(HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 1, 0));
+        HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 2, 0).toString());
+    Assert.assertNull(HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 1, 0));
     Assert.assertEquals("9",
-        HiveDecimal.enforcePrecisionScale(new BigDecimal("9.4"), 1, 0).toString());
+        HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.4"), 1, 0).toString());
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 700ea63..9c9f4cc 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -667,6 +667,7 @@ spark.query.files=add_part_multiple.q, \
   custom_input_output_format.q, \
   date_join1.q, \
   date_udf.q, \
+  decimal_1_1.q, \
   decimal_join.q, \
   disable_merge_for_bucketing.q, \
   dynamic_rdd_cache.q, \

http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/ql/src/test/queries/clientpositive/decimal_1_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/decimal_1_1.q b/ql/src/test/queries/clientpositive/decimal_1_1.q
new file mode 100644
index 0000000..83ce521
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/decimal_1_1.q
@@ -0,0 +1,9 @@
+drop table if exists decimal_1_1;
+
+create table decimal_1_1 (d decimal(1,1));
+load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1;
+select * from decimal_1_1;
+
+select d from decimal_1_1 order by d desc;
+
+drop table decimal_1_1;

http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/ql/src/test/results/clientpositive/decimal_1_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_1_1.q.out b/ql/src/test/results/clientpositive/decimal_1_1.q.out
new file mode 100644
index 0000000..b2704c6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/decimal_1_1.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: drop table if exists decimal_1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: select * from decimal_1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from decimal_1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0
+0
+0
+0.1
+0.2
+0.9
+0.9
+NULL
+0.3
+NULL
+NULL
+0
+0
+NULL
+0
+0
+0
+0
+-0.1
+-0.2
+-0.9
+-0.9
+NULL
+-0.3
+NULL
+NULL
+0
+0
+NULL
+0
+PREHOOK: query: select d from decimal_1_1 order by d desc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select d from decimal_1_1 order by d desc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0.9
+0.9
+0.3
+0.2
+0.1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+-0.1
+-0.2
+-0.3
+-0.9
+-0.9
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: drop table decimal_1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_1_1
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: drop table decimal_1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_1_1
+POSTHOOK: Output: default@decimal_1_1

http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
new file mode 100644
index 0000000..b2704c6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: drop table if exists decimal_1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: select * from decimal_1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from decimal_1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0
+0
+0
+0.1
+0.2
+0.9
+0.9
+NULL
+0.3
+NULL
+NULL
+0
+0
+NULL
+0
+0
+0
+0
+-0.1
+-0.2
+-0.9
+-0.9
+NULL
+-0.3
+NULL
+NULL
+0
+0
+NULL
+0
+PREHOOK: query: select d from decimal_1_1 order by d desc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select d from decimal_1_1 order by d desc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0.9
+0.9
+0.3
+0.2
+0.1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+-0.1
+-0.2
+-0.3
+-0.9
+-0.9
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: drop table decimal_1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_1_1
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: drop table decimal_1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_1_1
+POSTHOOK: Output: default@decimal_1_1

http://git-wip-us.apache.org/repos/asf/hive/blob/c0c19d07/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
index 12a3936..4ed17a2 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
@@ -272,11 +272,19 @@ public class HiveDecimal implements Comparable<HiveDecimal> {
     return bd;
   }
 
-  public static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale)
{
+  private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale)
{
     if (bd == null) {
       return null;
     }
 
+    /**
+     * Specially handling the case that bd=0, and we are converting it to a type where precision=scale,
+     * such as decimal(1, 1).
+     */
+    if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision
== maxScale) {
+      return bd.setScale(maxScale);
+    }
+
     bd = trim(bd);
 
     if (bd.scale() > maxScale) {


Mime
View raw message