hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pxi...@apache.org
Subject hive git commit: HIVE-16440: Fix failing test columnstats_partlvl_invalid_values when autogather column stats is on (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Date Fri, 14 Apr 2017 03:34:58 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2.3 fe427ee63 -> 578293ec8


HIVE-16440: Fix failing test columnstats_partlvl_invalid_values when autogather column stats
is on (Pengcheng Xiong, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/578293ec
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/578293ec
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/578293ec

Branch: refs/heads/branch-2.3
Commit: 578293ec84de770b362993692f8ba66f1cedad3a
Parents: fe427ee
Author: Pengcheng Xiong <pxiong@hortonworks.com>
Authored: Thu Apr 13 20:34:09 2017 -0700
Committer: Pengcheng Xiong <pxiong@hortonworks.com>
Committed: Thu Apr 13 20:34:53 2017 -0700

----------------------------------------------------------------------
 .../ql/parse/ColumnStatsSemanticAnalyzer.java   |  8 +--
 ...umnstats_partlvl_invalid_values_autogather.q | 17 +++++
 ...tats_partlvl_invalid_values_autogather.q.out | 69 ++++++++++++++++++++
 3 files changed, 90 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/578293ec/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
index 08d0544..936a000 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
@@ -109,7 +109,7 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
     }
   }
 
-  private void handlePartialPartitionSpec(Map<String,String> partSpec) throws
+  private void handlePartialPartitionSpec(Map<String,String> partSpec, ColumnStatsAutoGatherContext
context) throws
     SemanticException {
 
     // If user has fully specified partition, validate that partition exists
@@ -120,7 +120,7 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
     try {
       // for static partition, it may not exist when HIVESTATSCOLAUTOGATHER is
       // set to true
-      if (!conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER)) {
+      if (context == null) {
         if ((partValsSpecified == tbl.getPartitionKeys().size())
             && (db.getPartition(tbl, partSpec, false, null, false) == null)) {
           throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg()
@@ -377,7 +377,7 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
       if (isPartitionStats) {
         isTableLevel = false;
         partSpec = AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf);
-        handlePartialPartitionSpec(partSpec);
+        handlePartialPartitionSpec(partSpec, null);
       } else {
         isTableLevel = true;
       }
@@ -447,7 +447,7 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
     if (isPartitionStats) {
       isTableLevel = false;
       partSpec = AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf);
-      handlePartialPartitionSpec(partSpec);
+      handlePartialPartitionSpec(partSpec, context);
     } else {
       isTableLevel = true;
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/578293ec/ql/src/test/queries/clientnegative/columnstats_partlvl_invalid_values_autogather.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/columnstats_partlvl_invalid_values_autogather.q
b/ql/src/test/queries/clientnegative/columnstats_partlvl_invalid_values_autogather.q
new file mode 100644
index 0000000..5f5cb42
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/columnstats_partlvl_invalid_values_autogather.q
@@ -0,0 +1,17 @@
+set hive.stats.column.autogather=true;
+DROP TABLE Employee_Part;
+
+CREATE TABLE Employee_Part(employeeID int, employeeName String) partitioned by (employeeSalary
double, country string)
+row format delimited fields terminated by '|'  stored as textfile;
+
+LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part partition(employeeSalary='2000.0',
country='USA');
+LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part partition(employeeSalary='2000.0',
country='UK');
+LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part partition(employeeSalary='3000.0',
country='USA');
+LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part partition(employeeSalary='4000.0',
country='USA');
+LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part partition(employeeSalary='3500.0',
country='UK');
+LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part partition(employeeSalary='3000.0',
country='UK');
+
+-- specify invalid values for the partitioning keys
+explain
+analyze table Employee_Part partition (employeeSalary='4000.0', country='Canada') compute
statistics for columns employeeName, employeeID;
+analyze table Employee_Part partition (employeeSalary='4000.0', country='Canada') compute
statistics for columns employeeName, employeeID;

http://git-wip-us.apache.org/repos/asf/hive/blob/578293ec/ql/src/test/results/clientnegative/columnstats_partlvl_invalid_values_autogather.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/columnstats_partlvl_invalid_values_autogather.q.out
b/ql/src/test/results/clientnegative/columnstats_partlvl_invalid_values_autogather.q.out
new file mode 100644
index 0000000..3261f78
--- /dev/null
+++ b/ql/src/test/results/clientnegative/columnstats_partlvl_invalid_values_autogather.q.out
@@ -0,0 +1,69 @@
+PREHOOK: query: DROP TABLE Employee_Part
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE Employee_Part
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE Employee_Part(employeeID int, employeeName String) partitioned
by (employeeSalary double, country string)
+row format delimited fields terminated by '|'  stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@Employee_Part
+POSTHOOK: query: CREATE TABLE Employee_Part(employeeID int, employeeName String) partitioned
by (employeeSalary double, country string)
+row format delimited fields terminated by '|'  stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@Employee_Part
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='2000.0', country='USA')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@employee_part
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='2000.0', country='USA')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@employee_part
+POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='2000.0', country='UK')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@employee_part
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='2000.0', country='UK')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@employee_part
+POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='3000.0', country='USA')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@employee_part
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='3000.0', country='USA')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@employee_part
+POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='4000.0', country='USA')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@employee_part
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='4000.0', country='USA')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@employee_part
+POSTHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='3500.0', country='UK')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@employee_part
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='3500.0', country='UK')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@employee_part
+POSTHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='3000.0', country='UK')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@employee_part
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/employee2.dat" INTO TABLE Employee_Part
partition(employeeSalary='3000.0', country='UK')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@employee_part
+POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK
+FAILED: SemanticException [Error 30007]: Invalid partitioning key/value specified in ANALYZE
statement : {employeesalary=4000.0, country=Canada}


Mime
View raw message