hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject [12/50] [abbrv] hive git commit: HIVE-11198: Fix load data query file format check for partitioned tables (Prasanth Jayachandran reviewed by Sushanth Sowmyan)
Date Thu, 16 Jul 2015 04:40:59 GMT
HIVE-11198: Fix load data query file format check for partitioned tables (Prasanth Jayachandran
reviewed by Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a2dabcb8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a2dabcb8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a2dabcb8

Branch: refs/heads/beeline-cli
Commit: a2dabcb8c7bc52bd8def1402fa649420ef41e3bd
Parents: be89eac
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Thu Jul 9 11:20:29 2015 -0700
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Thu Jul 9 11:20:58 2015 -0700

----------------------------------------------------------------------
 .../hive/ql/parse/LoadSemanticAnalyzer.java     | 12 +++-
 .../clientnegative/load_orc_negative_part.q     | 14 ++++
 .../test/queries/clientpositive/load_orc_part.q | 15 +++++
 .../clientnegative/load_orc_negative_part.q.out | 52 +++++++++++++++
 .../results/clientpositive/load_orc_part.q.out  | 70 ++++++++++++++++++++
 5 files changed, 162 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 187dc20..944cee4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -335,7 +335,17 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
   }
 
   private void ensureFileFormatsMatch(TableSpec ts, URI fromURI) throws SemanticException
{
-    Class<? extends InputFormat> destInputFormat = ts.tableHandle.getInputFormatClass();
+    final Class<? extends InputFormat> destInputFormat;
+    try {
+      if (ts.getPartSpec() == null || ts.getPartSpec().isEmpty()) {
+        destInputFormat = ts.tableHandle.getInputFormatClass();
+      } else {
+        destInputFormat = ts.partHandle.getInputFormatClass();
+      }
+    } catch (HiveException e) {
+      throw new SemanticException(e);
+    }
+
     // Other file formats should do similar check to make sure file formats match
     // when doing LOAD DATA .. INTO TABLE
     if (OrcInputFormat.class.equals(destInputFormat)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/queries/clientnegative/load_orc_negative_part.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/load_orc_negative_part.q b/ql/src/test/queries/clientnegative/load_orc_negative_part.q
new file mode 100644
index 0000000..5de4917
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/load_orc_negative_part.q
@@ -0,0 +1,14 @@
+set hive.default.fileformat=ORC;
+create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal,
ts timestamp);
+create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts
timestamp) partitioned by (ds string);
+
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_staging/;
+
+load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/orc_split_elim.orc'
into table orc_test partition (ds='10');
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition
(ds='10');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/;
+
+alter table orc_test add partition(ds='11');
+load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=11/;

http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/queries/clientpositive/load_orc_part.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/load_orc_part.q b/ql/src/test/queries/clientpositive/load_orc_part.q
new file mode 100644
index 0000000..0927ea4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/load_orc_part.q
@@ -0,0 +1,15 @@
+set hive.default.fileformat=ORC;
+create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal,
ts timestamp);
+create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts
timestamp) partitioned by (ds string);
+
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_staging/;
+
+load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/orc_split_elim.orc'
into table orc_test partition (ds='10');
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition
(ds='10');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/;
+
+alter table orc_test add partition(ds='11');
+alter table orc_test partition(ds='11') set fileformat textfile;
+load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=11/;

http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/results/clientnegative/load_orc_negative_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/load_orc_negative_part.q.out b/ql/src/test/results/clientnegative/load_orc_negative_part.q.out
new file mode 100644
index 0000000..32dd627
--- /dev/null
+++ b/ql/src/test/results/clientnegative/load_orc_negative_part.q.out
@@ -0,0 +1,52 @@
+PREHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double,
decimal1 decimal, ts timestamp)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double,
decimal1 decimal, ts timestamp)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_staging
+PREHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_test
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table
orc_staging
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_staging
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=10
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test
partition (ds='10')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test@ds=10
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table
orc_test partition (ds='10')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test@ds=10
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: alter table orc_test add partition(ds='11')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: alter table orc_test add partition(ds='11')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=11
+FAILED: SemanticException [Error 30019]: The file that you are trying to load does not match
the file format of the destination table. Destination table is stored as ORC but the file
being loaded is not a valid ORC file.

http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/results/clientpositive/load_orc_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/load_orc_part.q.out b/ql/src/test/results/clientpositive/load_orc_part.q.out
new file mode 100644
index 0000000..34ca493
--- /dev/null
+++ b/ql/src/test/results/clientpositive/load_orc_part.q.out
@@ -0,0 +1,70 @@
+PREHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double,
decimal1 decimal, ts timestamp)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double,
decimal1 decimal, ts timestamp)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_staging
+PREHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1
decimal, ts timestamp) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_test
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table
orc_staging
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_staging
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=10
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test
partition (ds='10')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test@ds=10
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table
orc_test partition (ds='10')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test@ds=10
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: alter table orc_test add partition(ds='11')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: alter table orc_test add partition(ds='11')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=11
+PREHOOK: query: alter table orc_test partition(ds='11') set fileformat textfile
+PREHOOK: type: ALTERPARTITION_FILEFORMAT
+PREHOOK: Input: default@orc_test
+PREHOOK: Output: default@orc_test@ds=11
+POSTHOOK: query: alter table orc_test partition(ds='11') set fileformat textfile
+POSTHOOK: type: ALTERPARTITION_FILEFORMAT
+POSTHOOK: Input: default@orc_test
+POSTHOOK: Input: default@orc_test@ds=11
+POSTHOOK: Output: default@orc_test@ds=11
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test@ds=11
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test@ds=11
+Found 1 items
+#### A masked pattern was here ####


Mime
View raw message