hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-16454: Add blobstore tests for inserting empty into dynamic partition/list bucket tables & inserting cross blobstore tables (Rentao Wu, reviewed by Sergio Pena)
Date Thu, 20 Apr 2017 18:15:00 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2.3 a95b3497a -> 5ee0e31d8


HIVE-16454: Add blobstore tests for inserting empty into dynamic partition/list bucket tables
& inserting cross blobstore tables (Rentao Wu, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5ee0e31d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5ee0e31d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5ee0e31d

Branch: refs/heads/branch-2.3
Commit: 5ee0e31d821c3ae5e2c37dd58c898d8b9582784f
Parents: a95b349
Author: Rentao Wu <rewu@amazon.com>
Authored: Thu Apr 20 13:07:51 2017 -0500
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Thu Apr 20 13:14:51 2017 -0500

----------------------------------------------------------------------
 .../insert_blobstore_to_blobstore.q             |  29 ++++
 .../insert_empty_into_blobstore.q               |  53 +++++++
 .../insert_blobstore_to_blobstore.q.out         | 110 +++++++++++++
 .../insert_empty_into_blobstore.q.out           | 155 +++++++++++++++++++
 4 files changed, 347 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5ee0e31d/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q
b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q
new file mode 100644
index 0000000..8219ee2
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q
@@ -0,0 +1,29 @@
+-- Test inserting into a blobstore table from another blobstore table.
+
+DROP TABLE blobstore_source;
+CREATE TABLE blobstore_source (
+    a string,
+    b string,
+    c double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_source';
+
+LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
+
+DROP TABLE blobstore_table;
+CREATE TABLE blobstore_table LIKE blobstore_source
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_table';
+
+INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source;
+
+SELECT COUNT(*) FROM blobstore_table;
+
+-- INSERT INTO should append all records to existing ones.
+INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source;
+
+SELECT COUNT(*) FROM blobstore_table;
+
+SELECT * FROM blobstore_table;

http://git-wip-us.apache.org/repos/asf/hive/blob/5ee0e31d/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q
b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q
new file mode 100644
index 0000000..d4f0c71
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q
@@ -0,0 +1,53 @@
+-- Test inserting empty rows into dynamic partitioned and list bucketed blobstore tables
+
+SET hive.blobstore.optimizations.enabled=true;
+
+DROP TABLE empty;
+DROP TABLE blobstore_dynamic_partitioning;
+DROP TABLE blobstore_list_bucketing;
+
+CREATE TABLE empty (
+    id int,
+    name string,
+    dept string,
+    pt string,
+    dt string,
+    hr string);
+
+CREATE TABLE blobstore_dynamic_partitioning (
+    id int,
+    name string,
+    dept string)
+PARTITIONED BY (
+    pt string,
+    dt string,
+    hr string)
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_dynamic_partitioning';
+
+INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name,
dept, dt, hr FROM empty;
+
+SELECT COUNT(*) FROM blobstore_dynamic_partitioning;
+
+CREATE TABLE blobstore_list_bucketing (
+    id int,
+    name string,
+    dept string)
+PARTITIONED BY (
+    pt string,
+    dt string,
+    hr string)
+SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_list_bucketing';
+
+INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id,
name, dept FROM empty;
+
+SELECT COUNT(*) FROM blobstore_list_bucketing;
+
+-- Now test empty inserts with blobstore optimizations turned off. This should give us same
results.
+SET hive.blobstore.optimizations.enabled=false;
+
+INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name,
dept, dt, hr FROM empty;
+SELECT COUNT(*) FROM blobstore_dynamic_partitioning;
+
+INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id,
name, dept FROM empty;
+SELECT COUNT(*) FROM blobstore_list_bucketing;

http://git-wip-us.apache.org/repos/asf/hive/blob/5ee0e31d/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out
b/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out
new file mode 100644
index 0000000..4651899
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out
@@ -0,0 +1,110 @@
+PREHOOK: query: DROP TABLE blobstore_source
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_source
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_source (
+    a string,
+    b string,
+    c double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: CREATE TABLE blobstore_source (
+    a string,
+    b string,
+    c double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: DROP TABLE blobstore_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_table LIKE blobstore_source
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_table
+POSTHOOK: query: CREATE TABLE blobstore_table LIKE blobstore_source
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_table
+PREHOOK: query: INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@blobstore_table
+POSTHOOK: query: INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@blobstore_table
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+7
+PREHOOK: query: INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@blobstore_table
+POSTHOOK: query: INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@blobstore_table
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+14
+PREHOOK: query: SELECT * FROM blobstore_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM blobstore_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+1	abc	10.5
+2	def	11.5
+3	ajss	90.23232
+4	djns	89.02002
+5	random	2.99
+6	data	3.002
+7	ne	71.9084
+1	abc	10.5
+2	def	11.5
+3	ajss	90.23232
+4	djns	89.02002
+5	random	2.99
+6	data	3.002
+7	ne	71.9084

http://git-wip-us.apache.org/repos/asf/hive/blob/5ee0e31d/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out
b/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out
new file mode 100644
index 0000000..8e5e096
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out
@@ -0,0 +1,155 @@
+PREHOOK: query: DROP TABLE empty
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE empty
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE blobstore_dynamic_partitioning
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_dynamic_partitioning
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE blobstore_list_bucketing
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_list_bucketing
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE empty (
+    id int,
+    name string,
+    dept string,
+    pt string,
+    dt string,
+    hr string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@empty
+POSTHOOK: query: CREATE TABLE empty (
+    id int,
+    name string,
+    dept string,
+    pt string,
+    dt string,
+    hr string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@empty
+PREHOOK: query: CREATE TABLE blobstore_dynamic_partitioning (
+    id int,
+    name string,
+    dept string)
+PARTITIONED BY (
+    pt string,
+    dt string,
+    hr string)
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_dynamic_partitioning
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_dynamic_partitioning
+POSTHOOK: query: CREATE TABLE blobstore_dynamic_partitioning (
+    id int,
+    name string,
+    dept string)
+PARTITIONED BY (
+    pt string,
+    dt string,
+    hr string)
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_dynamic_partitioning
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_dynamic_partitioning
+PREHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr)
SELECT id, name, dept, dt, hr FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_dynamic_partitioning@pt=a
+POSTHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt,
hr) SELECT id, name, dept, dt, hr FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+0
+PREHOOK: query: CREATE TABLE blobstore_list_bucketing (
+    id int,
+    name string,
+    dept string)
+PARTITIONED BY (
+    pt string,
+    dt string,
+    hr string)
+SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_list_bucketing
+POSTHOOK: query: CREATE TABLE blobstore_list_bucketing (
+    id int,
+    name string,
+    dept string)
+PARTITIONED BY (
+    pt string,
+    dt string,
+    hr string)
+SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_list_bucketing
+PREHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a')
SELECT id, name, dept FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_list_bucketing@pt=a/dt=a/hr=a
+POSTHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a')
SELECT id, name, dept FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+POSTHOOK: Output: default@blobstore_list_bucketing@pt=a/dt=a/hr=a
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+0
+PREHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr)
SELECT id, name, dept, dt, hr FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_dynamic_partitioning@pt=b
+POSTHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt,
hr) SELECT id, name, dept, dt, hr FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+0
+PREHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b')
SELECT id, name, dept FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_list_bucketing@pt=b/dt=b/hr=b
+POSTHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b')
SELECT id, name, dept FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+POSTHOOK: Output: default@blobstore_list_bucketing@pt=b/dt=b/hr=b
+POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).dept SIMPLE [(empty)empty.FieldSchema(name:dept,
type:string, comment:null), ]
+POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).id SIMPLE [(empty)empty.FieldSchema(name:id,
type:int, comment:null), ]
+POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).name SIMPLE [(empty)empty.FieldSchema(name:name,
type:string, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+0


Mime
View raw message