hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-18607 : HBase HFile write does strange things (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Date Fri, 09 Feb 2018 22:37:06 GMT
Repository: hive
Updated Branches:
  refs/heads/master 58bbfc733 -> 717ef18d9


HIVE-18607 : HBase HFile write does strange things (Sergey Shelukhin, reviewed by Ashutosh
Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/717ef18d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/717ef18d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/717ef18d

Branch: refs/heads/master
Commit: 717ef18d96bd86cbe4448350c22c3766cd90e184
Parents: 58bbfc7
Author: sergey <sershe@apache.org>
Authored: Fri Feb 9 14:31:26 2018 -0800
Committer: sergey <sershe@apache.org>
Committed: Fri Feb 9 14:37:03 2018 -0800

----------------------------------------------------------------------
 .../hive/hbase/HiveHFileOutputFormat.java       |  4 --
 .../test/queries/positive/hbase_handler_bulk.q  | 21 ++++++++++
 .../results/positive/hbase_handler_bulk.q.out   | 40 ++++++++++++++++++++
 3 files changed, 61 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/717ef18d/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
index 81318be..d8dad06 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
@@ -176,10 +176,6 @@ public class HiveHFileOutputFormat extends
                 columnFamilyPath,
                 regionFile.getPath().getName()));
           }
-          // Hive actually wants a file as task output (not a directory), so
-          // replace the empty directory with an empty file to keep it happy.
-          fs.delete(taskAttemptOutputdir, true);
-          fs.createNewFile(taskAttemptOutputdir);
         } catch (InterruptedException ex) {
           throw new IOException(ex);
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/717ef18d/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q b/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
index ac2fdfa..5ac4dc8 100644
--- a/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
+++ b/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
@@ -14,6 +14,7 @@ set mapreduce.input.fileinputformat.split.maxsize=200;
 set mapreduce.input.fileinputformat.split.minsize=200;
 set mapred.reduce.tasks=2;
 
+
 -- this should produce three files in /tmp/hb_target/cf
 insert overwrite table hb_target select distinct key, value from src cluster by key;
 
@@ -24,3 +25,23 @@ insert overwrite table hb_target select distinct key, value from src cluster
by
 
 drop table hb_target;
 dfs -rmr /tmp/hb_target/cf;
+
+
+create table hb_target(key int, val string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
+tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk');
+
+-- do it twice - regression test for HIVE-18607
+
+insert overwrite table hb_target select distinct key, value from src cluster by key;
+
+dfs -rmr /tmp/hb_target/cf;
+
+insert overwrite table hb_target select distinct key, value from src cluster by key;
+
+drop table hb_target;
+dfs -rmr /tmp/hb_target/cf;
+
+
+

http://git-wip-us.apache.org/repos/asf/hive/blob/717ef18d/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out b/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
index 10e1c0a..cd8930f 100644
--- a/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
@@ -33,3 +33,43 @@ POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@hb_target
 POSTHOOK: Output: default@hb_target
 #### A masked pattern was here ####
+PREHOOK: query: create table hb_target(key int, val string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
+tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hb_target
+POSTHOOK: query: create table hb_target(key int, val string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
+tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hb_target
+PREHOOK: query: insert overwrite table hb_target select distinct key, value from src cluster
by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hb_target
+POSTHOOK: query: insert overwrite table hb_target select distinct key, value from src cluster
by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hb_target
+#### A masked pattern was here ####
+PREHOOK: query: insert overwrite table hb_target select distinct key, value from src cluster
by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hb_target
+POSTHOOK: query: insert overwrite table hb_target select distinct key, value from src cluster
by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hb_target
+PREHOOK: query: drop table hb_target
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hb_target
+PREHOOK: Output: default@hb_target
+POSTHOOK: query: drop table hb_target
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hb_target
+POSTHOOK: Output: default@hb_target
+#### A masked pattern was here ####


Mime
View raw message