hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aihu...@apache.org
Subject hive git commit: HIVE-12635: Hive should return the latest hbase cell timestamp as the row timestamp value
Date Mon, 21 Dec 2015 21:51:54 GMT
Repository: hive
Updated Branches:
  refs/heads/master dfe929c9b -> 34c83fb36


HIVE-12635: Hive should return the latest hbase cell timestamp as the row timestamp value


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/34c83fb3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/34c83fb3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/34c83fb3

Branch: refs/heads/master
Commit: 34c83fb36f2baf116cbd128ba47773472a24a9c4
Parents: dfe929c
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Wed Dec 9 17:16:04 2015 -0500
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Mon Dec 21 16:44:38 2015 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hive/hbase/LazyHBaseRow.java  |  4 ++
 .../src/test/queries/positive/hbase_timestamp.q |  9 +++
 .../test/results/positive/hbase_timestamp.q.out | 72 ++++++++++++++++++++
 3 files changed, 85 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/34c83fb3/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
index 003dd1c..ffb005e 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
@@ -154,7 +154,11 @@ public class LazyHBaseRow extends LazyStruct {
       }
 
       if (colMap.hbaseTimestamp) {
+        // Get the latest timestamp of all the cells as the row timestamp
         long timestamp = result.rawCells()[0].getTimestamp(); // from hbase-0.96.0
+        for (int i = 1; i < result.rawCells().length; i++) {
+          timestamp = Math.max(timestamp, result.rawCells()[i].getTimestamp());
+        }
         LazyObjectBase lz = fields[fieldID];
         if (lz instanceof LazyTimestamp) {
           ((LazyTimestamp) lz).getWritableObject().setTime(timestamp);

http://git-wip-us.apache.org/repos/asf/hive/blob/34c83fb3/hbase-handler/src/test/queries/positive/hbase_timestamp.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_timestamp.q b/hbase-handler/src/test/queries/positive/hbase_timestamp.q
index 6f1e205..0350afe 100644
--- a/hbase-handler/src/test/queries/positive/hbase_timestamp.q
+++ b/hbase-handler/src/test/queries/positive/hbase_timestamp.q
@@ -39,3 +39,12 @@ SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND
 explain
 SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000000;
 SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000000;
+
+DROP TABLE hbase_table;
+CREATE TABLE hbase_table(key string, value map<string, string>, time timestamp)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:,:timestamp");
+FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, MAP("name", CONCAT(value, " Jr")),
"2012-02-23 10:14:52" WHERE (key % 17) = 0;
+FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("age", '40'), "2015-12-12 12:12:12"
WHERE (key % 17) = 0;
+FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("name", value), "2000-01-01 01:01:01"
WHERE (key % 17) = 0;
+SELECT * FROM hbase_table;

http://git-wip-us.apache.org/repos/asf/hive/blob/34c83fb3/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
index 6c42fc3..54ddec7 100644
--- a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
@@ -355,3 +355,75 @@ POSTHOOK: Input: default@hbase_table
 222	val_222	1976-05-03 12:33:20
 296	val_296	1976-05-03 12:33:20
 333	val_333	1976-05-03 12:33:20
+PREHOOK: query: DROP TABLE hbase_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table
+PREHOOK: Output: default@hbase_table
+POSTHOOK: query: DROP TABLE hbase_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table
+POSTHOOK: Output: default@hbase_table
+PREHOOK: query: CREATE TABLE hbase_table(key string, value map<string, string>, time
timestamp)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:,:timestamp")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table
+POSTHOOK: query: CREATE TABLE hbase_table(key string, value map<string, string>, time
timestamp)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:,:timestamp")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table
+PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, MAP("name", CONCAT(value,
" Jr")), "2012-02-23 10:14:52" WHERE (key % 17) = 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table
+POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, MAP("name", CONCAT(value,
" Jr")), "2012-02-23 10:14:52" WHERE (key % 17) = 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table
+PREHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("age", '40'), "2015-12-12
12:12:12" WHERE (key % 17) = 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table
+POSTHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("age", '40'), "2015-12-12
12:12:12" WHERE (key % 17) = 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table
+PREHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("name", value), "2000-01-01
01:01:01" WHERE (key % 17) = 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table
+POSTHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("name", value), "2000-01-01
01:01:01" WHERE (key % 17) = 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table
+PREHOOK: query: SELECT * FROM hbase_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM hbase_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table
+#### A masked pattern was here ####
+0	{"age":"40","name":"val_0 Jr"}	2015-12-12 12:12:12
+119	{"age":"40","name":"val_119 Jr"}	2015-12-12 12:12:12
+136	{"age":"40","name":"val_136 Jr"}	2015-12-12 12:12:12
+153	{"age":"40","name":"val_153 Jr"}	2015-12-12 12:12:12
+17	{"age":"40","name":"val_17 Jr"}	2015-12-12 12:12:12
+170	{"age":"40","name":"val_170 Jr"}	2015-12-12 12:12:12
+187	{"age":"40","name":"val_187 Jr"}	2015-12-12 12:12:12
+221	{"age":"40","name":"val_221 Jr"}	2015-12-12 12:12:12
+238	{"age":"40","name":"val_238 Jr"}	2015-12-12 12:12:12
+255	{"age":"40","name":"val_255 Jr"}	2015-12-12 12:12:12
+272	{"age":"40","name":"val_272 Jr"}	2015-12-12 12:12:12
+289	{"age":"40","name":"val_289 Jr"}	2015-12-12 12:12:12
+306	{"age":"40","name":"val_306 Jr"}	2015-12-12 12:12:12
+323	{"age":"40","name":"val_323 Jr"}	2015-12-12 12:12:12
+34	{"age":"40","name":"val_34 Jr"}	2015-12-12 12:12:12
+374	{"age":"40","name":"val_374 Jr"}	2015-12-12 12:12:12
+459	{"age":"40","name":"val_459 Jr"}	2015-12-12 12:12:12
+493	{"age":"40","name":"val_493 Jr"}	2015-12-12 12:12:12
+51	{"age":"40","name":"val_51 Jr"}	2015-12-12 12:12:12
+85	{"age":"40","name":"val_85 Jr"}	2015-12-12 12:12:12


Mime
View raw message