hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject hive git commit: HIVE-11849: NPE in HiveHBaseTableShapshotInputFormat in query with just count(*) (Enis Soztutar via Jason Dere)
Date Mon, 21 Sep 2015 18:00:03 GMT
Repository: hive
Updated Branches:
  refs/heads/master 4ff5b258c -> 2a65989a4


HIVE-11849: NPE in HiveHBaseTableShapshotInputFormat in query with just count(*) (Enis Soztutar
via Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2a65989a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2a65989a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2a65989a

Branch: refs/heads/master
Commit: 2a65989a48e043404b7060296be8da9d3494911e
Parents: 4ff5b25
Author: Jason Dere <jdere@hortonworks.com>
Authored: Mon Sep 21 10:59:21 2015 -0700
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Mon Sep 21 10:59:21 2015 -0700

----------------------------------------------------------------------
 .../HiveHBaseTableSnapshotInputFormat.java      | 21 ++++++++++++-------
 .../queries/positive/hbase_handler_snapshot.q   |  4 ++++
 .../positive/hbase_handler_snapshot.q.out       | 22 ++++++++++++++++++++
 3 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
index 45e4de9..aa3a02f 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapred.TableInputFormat;
 import org.apache.hadoop.hbase.mapred.TableSnapshotInputFormat;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.InputFormat;
@@ -41,15 +44,17 @@ public class HiveHBaseTableSnapshotInputFormat
   TableSnapshotInputFormat delegate = new TableSnapshotInputFormat();
 
   private static void setColumns(JobConf job) throws IOException {
-    // hbase mapred API doesn't support scan at the moment.
     Scan scan = HiveHBaseInputFormatUtil.getScan(job);
-    byte[][] families = scan.getFamilies();
-    StringBuilder sb = new StringBuilder();
-    for (int i = 0; i < families.length; i++) {
-      if (i > 0) sb.append(" ");
-      sb.append(Bytes.toString(families[i]));
-    }
-    job.set(TableInputFormat.COLUMN_LIST, sb.toString());
+    job.set(org.apache.hadoop.hbase.mapreduce.TableInputFormat.SCAN,
+      convertScanToString(scan));
+  }
+
+  // TODO: Once HBASE-11163 is completed, use that API, or switch to
+  // using mapreduce version of the APIs. rather than mapred
+  // Copied from HBase's TableMapreduceUtil since it is not public API
+  static String convertScanToString(Scan scan) throws IOException {
+    ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
+    return Base64.encodeBytes(proto.toByteArray());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
index 11d52fd..ebdc63c 100644
--- a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
+++ b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
@@ -2,3 +2,7 @@ SET hive.hbase.snapshot.name=src_hbase_snapshot;
 SET hive.hbase.snapshot.restoredir=/tmp;
 
 SELECT * FROM src_hbase LIMIT 5;
+
+SELECT value FROM src_hbase LIMIT 5;
+
+select count(*) from src_hbase;

http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
index 1cb18b2..731646c 100644
--- a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
@@ -11,3 +11,25 @@ POSTHOOK: Input: default@src_hbase
 100	val_100
 103	val_103
 104	val_104
+PREHOOK: query: SELECT value FROM src_hbase LIMIT 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value FROM src_hbase LIMIT 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+val_0
+val_10
+val_100
+val_103
+val_104
+PREHOOK: query: select count(*) from src_hbase
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from src_hbase
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+309


Mime
View raw message