hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1580176 - in /hive/trunk/hbase-handler/src: java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java test/queries/positive/hbase_pushdown.q test/results/positive/hbase_pushdown.q.out
Date Sat, 22 Mar 2014 06:02:47 GMT
Author: hashutosh
Date: Sat Mar 22 06:02:46 2014
New Revision: 1580176

URL: http://svn.apache.org/r1580176
Log:
HIVE-6650 : hive.optimize.index.filter breaks non-index where with HBaseStorageHandler (Nick
Dimiduk via Ashutosh Chauhan)

Modified:
    hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
    hive/trunk/hbase-handler/src/test/queries/positive/hbase_pushdown.q
    hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1580176&r1=1580175&r2=1580176&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
(original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
Sat Mar 22 06:02:46 2014
@@ -253,6 +253,8 @@ public class HiveHBaseTableInputFormat e
   private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary)
       throws IOException {
 
+    // TODO: assert iKey is HBaseSerDe#HBASE_KEY_COL
+
     Scan scan = new Scan();
     String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
     if (filterExprSerialized == null) {
@@ -270,18 +272,15 @@ public class HiveHBaseTableInputFormat e
     ExprNodeDesc residualPredicate =
       analyzer.analyzePredicate(filterExpr, searchConditions);
 
-    // There should be no residual since we already negotiated
-    // that earlier in HBaseStorageHandler.decomposePredicate.
+    // There should be no residual since we already negotiated that earlier in
+    // HBaseStorageHandler.decomposePredicate. However, with hive.optimize.index.filter
+    // OpProcFactory#pushFilterToStorageHandler pushes the original filter back down again.
+    // Since pushed-down filters are not ommitted at the higher levels (and thus the
+    // contract of negotiation is ignored anyway), just ignore the residuals.
+    // Re-assess this when negotiation is honored and the duplicate evaluation is removed.
+    // THIS IGNORES RESIDUAL PARSING FROM HBaseStorageHandler#decomposePredicate
     if (residualPredicate != null) {
-      throw new RuntimeException(
-        "Unexpected residual predicate " + residualPredicate.getExprString());
-    }
-
-    // There should be exactly one predicate since we already
-    // negotiated that also.
-    if (searchConditions.size() < 1 || searchConditions.size() > 2) {
-      throw new RuntimeException(
-        "Either one or two search conditions expected in push down");
+      LOG.debug("Ignoring residual predicate " + residualPredicate.getExprString());
     }
 
     // Convert the search condition into a restriction on the HBase scan
@@ -292,7 +291,7 @@ public class HiveHBaseTableInputFormat e
       PrimitiveObjectInspector objInspector;
       Object writable;
 
-      try{
+      try {
         objInspector = (PrimitiveObjectInspector)eval.initialize(null);
         writable = eval.evaluate(null);
       } catch (ClassCastException cce) {

Modified: hive/trunk/hbase-handler/src/test/queries/positive/hbase_pushdown.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_pushdown.q?rev=1580176&r1=1580175&r2=1580176&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_pushdown.q (original)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_pushdown.q Sat Mar 22 06:02:46
2014
@@ -17,6 +17,12 @@ explain select * from hbase_pushdown whe
 
 select * from hbase_pushdown where key=90 and value like '%90%';
 
+set hive.optimize.index.filter=true;
+-- with partial pushdown with optimization (HIVE-6650)
+explain select * from hbase_pushdown where key=90 and value like '%90%';
+select * from hbase_pushdown where key=90 and value like '%90%';
+set hive.optimize.index.filter=false;
+
 -- with two residuals
 
 explain select * from hbase_pushdown

Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out?rev=1580176&r1=1580175&r2=1580176&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out Sat Mar 22 06:02:46
2014
@@ -115,6 +115,52 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@hbase_pushdown
 #### A masked pattern was here ####
 90	val_90
+PREHOOK: query: -- with partial pushdown with optimization (HIVE-6650)
+explain select * from hbase_pushdown where key=90 and value like '%90%'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- with partial pushdown with optimization (HIVE-6650)
+explain select * from hbase_pushdown where key=90 and value like '%90%'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: hbase_pushdown
+            filterExpr: (key = 90) (type: boolean)
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+            Filter Operator
+              predicate: (value like '%90%') (type: boolean)
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              Select Operator
+                expressions: key (type: int), value (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+PREHOOK: query: select * from hbase_pushdown where key=90 and value like '%90%'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_pushdown
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_pushdown where key=90 and value like '%90%'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_pushdown
+#### A masked pattern was here ####
+90	val_90
 PREHOOK: query: -- with two residuals
 
 explain select * from hbase_pushdown



Mime
View raw message