hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject hive git commit: HIVE-16746: Reduce number of index lookups for same table in IndexWhereTaskDispatcher (Rajesh Balamohan, reviewed by Ashutosh Chauhan)
Date Wed, 24 May 2017 22:34:18 GMT
Repository: hive
Updated Branches:
  refs/heads/master a74c1e7ce -> 3fe65a3c5


HIVE-16746: Reduce number of index lookups for same table in IndexWhereTaskDispatcher (Rajesh
Balamohan, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3fe65a3c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3fe65a3c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3fe65a3c

Branch: refs/heads/master
Commit: 3fe65a3c5b0f3bb60da0a866b9c00ec7a6448055
Parents: a74c1e7
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Thu May 25 04:03:54 2017 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Thu May 25 04:03:54 2017 +0530

----------------------------------------------------------------------
 .../index/IndexWhereTaskDispatcher.java         | 34 ++++++++++++++++----
 1 file changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/3fe65a3c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
index ae96def..7e3fb1a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
@@ -20,14 +20,16 @@ package org.apache.hadoop.hive.ql.optimizer.physical.index;
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Stack;
 
+import com.google.common.collect.Maps;
 import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.cache.CacheUtils;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
@@ -49,6 +51,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.MapredWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hive.common.util.HiveStringUtils;
 
 /**
  *
@@ -60,10 +63,17 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 public class IndexWhereTaskDispatcher implements Dispatcher {
 
   private final PhysicalContext physicalContext;
+  // To store table to index mapping
+  private final Map<String, List<Index>> indexMap;
+  private final List<String> supportedIndexes;
 
   public IndexWhereTaskDispatcher(PhysicalContext context) {
     super();
     physicalContext = context;
+    indexMap = Maps.newHashMap();
+    supportedIndexes = new ArrayList<String>();
+    supportedIndexes.add(CompactIndexHandler.class.getName());
+    supportedIndexes.add(BitmapIndexHandler.class.getName());
   }
 
   @Override
@@ -104,6 +114,21 @@ public class IndexWhereTaskDispatcher implements Dispatcher {
     return null;
   }
 
+  private List<Index> getIndex(Table table) throws SemanticException {
+    String indexCacheKey = CacheUtils.buildKey(
+        HiveStringUtils.normalizeIdentifier(table.getDbName()),
+        HiveStringUtils.normalizeIdentifier(table.getTableName()));
+    List<Index>indexList = indexMap.get(indexCacheKey);
+    if (indexList == null) {
+      indexList =  IndexUtils.getIndexes(table, supportedIndexes);
+      if (indexList == null) {
+        indexList = Collections.emptyList();
+      }
+      indexMap.put(indexCacheKey, indexList);
+    }
+    return indexList;
+  }
+
   /**
    * Create a set of rules that only matches WHERE predicates on columns we have
    * an index on.
@@ -112,16 +137,11 @@ public class IndexWhereTaskDispatcher implements Dispatcher {
   private Map<Rule, NodeProcessor> createOperatorRules(ParseContext pctx) throws SemanticException
{
     Map<Rule, NodeProcessor> operatorRules = new LinkedHashMap<Rule, NodeProcessor>();
 
-    List<String> supportedIndexes = new ArrayList<String>();
-    supportedIndexes.add(CompactIndexHandler.class.getName());
-    supportedIndexes.add(BitmapIndexHandler.class.getName());
-
     // query the metastore to know what columns we have indexed
     Map<TableScanOperator, List<Index>> indexes = new HashMap<TableScanOperator,
List<Index>>();
     for (Operator<? extends OperatorDesc> op : pctx.getTopOps().values()) {
       if (op instanceof TableScanOperator) {
-        List<Index> tblIndexes = IndexUtils.getIndexes(((TableScanOperator) op).getConf()
-            .getTableMetadata(), supportedIndexes);
+        List<Index> tblIndexes = getIndex(((TableScanOperator) op).getConf().getTableMetadata());
         if (tblIndexes.size() > 0) {
           indexes.put((TableScanOperator) op, tblIndexes);
         }


Mime
View raw message