kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject kylin git commit: KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable
Date Thu, 14 Apr 2016 04:13:34 GMT
Repository: kylin
Updated Branches:
  refs/heads/yang-m1 96eca30d8 -> 29b3914a2


KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29b3914a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29b3914a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29b3914a

Branch: refs/heads/yang-m1
Commit: 29b3914a26a1d94ecfbb89ed942d067966bdf86a
Parents: 96eca30
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Thu Apr 14 12:12:13 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Thu Apr 14 12:12:13 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |  4 +++
 .../kylin/gridtable/GTScanRangePlanner.java     | 35 ++++++++++++++------
 2 files changed, 28 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3914a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4a5106e..94d8f62 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -480,6 +480,10 @@ abstract public class KylinConfigBase implements Serializable {
         return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function",
"false"));
     }
 
+    public int getQueryScanFuzzyKeyMax() {
+        return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+    }
+
     public int getHBaseKeyValueSize() {
         return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760"));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3914a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 2307aaf..90b97ad 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -18,9 +18,19 @@
 
 package org.apache.kylin.gridtable;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.ImmutableBitSet;
@@ -35,13 +45,15 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.*;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 
 public class GTScanRangePlanner {
 
     private static final Logger logger = LoggerFactory.getLogger(GTScanRangePlanner.class);
 
-    private static final int MAX_HBASE_FUZZY_KEYS = 100;
+    private int maxFuzzyKey = 200;
 
     final private GTInfo info;
     final private Pair<ByteArray, ByteArray> segmentStartAndEnd;
@@ -70,6 +82,8 @@ public class GTScanRangePlanner {
         this.rangeEndComparator = getRangeEndComparator(comp);
         //start key GTRecord compare to stop key GTRecord
         this.rangeStartEndComparator = getRangeStartEndComparator(comp);
+
+        maxFuzzyKey = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
     }
 
     // return empty list meaning filter is always false
@@ -116,11 +130,11 @@ public class GTScanRangePlanner {
             if (partitionColRef != null && range.column.equals(partitionColRef))
{
                 if (rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(),
range.end) <= 0 //
                         && (rangeStartEndComparator.comparator.compare(range.begin,
segmentStartAndEnd.getSecond()) < 0 //
-                        || rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond())
== 0 //
-                        && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE
|| range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) {
+                                || rangeStartEndComparator.comparator.compare(range.begin,
segmentStartAndEnd.getSecond()) == 0 //
+                                        && (range.op == FilterOperatorEnum.EQ ||
range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN)))
{
                     //segment range is [Closed,Open), but segmentStartAndEnd.getSecond()
might be rounded, so use <= when has equals in condition. 
                 } else {
-                    logger.debug("Pre-check partition col filter failed, partitionColRef
{}, segment start {}, segment end {}, range begin {}, range end {}",//
+                    logger.debug("Pre-check partition col filter failed, partitionColRef
{}, segment start {}, segment end {}, range begin {}, range end {}", //
                             new Object[] { partitionColRef, makeReadable(segmentStartAndEnd.getFirst()),
makeReadable(segmentStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end)
});
                     return null;
                 }
@@ -154,9 +168,8 @@ public class GTScanRangePlanner {
             logger.info("The execution of this query will not use fuzzy key");
             return result;
         }
-        
 
-        List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet,
MAX_HBASE_FUZZY_KEYS);
+        List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet,
maxFuzzyKey);
 
         for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) {
 
@@ -332,7 +345,7 @@ public class GTScanRangePlanner {
 
         // if any range is non-fuzzy, then all fuzzy keys must be cleared
         // also too many fuzzy keys will slow down HBase scan
-        if (hasNonFuzzyRange || newFuzzyKeys.size() > MAX_HBASE_FUZZY_KEYS) {
+        if (hasNonFuzzyRange || newFuzzyKeys.size() > maxFuzzyKey) {
             newFuzzyKeys.clear();
         }
 


Mime
View raw message