kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [08/21] kylin git commit: KYLIN-2238 Add query server scan threshold
Date Tue, 17 Jan 2017 03:32:13 GMT
KYLIN-2238 Add query server scan threshold


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b00ec26
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b00ec26
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b00ec26

Branch: refs/heads/master-cdh5.7
Commit: 4b00ec26de1e1cc9bf9b2e55a1b162d123114df8
Parents: 20c3294
Author: kangkaisen <kangkaisen@live.com>
Authored: Tue Nov 29 19:51:39 2016 +0800
Committer: kangkaisen <kangkaisen@163.com>
Committed: Sat Jan 14 19:00:09 2017 +0800

----------------------------------------------------------------------
 .../storage/gtrecord/SequentialCubeTupleIterator.java | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/4b00ec26/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index ee868c7..c621215 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -26,6 +26,7 @@ import java.util.Set;
 
 import javax.annotation.Nullable;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -33,6 +34,7 @@ import org.apache.kylin.metadata.tuple.ITuple;
 import org.apache.kylin.metadata.tuple.ITupleIterator;
 import org.apache.kylin.metadata.tuple.TupleInfo;
 import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.exception.ScanOutOfLimitException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,6 +47,8 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
 
     private static final Logger logger = LoggerFactory.getLogger(SequentialCubeTupleIterator.class);
 
+    private final int SCAN_THRESHOLD = KylinConfig.getInstanceFromEnv().getScanThreshold();
+
     protected List<CubeSegmentScanner> scanners;
     protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators;
     protected Iterator<ITuple> tupleIterator;
@@ -78,7 +82,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
             tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed,
context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, returnTupleInfo)).getIterator();
         }
     }
-    
+
     public Comparator<ITuple> getTupleDimensionComparator(Cuboid cuboid, TupleInfo
returnTupleInfo) {
         // dimensionIndexOnTuple is for SQL with limit
         List<Integer> temp = Lists.newArrayList();
@@ -92,7 +96,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
         for (int i = 0; i < temp.size(); i++) {
             dimensionIndexOnTuple[i] = temp.get(i);
         }
-        
+
         return new Comparator<ITuple>() {
             @Override
             public int compare(ITuple o1, ITuple o2) {
@@ -137,7 +141,11 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
 
     @Override
     public ITuple next() {
-        scanCount++;
+        // prevent the big query to make the Query Server OOM
+        if (scanCount++ > SCAN_THRESHOLD) {
+            throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " +
SCAN_THRESHOLD);
+        }
+
         if (++scanCountDelta >= 1000)
             flushScanCountDelta();
 


Mime
View raw message