kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billy...@apache.org
Subject [09/15] kylin git commit: KYLIN-2290 minor improvements on limit
Date Mon, 19 Dec 2016 11:23:32 GMT
KYLIN-2290 minor improvements on limit

KYLIN-2290 fix compile

KYLIN-2290 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bd4a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bd4a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bd4a9

Branch: refs/heads/KYLIN-2293
Commit: 6f9bd4a9998cff24c97bce0e6d86282f48fef388
Parents: 699a88e
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Fri Dec 16 16:28:18 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Mon Dec 19 14:16:00 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |  1 +
 .../kylin/cube/CubeCapabilityChecker.java       |  5 ++
 .../kylin/gridtable/GTScanRequestBuilder.java   |  2 +-
 .../kylin/metadata/realization/SQLDigest.java   |  4 +-
 .../apache/kylin/storage/StorageContext.java    | 74 +++++++-------------
 .../storage/gtrecord/CubeScanRangePlanner.java  |  9 +--
 .../gtrecord/GTCubeStorageQueryBase.java        |  1 -
 .../gtrecord/SequentialCubeTupleIterator.java   |  6 +-
 .../apache/kylin/query/ITKylinQueryTest.java    |  1 -
 .../org/apache/kylin/query/KylinTestBase.java   |  7 +-
 .../kylin/storage/hbase/ITStorageTest.java      |  2 +-
 .../kylin/query/relnode/OLAPAggregateRel.java   |  4 ++
 .../apache/kylin/query/relnode/OLAPContext.java |  6 +-
 .../kylin/query/relnode/OLAPFilterRel.java      |  2 +-
 .../kylin/query/relnode/OLAPLimitRel.java       |  6 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  7 +-
 16 files changed, 60 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 610c2af..5153562 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -778,6 +778,7 @@ abstract public class KylinConfigBase implements Serializable {
         return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
     }
 
+    @Deprecated//Limit is good even it's large. This config is meaning less since we already
have scan threshold 
     public int getStoragePushDownLimitMax() {
         return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 38faed9..c45144b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -120,6 +120,11 @@ public class CubeCapabilityChecker {
             return result;
         }
 
+        if (digest.limitPrecedesAggr) {
+            logger.info("Exclude cube " + cube.getName() + " because there's limit preceding
aggregation");
+            return result;
+        }
+
         if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) {
             result.influences.add(new CapabilityInfluence() {
                 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index f542de1..bcec1f4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -36,7 +36,7 @@ public class GTScanRequestBuilder {
     private boolean allowStorageAggregation = true;
     private double aggCacheMemThreshold = 0;
     private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate
itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception.   
-    private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit working when
$toragePushDownLimit aggregated rows are produced. 
+    private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit scanning safely
when $toragePushDownLimit aggregated rows are produced. 
     private long startTime = -1;
     private long timeout = -1;
     private String storageBehavior = null;

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 83fc05c..36f303b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -59,10 +59,11 @@ public class SQLDigest {
     public List<TblColRef> sortColumns;
     public List<OrderEnum> sortOrders;
     public boolean isRawQuery;
+    public boolean limitPrecedesAggr;
 
     public SQLDigest(String factTable, TupleFilter filter, List<JoinDesc> joinDescs,
Set<TblColRef> allColumns, //
             List<TblColRef> groupbyColumns, Set<TblColRef> subqueryJoinParticipants,
Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, //
-            List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef>
sortColumns, List<OrderEnum> sortOrders) {
+            List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef>
sortColumns, List<OrderEnum> sortOrders, boolean limitPrecedesAggr) {
         this.factTable = factTable;
         this.filter = filter;
         this.joinDescs = joinDescs;
@@ -76,6 +77,7 @@ public class SQLDigest {
         this.sortColumns = sortColumns;
         this.sortOrders = sortOrders;
         this.isRawQuery = isRawQuery();
+        this.limitPrecedesAggr = limitPrecedesAggr;
     }
 
     private boolean isRawQuery() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index bc43a87..9ef59fd 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -34,43 +34,29 @@ import com.google.common.collect.Range;
 public class StorageContext {
     private static final Logger logger = LoggerFactory.getLogger(StorageContext.class);
 
-    public static final int DEFAULT_THRESHOLD = 1000000;
-
     private String connUrl;
     private int threshold;
-    private int limit;
-    private int offset;
-    private int finalPushDownLimit;
-    private boolean hasSort;
-    private boolean acceptPartialResult;
-
-    private boolean exactAggregation;
-    private boolean needStorageAggregation;
-    private boolean enableLimit;
-    private boolean enableCoprocessor;
-
-    private AtomicLong totalScanCount;
+    private int limit = Integer.MAX_VALUE;
+    private int offset = 0;
+    private int finalPushDownLimit = Integer.MAX_VALUE;
+    private boolean hasSort = false;
+    private boolean acceptPartialResult = false;
+
+    private boolean exactAggregation = false;
+    private boolean needStorageAggregation = false;
+    private boolean limitEnabled = false;
+    private boolean enableCoprocessor = false;
+
+    private AtomicLong totalScanCount = new AtomicLong();
     private Cuboid cuboid;
-    private boolean partialResultReturned;
-
-    private Range<Long> reusedPeriod;
+    private boolean partialResultReturned = false;
 
     public StorageContext() {
-        this.threshold = DEFAULT_THRESHOLD;
-        this.limit = DEFAULT_THRESHOLD;
-        this.totalScanCount = new AtomicLong();
-        this.cuboid = null;
-        this.hasSort = false;
-
-        this.exactAggregation = false;
-        this.enableLimit = false;
-        this.enableCoprocessor = false;
-
-        this.acceptPartialResult = false;
-        this.partialResultReturned = false;
-        this.finalPushDownLimit = Integer.MAX_VALUE;
+        this.threshold = KylinConfig.getInstanceFromEnv().getScanThreshold();
     }
 
+    private Range<Long> reusedPeriod;
+
     public String getConnUrl() {
         return connUrl;
     }
@@ -92,11 +78,10 @@ public class StorageContext {
     }
 
     public void setLimit(int l) {
-        if (l > limit) {
-            //cases like : select price from (select * from kylin_sales limit 10) limit 5000
-            logger.info("Setting limit to {} but in current olap context, the limit is already
{}, won't apply", l, limit);
+        if (limit != Integer.MAX_VALUE) {
+            logger.warn("Setting limit to {} but in current olap context, the limit is already
{}, won't apply", l, limit);
         } else {
-            this.limit = l;
+            limit = l;
         }
     }
 
@@ -109,15 +94,11 @@ public class StorageContext {
     }
 
     public void enableLimit() {
-        this.enableLimit = true;
+        this.limitEnabled = true;
     }
 
     public boolean isLimitEnabled() {
-        return this.enableLimit;
-    }
-
-    private int getStoragePushDownLimit() {
-        return this.isLimitEnabled() ? this.getOffset() + this.getLimit() : Integer.MAX_VALUE;
+        return this.limitEnabled;
     }
 
     public int getFinalPushDownLimit() {
@@ -126,19 +107,16 @@ public class StorageContext {
 
     public void setFinalPushDownLimit(IRealization realization) {
 
-        //decide the final limit push down
-        int tempPushDownLimit = this.getStoragePushDownLimit();
-        if (tempPushDownLimit == Integer.MAX_VALUE) {
+        if (this.getLimit() == Integer.MAX_VALUE) {
             return;
         }
 
-        int pushDownLimitMax = KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax();
+        int tempPushDownLimit = this.getOffset() + this.getLimit();
+
         if (!realization.supportsLimitPushDown()) {
-            logger.info("Not enabling limit push down because cube storage type not supported");
-        } else if (tempPushDownLimit > pushDownLimitMax) {
-            logger.info("Not enabling limit push down because the limit(including offset)
{} is larger than kylin.query.max-limit-pushdown {}", //
-                    tempPushDownLimit, pushDownLimitMax);
+            logger.warn("Not enabling limit push down because cube storage type not supported");
         } else {
+            this.limitEnabled = true;
             this.finalPushDownLimit = tempPushDownLimit;
             logger.info("Enable limit: " + tempPushDownLimit);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
index 8d5a3d4..b05a629 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -148,15 +148,10 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase {
         GTScanRequest scanRequest;
         List<GTScanRange> scanRanges = this.planScanRanges();
         if (scanRanges != null && scanRanges.size() != 0) {
-            GTScanRequestBuilder builder = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
+            scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
                     setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).//
                     setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getCubeInstance().getConfig().getQueryCoprocessorMemGB()).//
-                    setStorageScanRowNumThreshold(context.getThreshold());
-
-            if (context.getFinalPushDownLimit() != Integer.MAX_VALUE)
-                builder.setStoragePushDownLimit(context.getFinalPushDownLimit());
-
-            scanRequest = builder.createGTScanRequest();
+                    setStoragePushDownLimit(context.getFinalPushDownLimit()).setStorageScanRowNumThreshold(context.getThreshold()).createGTScanRequest();
         } else {
             scanRequest = null;
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 31573d0..7b0cbb9 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -383,7 +383,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
         }
 
         if (possible) {
-            context.enableLimit();
             context.setFinalPushDownLimit(cubeInstance);
         }
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index bef0e88..3a64de7 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -46,7 +46,6 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
     protected List<CubeSegmentScanner> scanners;
     protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators;
     protected Iterator<ITuple> tupleIterator;
-    protected final int storagePushDownLimit;
     protected StorageContext context;
 
     private int scanCount;
@@ -62,8 +61,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
             segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions,
selectedMetrics, returnTupleInfo, context));
         }
 
-        this.storagePushDownLimit = context.getFinalPushDownLimit();
-        if (storagePushDownLimit == Integer.MAX_VALUE) {
+        if (!context.isLimitEnabled()) {
             //normal case
             tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
         } else {
@@ -75,7 +73,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
                     return input;
                 }
             });
-            tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed,
storagePushDownLimit, segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator();
+            tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed,
context.getFinalPushDownLimit(), segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator();
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 7f04bbb..c2aeabd 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -331,7 +331,6 @@ public class ITKylinQueryTest extends KylinTestBase {
             List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix()
+ "src/test/resources/query/sql_limit"), ".sql");
             for (File sqlFile : sqlFiles) {
                 runSQL(sqlFile, false, false);
-                assertTrue(checkLimitEnabled());
                 assertTrue(checkFinalPushDownLimit());
             }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index ddb996c..47b9e37 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -653,14 +653,9 @@ public class KylinTestBase {
 
     }
 
-    protected boolean checkLimitEnabled() {
-        OLAPContext context = getFirstOLAPContext();
-        return (context.storageContext.isLimitEnabled());
-    }
-
     protected boolean checkFinalPushDownLimit() {
         OLAPContext context = getFirstOLAPContext();
-        return (context.storageContext.getFinalPushDownLimit() != Integer.MAX_VALUE);
+        return context.storageContext.isLimitEnabled();
 
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index db9d133..aea8bef 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
@@ -148,7 +148,7 @@ public class ITStorageTest extends HBaseMetadataTestCase {
         int count = 0;
         ITupleIterator iterator = null;
         try {
-            SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null,
Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef>
emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall>
emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>());
+            SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null,
Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef>
emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall>
emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(),
false);
             iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups,
aggregations));
             while (iterator.hasNext()) {
                 ITuple tuple = iterator.next();

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index f5fa74d..c457545 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -156,6 +156,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
             this.context.groupByColumns.addAll(this.groups);
             this.context.aggregations.addAll(this.aggregations);
             this.context.afterAggregate = true;
+
+            if (this.context.afterLimit) {
+                this.context.limitPrecedesAggr = true;
+            }
         } else {
             for (AggregateCall aggCall : aggCalls) {
                 // check if supported by kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 8278fb0..dde98a6 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -112,7 +112,9 @@ public class OLAPContext {
     public Set<OLAPTableScan> allTableScans = new HashSet<>();
     public TupleInfo returnTupleInfo = null;
     public boolean afterAggregate = false;
-    public boolean afterSkippedFilter = false;
+    public boolean afterHavingClauseFilter = false;
+    public boolean afterLimit = false;
+    public boolean limitPrecedesAggr = false;
     public boolean afterJoin = false;
     public boolean hasJoin = false;
 
@@ -148,7 +150,7 @@ public class OLAPContext {
 
     public SQLDigest getSQLDigest() {
         if (sqlDigest == null)
-            sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns,
groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls,
sortColumns, sortOrders);
+            sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns,
groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls,
sortColumns, sortOrders, limitPrecedesAggr);
         return sqlDigest;
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 8023df4..151131e 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -306,7 +306,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
         if (!context.afterAggregate) {
             translateFilter(context);
         } else {
-            context.afterSkippedFilter = true;//having clause is skipped
+            context.afterHavingClauseFilter = true;//having clause is skipped
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index f0af863..8179807 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -77,7 +77,9 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
         this.columnRowType = buildColumnRowType();
         this.context = implementor.getContext();
 
-        if (!context.afterSkippedFilter) {
+        // ignore limit after having clause
+        // ignore limit after another limit, e.g. select A, count(*) from (select A,B from
fact group by A,B limit 100) limit 10
+        if (!context.afterHavingClauseFilter && !context.afterLimit) {
             Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
             int limit = limitValue.intValue();
             this.context.storageContext.setLimit(limit);
@@ -87,6 +89,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
                 int offset = offsetValue.intValue();
                 this.context.storageContext.setOffset(offset);
             }
+
+            context.afterLimit = true;
         }
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index da9c932..38efecc 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -254,14 +254,15 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService
implement
                 @Override
                 public boolean hasNext() {
 
+                    counter++;
+
                     if (counter > scanReq.getStorageScanRowNumThreshold()) {
-                        throw new GTScanExceedThresholdException("Exceed scan threshold at
" + counter);
+                        throw new GTScanExceedThresholdException("Exceed scan threshold at
" + counter + ", consider increasing kylin.query.memory-budget-bytes and kylin.query.scan-threshold");
                     }
 
                     if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) {
-                        logger.info("Scanned " + counter + " rows from HBase.");
+                        logger.info("scanning " + counter + "th row from HBase.");
                     }
-                    counter++;
                     return allCellLists.hasNext();
                 }
 


Mime
View raw message