kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject incubator-kylin git commit: KYLIN-942 only check mem use at query time
Date Mon, 02 Nov 2015 10:48:33 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/2.x-staging 2f865b80e -> 101ef86ac


KYLIN-942 only check mem use at query time


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/101ef86a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/101ef86a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/101ef86a

Branch: refs/heads/2.x-staging
Commit: 101ef86ac8a411d290d4bde80eeb29d91afcfa95
Parents: 2f865b8
Author: honma <honma@ebay.com>
Authored: Mon Nov 2 18:52:36 2015 +0800
Committer: honma <honma@ebay.com>
Committed: Mon Nov 2 18:52:58 2015 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/util/MemoryBudgetController.java    | 2 +-
 .../org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java     | 2 +-
 .../java/org/apache/kylin/gridtable/GTAggregateScanner.java     | 5 +++--
 .../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 2c26666..dc49b17 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -57,7 +57,7 @@ public class MemoryBudgetController {
 
     public static final MemoryBudgetController ZERO_BUDGET = new MemoryBudgetController(0);
     public static final int ONE_MB = 1024 * 1024;
-    public static final int ONE_GB = 1024 * 1024 * 1024;
+    public static final long ONE_GB = 1024 * 1024 * 1024;
 
     private static final Logger logger = LoggerFactory.getLogger(MemoryBudgetController.class);
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index d7aa50a..a179d70 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -343,7 +343,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
 
         Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId,
measureCount);
         GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(),
dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
-        GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, false);
+        GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true);
 
         long startTime = System.currentTimeMillis();
         logger.info("Calculating cuboid " + baseCuboidId);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 35c1210..cd2881e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -142,8 +142,9 @@ public class GTAggregateScanner implements IGTScanner {
 
         void aggregate(GTRecord r) {
             if (enableMemCheck && (++aggregatedRowCount % 100000 == 0)) {
-                if (estimatedMemSize() > MemoryBudgetController.ONE_GB) {
-                    throw new RuntimeException("AggregationCache exceed 1GB");
+                long estimated = estimatedMemSize();
+                if (estimated > 10 * MemoryBudgetController.ONE_GB) {
+                    throw new RuntimeException("AggregationCache exceed 10GB, estimated size
is: " + estimated);
                 }
             }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index ba997ec..2c284c9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -118,7 +118,7 @@ public class GTScanRequest {
     }
 
     public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
-        return decorateScanner(scanner, true, true, true);
+        return decorateScanner(scanner, true, true, false);//by default do not check mem
     }
 
     /**


Mime
View raw message