kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject kylin git commit: KYLIN-1922 Improve the logic to decide whether to pre aggregate on Region server
Date Wed, 27 Jul 2016 03:30:39 GMT
Repository: kylin
Updated Branches:
  refs/heads/beforepomupdate a055b8891 -> 70dc19047


KYLIN-1922 Improve the logic to decide whether to pre aggregate on Region server


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70dc1904
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70dc1904
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70dc1904

Branch: refs/heads/beforepomupdate
Commit: 70dc190477d4b1233e36fba59c58970a5f8fc1a4
Parents: a055b88
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Wed Jul 27 11:09:26 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Wed Jul 27 11:09:26 2016 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/gridtable/GTScanRequest.java  | 12 +++++++++++-
 .../java/org/apache/kylin/storage/StorageContext.java   |  9 +++++++++
 .../kylin/storage/gtrecord/CubeSegmentScanner.java      |  2 +-
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java  | 12 ++++++++++--
 4 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index d4e6298..d1bd1b3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -33,12 +33,16 @@ import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.SerializeToByteBuffer;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 public class GTScanRequest {
 
+    private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class);
+
     private GTInfo info;
     private List<GTScanRange> ranges;
     private ImmutableBitSet columns;
@@ -165,11 +169,17 @@ public class GTScanRequest {
                 return new EmptyGTScanner(scanned);
             }
 
-            if (this.allowPreAggregation && this.hasAggregation()) {
+            if (!this.allowPreAggregation) {
+                logger.info("pre aggregation is not beneficial, skip it");
+            } else if (this.hasAggregation()) {
+                logger.info("pre aggregating results before returning");
                 result = new GTAggregateScanner(result, this);
+            } else {
+                logger.info("has no aggregation, skip it");
             }
             return result;
         }
+
     }
 
     //touch every byte of the cell so that the cost of scanning will be truly reflected

http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 90f950c..aad37ce 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -39,6 +39,7 @@ public class StorageContext {
     private boolean acceptPartialResult;
 
     private boolean exactAggregation;
+    private boolean groupingByAllCuboidDimensions;
     private boolean enableLimit;
     private boolean enableCoprocessor;
 
@@ -143,6 +144,14 @@ public class StorageContext {
         this.partialResultReturned = partialResultReturned;
     }
 
+    public boolean isGroupingByAllCuboidDimensions() {
+        return groupingByAllCuboidDimensions;
+    }
+
+    public void setGroupingByAllCuboidDimensions(boolean groupingByAllCuboidDimensions) {
+        this.groupingByAllCuboidDimensions = groupingByAllCuboidDimensions;
+    }
+
     public void setExactAggregation(boolean isExactAggregation) {
         this.exactAggregation = isExactAggregation;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 15e3272..f539179 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -69,7 +69,7 @@ public class CubeSegmentScanner implements IGTScanner {
         }
         scanRequest = scanRangePlanner.planScanRequest();
         if (scanRequest != null) {
-            scanRequest.setAllowPreAggregation(!context.isExactAggregation());
+            scanRequest.setAllowPreAggregation(!context.isGroupingByAllCuboidDimensions());
             scanRequest.setAggrCacheGB(cubeSeg.getCubeInstance().getConfig().getQueryCoprocessorMemGB());
             if (context.isLimitEnabled())
                 scanRequest.setRowLimit(context.getLimit());

http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 65aa90a..d03cfc6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -99,8 +99,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
 
         // isExactAggregation? meaning: tuples returned from storage requires no further
aggregation in query engine
         Set<TblColRef> singleValuesD = findSingleValueColumns(filter);
-        boolean isExactAggregation = isExactAggregation(cuboid, groups, otherDimsD, singleValuesD,
derivedPostAggregation);
-        context.setExactAggregation(isExactAggregation);
+        context.setExactAggregation(isExactAggregation(cuboid, groups, otherDimsD, singleValuesD,
derivedPostAggregation));
+        context.setGroupingByAllCuboidDimensions(isGroupingByAllCuboidDimensions(cuboid,
groupsD));
 
         // replace derived columns in filter with host columns; columns on loosened condition
must be added to group by
         TupleFilter filterD = translateDerived(filter, groupsD);
@@ -221,6 +221,14 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
         return resultD;
     }
 
+    private boolean isGroupingByAllCuboidDimensions(Cuboid cuboid, Collection<TblColRef>
groupD) {
+        if (cuboid.getColumns().size() != groupD.size()) {
+            return false;
+        }
+
+        return groupD.containsAll(cuboid.getColumns());
+    }
+
     private boolean isExactAggregation(Cuboid cuboid, Collection<TblColRef> groups,
Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation)
{
         boolean exact = true;
 


Mime
View raw message