kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject kylin git commit: KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey
Date Mon, 14 Nov 2016 01:13:19 GMT
Repository: kylin
Updated Branches:
  refs/heads/yang21 c23205d93 -> 61335d9ce


KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers
in the rowkey


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c

Branch: refs/heads/yang21
Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9
Parents: c23205d
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Mon Nov 14 09:12:51 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Mon Nov 14 09:12:51 2016 +0800

----------------------------------------------------------------------
 .../gtrecord/GTCubeStorageQueryBase.java        | 22 ++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b51af59..d648469 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
         //set whether to aggr at storage
         context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD));
         // set limit push down
-        enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter,
sqlDigest.aggregations, context);
+        enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, groups, derivedPostAggregation,
groupsD, filter, sqlDigest.aggregations, context);
         context.setFinalPushDownLimit(cubeInstance);
         // set cautious threshold to prevent out of memory
         setThresholdIfNecessary(dimensionsD, metrics, context);
@@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
         }
     }
 
-    private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef>
groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD,
TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context)
{
+    private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid cuboid, Collection<TblColRef>
groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD,
TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context)
{
         boolean possible = true;
 
+        for (TblColRef col : cuboid.getColumns()) {
+            RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col);
+            if (rowKeyDesc != null) {
+                String encodingName = rowKeyDesc.getEncodingName();
+
+                //for numbers, if it's fixed_length encoding, order is no longer preserved
+                if (encodingName != null && encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME)
&& //
+                        (col.getType().isNumberFamily() || col.getType().isIntegerFamily()))
{
+                    possible = false;
+                    logger.info("Storage limit push down is impossible because integer/number
is encoded as fixedlength");
+                }
+            } else {
+                logger.warn("RowKeyColDesc for {} does not exist", col);
+            }
+        }
+
         boolean goodFilter = filter == null || TupleFilter.isEvaluableRecursively(filter);
         if (!goodFilter) {
             possible = false;


Mime
View raw message