kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kangkai...@apache.org
Subject [36/50] [abbrv] kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection.
Date Thu, 13 Apr 2017 11:22:02 GMT
KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/177079d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/177079d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/177079d1

Branch: refs/heads/KYLIN-2506
Commit: 177079d10c1dfd4decadd743eafdfc36a712e470
Parents: c17b3eb
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Mon Apr 10 21:22:09 2017 +0800

----------------------------------------------------------------------
 .../metadata/filter/CompareTupleFilter.java     | 14 +++
 .../filter/FilterOptimizeTransformer.java       | 89 ++++++++++++++++++++
 .../gtrecord/GTCubeStorageQueryBase.java        |  6 +-
 3 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d567c89..d783e52 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -218,6 +218,20 @@ public class CompareTupleFilter extends TupleFilter {
                 && !conditionValues.isEmpty() && secondColumn == null;
     }
 
+    public boolean alwaysReturnTrue() {
+        // 1 = 1
+        if (this.operator == FilterOperatorEnum.EQ) {
+            if (this.children != null && this.children.size() == 2 && //
+                    this.children.get(0) instanceof ConstantTupleFilter && //
+                    this.children.get(1) instanceof ConstantTupleFilter && //
+                    ((ConstantTupleFilter) this.children.get(0)).getValues().equals(((ConstantTupleFilter)
this.children.get(1)).getValues())) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @SuppressWarnings({ "unchecked", "rawtypes" })
     @Override
     public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 0000000..a3bbad6
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * <p>
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+    public static final Logger logger = LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+    @Override
+    public TupleFilter transform(TupleFilter tupleFilter) {
+        TupleFilter translated = null;
+        if (tupleFilter instanceof CompareTupleFilter) {
+            //normal case
+            translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) tupleFilter);
+        } else if (tupleFilter instanceof LogicalTupleFilter) {
+            @SuppressWarnings("unchecked")
+            ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>)
tupleFilter.getChildren().listIterator();
+            while (childIterator.hasNext()) {
+                TupleFilter transformed = transform(childIterator.next());
+                if (transformed != null) {
+                    childIterator.set(transformed);
+                } else {
+                    throw new IllegalStateException("Should not be null");
+                }
+            }
+
+            translated = replaceAlwaysTrueLogicalFilter((LogicalTupleFilter) tupleFilter);
+
+        }
+        return translated == null ? tupleFilter : translated;
+
+    }
+
+    private TupleFilter replaceAlwaysTrueLogicalFilter(LogicalTupleFilter logicalTupleFilter)
{
+        if (logicalTupleFilter == null) {
+            return null;
+        }
+
+        if (logicalTupleFilter.getOperator() == TupleFilter.FilterOperatorEnum.OR) {
+            @SuppressWarnings("unchecked")
+            ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>)
logicalTupleFilter.getChildren().listIterator();
+            while (childIterator.hasNext()) {
+                TupleFilter next = childIterator.next();
+                if (ConstantTupleFilter.TRUE == next) {
+                    logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", logicalTupleFilter);
+                    return ConstantTupleFilter.TRUE;
+                }
+            }
+        }
+
+        return logicalTupleFilter;
+    }
+
+    private TupleFilter replaceAlwaysTrueCompareFilter(CompareTupleFilter compareTupleFilter)
{
+
+        if (compareTupleFilter != null && compareTupleFilter.alwaysReturnTrue())
{
+            logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", compareTupleFilter);
+            return ConstantTupleFilter.TRUE;
+        }
+
+        return compareTupleFilter;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 065fd6e..abac33a 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,6 +38,7 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -148,13 +149,16 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
         // set query deadline
         context.setDeadline(cubeInstance);
 
+        // optimize the filter, the optimization has to be segment-irrelevant
+        new FilterOptimizeTransformer().transform(filterD);
+
         logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={},
storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(),
context.isNeedStorageAggregation());
 
         return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD,
context);
     }
 
     protected abstract String getGTStorage();
-    
+
     protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef>
selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo)
{
         return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics,
gtColIdx, tupleInfo);
     }


Mime
View raw message