kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject [1/4] kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection. [Forced Update!]
Date Mon, 10 Apr 2017 14:13:53 GMT
Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x a8709afc2 -> de4019465 (forced update)


KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d4d741b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d4d741b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d4d741b

Branch: refs/heads/yang22-hbase1.x
Commit: 9d4d741b254d08c9a5f30921478ebf956c09b9e9
Parents: ea778d2
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Mon Apr 10 21:31:50 2017 +0800

----------------------------------------------------------------------
 .../metadata/filter/CompareTupleFilter.java     | 14 +++
 .../filter/FilterOptimizeTransformer.java       | 89 ++++++++++++++++++++
 .../gtrecord/GTCubeStorageQueryBase.java        |  5 +-
 3 files changed, 107 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..f7cfb6a 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,6 +215,20 @@ public class CompareTupleFilter extends TupleFilter {
         return ((function != null && function.isEvaluable()) || column != null) &&
!conditionValues.isEmpty();
     }
 
+    public boolean alwaysReturnTrue() {
+        // 1 = 1
+        if (this.operator == FilterOperatorEnum.EQ) {
+            if (this.children != null && this.children.size() == 2 && //
+                    this.children.get(0) instanceof ConstantTupleFilter && //
+                    this.children.get(1) instanceof ConstantTupleFilter && //
+                    ((ConstantTupleFilter) this.children.get(0)).getValues().equals(((ConstantTupleFilter)
this.children.get(1)).getValues())) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @SuppressWarnings({ "unchecked", "rawtypes" })
     @Override
     public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 0000000..a3bbad6
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * <p>
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+    public static final Logger logger = LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+    @Override
+    public TupleFilter transform(TupleFilter tupleFilter) {
+        TupleFilter translated = null;
+        if (tupleFilter instanceof CompareTupleFilter) {
+            //normal case
+            translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) tupleFilter);
+        } else if (tupleFilter instanceof LogicalTupleFilter) {
+            @SuppressWarnings("unchecked")
+            ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>)
tupleFilter.getChildren().listIterator();
+            while (childIterator.hasNext()) {
+                TupleFilter transformed = transform(childIterator.next());
+                if (transformed != null) {
+                    childIterator.set(transformed);
+                } else {
+                    throw new IllegalStateException("Should not be null");
+                }
+            }
+
+            translated = replaceAlwaysTrueLogicalFilter((LogicalTupleFilter) tupleFilter);
+
+        }
+        return translated == null ? tupleFilter : translated;
+
+    }
+
+    private TupleFilter replaceAlwaysTrueLogicalFilter(LogicalTupleFilter logicalTupleFilter)
{
+        if (logicalTupleFilter == null) {
+            return null;
+        }
+
+        if (logicalTupleFilter.getOperator() == TupleFilter.FilterOperatorEnum.OR) {
+            @SuppressWarnings("unchecked")
+            ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>)
logicalTupleFilter.getChildren().listIterator();
+            while (childIterator.hasNext()) {
+                TupleFilter next = childIterator.next();
+                if (ConstantTupleFilter.TRUE == next) {
+                    logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", logicalTupleFilter);
+                    return ConstantTupleFilter.TRUE;
+                }
+            }
+        }
+
+        return logicalTupleFilter;
+    }
+
+    private TupleFilter replaceAlwaysTrueCompareFilter(CompareTupleFilter compareTupleFilter)
{
+
+        if (compareTupleFilter != null && compareTupleFilter.alwaysReturnTrue())
{
+            logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", compareTupleFilter);
+            return ConstantTupleFilter.TRUE;
+        }
+
+        return compareTupleFilter;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 24588ad..575ca22 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,6 +38,7 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -131,6 +132,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
                 logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
                 continue;
             }
+            // optimize the filter, the optimization has to be segment-irrelevant
+            new FilterOptimizeTransformer().transform(filterD);
 
             scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics,
filterD, context);
             scanners.add(scanner);
@@ -143,7 +146,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
     }
 
     protected abstract String getGTStorage();
-    
+
     protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef>
selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo tupleInfo) {
         return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics,
tupleInfo);
     }


Mime
View raw message