kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [02/42] kylin git commit: KYLIN-2301 various fixes to support NOT IN subquery
Date Fri, 30 Dec 2016 02:30:54 GMT
KYLIN-2301 various fixes to support NOT IN subquery


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0453d3da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0453d3da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0453d3da

Branch: refs/heads/sparkcubing-rebase
Commit: 0453d3daada2e32f79954dfd12ac9f19062e825c
Parents: ed8b168
Author: Li Yang <liyang@apache.org>
Authored: Fri Dec 23 08:17:28 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Mon Dec 26 15:26:25 2016 +0800

----------------------------------------------------------------------
 .../kylin/metadata/filter/LogicalTupleFilter.java      | 13 -------------
 .../org/apache/kylin/metadata/filter/TupleFilter.java  |  7 ++++++-
 .../org/apache/kylin/metadata/model/TblColRef.java     |  7 ++-----
 .../java/org/apache/kylin/query/ITKylinQueryTest.java  |  2 +-
 .../src/test/resources/query/sql_subquery/query13.sql  |  7 +++++++
 .../apache/kylin/query/relnode/OLAPAggregateRel.java   | 11 +++++++++--
 .../org/apache/kylin/query/relnode/OLAPJoinRel.java    |  8 +++++++-
 7 files changed, 32 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 1744309..373acdb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -47,19 +47,6 @@ public class LogicalTupleFilter extends TupleFilter {
         return cloneTuple;
     }
 
-    //    private TupleFilter reverseNestedNots(TupleFilter filter, int depth) {
-    //        if ((filter instanceof LogicalTupleFilter) && (filter.operator == FilterOperatorEnum.NOT))
{
-    //            assert (filter.children.size() == 1);
-    //            return reverseNestedNots(filter.children.get(0), depth + 1);
-    //        }
-    //
-    //        if (depth % 2 == 1) {
-    //            return filter;
-    //        } else {
-    //            return filter.reverse();
-    //        }
-    //    }
-
     @Override
     public TupleFilter reverse() {
         switch (operator) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 285172c..31de50b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -27,6 +27,8 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Maps;
 
@@ -37,6 +39,8 @@ import com.google.common.collect.Maps;
  */
 public abstract class TupleFilter {
 
+    static final Logger logger = LoggerFactory.getLogger(TupleFilter.class);
+
     public enum FilterOperatorEnum {
         EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10),
AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35),
MASSIN(36), EVAL_FUNC(37), UNSUPPORTED(38);
 
@@ -112,7 +116,8 @@ public abstract class TupleFilter {
     }
 
     public TupleFilter reverse() {
-        throw new UnsupportedOperationException();
+        logger.warn("Cannot reverse " + this + ", loosen the filter to true");
+        return ConstantTupleFilter.TRUE;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index fd0224f..88abb9d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -27,7 +27,7 @@ import org.apache.kylin.metadata.datatype.DataType;
 
 /**
  */
-@SuppressWarnings("serial")
+@SuppressWarnings({ "serial", "deprecation" })
 public class TblColRef implements Serializable {
 
     private static final String INNER_TABLE_NAME = "_kylin_table";
@@ -82,6 +82,7 @@ public class TblColRef implements Serializable {
         TableRef tableRef = model.findTable(alias);
         checkArgument(tableRef.getTableDesc() == col.column.getTable());
         col.table = tableRef;
+        col.identity = null;
     }
 
     // for test mainly
@@ -160,10 +161,6 @@ public class TblColRef implements Serializable {
         return InnerDataTypeEnum.contains(getDatatype());
     }
 
-    public boolean isDerivedDataType() {
-        return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype());
-    }
-
     public int hashCode() {
         // NOTE: tableRef MUST NOT participate in hashCode().
         // Because fixUnknownModel() can change tableRef while TblColRef is held as set/map
keys.

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 4329141..840a03f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -149,7 +149,7 @@ public class ITKylinQueryTest extends KylinTestBase {
     @Test
     public void testSingleRunQuery() throws Exception {
 
-        String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_limit/query01.sql";
+        String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/temp/sample.sql";
 
         File sqlFile = new File(queryFileName);
         if (sqlFile.exists()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/kylin-it/src/test/resources/query/sql_subquery/query13.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query13.sql b/kylin-it/src/test/resources/query/sql_subquery/query13.sql
new file mode 100644
index 0000000..d7f6109
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query13.sql
@@ -0,0 +1,7 @@
+select
+    f.cal_dt
+from test_kylin_fact f
+where
+    f.cal_dt not in (
+        select cal_dt from EDW.TEST_CAL_DT where week_beg_dt = date'2012-01-01'
+    )

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index c457545..8d7c597 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -153,7 +153,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
 
         // only translate the innermost aggregation
         if (!this.afterAggregate) {
-            this.context.groupByColumns.addAll(this.groups);
+            addToContextGroupBy(this.groups);
             this.context.aggregations.addAll(this.aggregations);
             this.context.afterAggregate = true;
 
@@ -326,7 +326,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
             FunctionDesc aggFunc = this.aggregations.get(i);
 
             if (aggFunc.isDimensionAsMetric()) {
-                this.context.groupByColumns.addAll(aggFunc.getParameter().getColRefs());
+                addToContextGroupBy(aggFunc.getParameter().getColRefs());
                 continue; // skip rewrite, let calcite handle
             }
 
@@ -350,6 +350,13 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
         }
     }
 
+    private void addToContextGroupBy(List<TblColRef> colRefs) {
+        for (TblColRef col : colRefs) {
+            if (col.isInnerColumn() == false)
+                this.context.groupByColumns.add(col);
+        }
+    }
+
     private void fillbackOptimizedColumn() {
         // some aggcall will be optimized out in sub-query (e.g. tableau generated sql),
we need to fill them back
         RelDataType inputAggRow = getInput().getRowType();

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 6869858..54b851f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -164,7 +164,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
         } else if (leftHasSubquery != rightHasSubquery) {
             //When join contains subquery, the join-condition fields of fact_table will add
into context.
             Map<TblColRef, TblColRef> joinCol = new HashMap<TblColRef, TblColRef>();
-            translateJoinColumn((RexCall) this.getCondition(), joinCol);
+            translateJoinColumn(this.getCondition(), joinCol);
 
             for (Map.Entry<TblColRef, TblColRef> columnPair : joinCol.entrySet()) {
                 TblColRef fromCol = (rightHasSubquery ? columnPair.getKey() : columnPair.getValue());
@@ -219,6 +219,12 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
         return join;
     }
 
+    private void translateJoinColumn(RexNode condition, Map<TblColRef, TblColRef> joinCol)
{
+        if (condition instanceof RexCall) {
+            translateJoinColumn((RexCall) condition, joinCol);
+        }
+    }
+
     private void translateJoinColumn(RexCall condition, Map<TblColRef, TblColRef> joinColumns)
{
         SqlKind kind = condition.getOperator().getKind();
         if (kind == SqlKind.AND) {


Mime
View raw message