kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [20/37] kylin git commit: KYLIN-2598 Should not translate filter to a in-clause filter with too many elements
Date Fri, 19 May 2017 10:04:47 GMT
KYLIN-2598 Should not translate filter to a in-clause filter with too many elements


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9dc6553d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9dc6553d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9dc6553d

Branch: refs/heads/master
Commit: 9dc6553d29eb3f106823026f45dc878347a2a108
Parents: 529b1be
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Tue May 9 18:33:21 2017 +0800
Committer: Dong Li <lidong@apache.org>
Committed: Tue May 9 20:30:21 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |  4 +++
 .../kylin/dict/BuiltInFunctionTransformer.java  | 10 +++++-
 .../test/resources/query/sql_like/query01.sql   |  6 ++--
 .../test/resources/query/sql_like/query05.sql   | 33 ++++++++++++++++++++
 .../test/resources/query/sql_like/query06.sql   | 33 ++++++++++++++++++++
 .../test/resources/query/sql_like/query22.sql   | 31 ++++++++++++++++++
 .../test/resources/query/sql_like/query23.sql   | 31 ++++++++++++++++++
 7 files changed, 144 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index eafdbbb..10209d7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -844,6 +844,10 @@ abstract public class KylinConfigBase implements Serializable {
         return value > 0 ? value : Long.MAX_VALUE;
     }
 
+    public int getTranslatedInClauseMaxSize() {
+        return Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size",
"1024"));
+    }
+
     public int getLargeQueryThreshold() {
         return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf(1000000)));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 6ef2f96..4d7b15d 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict;
 import java.util.Collection;
 import java.util.ListIterator;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.dimension.IDimensionEncodingMap;
 import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
@@ -93,12 +94,19 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer
{
         translated.addChild(new ColumnTupleFilter(columnRef));
 
         try {
+            int translatedInClauseMaxSize = KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize();
+
             for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
                 Object dictVal = dict.getValueFromId(i);
                 if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) {
                     translated.addChild(new ConstantTupleFilter(dictVal));
+
+                    if (translated.getChildren().size() > translatedInClauseMaxSize) {
+                        return null;
+                    }
                 }
             }
+
         } catch (Exception e) {
             logger.debug(e.getMessage());
             return null;
@@ -175,4 +183,4 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer
{
         }
         return translated;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql b/kylin-it/src/test/resources/query/sql_like/query01.sql
index e1f131d..e3a05ca 100644
--- a/kylin-it/src/test/resources/query/sql_like/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_like/query01.sql
@@ -16,7 +16,7 @@
 -- limitations under the License.
 --
 
-select lstg_format_name as lstg_format_name, count(*) as cnt 
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt 
  
  from test_kylin_fact 
 inner JOIN edw.test_cal_dt as test_cal_dt
@@ -27,5 +27,5 @@ inner JOIN edw.test_cal_dt as test_cal_dt
  ON test_kylin_fact.lstg_site_id = test_sites.site_id
  
  
-where lstg_format_name like '%BIN%'
-group by lstg_format_name
\ No newline at end of file
+where META_CATEG_NAME like '%ab%'
+group by META_CATEG_NAME

http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query05.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query05.sql b/kylin-it/src/test/resources/query/sql_like/query05.sql
new file mode 100644
index 0000000..6ac9758
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query05.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(meta_categ_name) as META_CATEG_NAME, count(*) as cnt 
+
+
+from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id
= test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where lower(meta_categ_name)='baby' and substring(meta_categ_name,1,3) in ('Bab') and upper(meta_categ_name)
> 'AAAA' and
+lower(meta_categ_name) like '%b%' and char_length(meta_categ_name) < 10 and char_length(meta_categ_name)
> 3 and meta_categ_name||'a'='Babya'
+group by meta_categ_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query06.sql b/kylin-it/src/test/resources/query/sql_like/query06.sql
new file mode 100644
index 0000000..6417584
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query06.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- test case for KYLIN-1954
+
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt 
+ 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id
= test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where META_CATEG_NAME like '%ab%' and META_CATEG_NAME > 'A'
+group by META_CATEG_NAME

http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query22.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query22.sql b/kylin-it/src/test/resources/query/sql_like/query22.sql
new file mode 100644
index 0000000..a962d3f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query22.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt 
+ 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id
= test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where META_CATEG_NAME not like '%ab%'
+group by META_CATEG_NAME

http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query23.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query23.sql b/kylin-it/src/test/resources/query/sql_like/query23.sql
new file mode 100644
index 0000000..b3a78c2
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query23.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt 
+ 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id
= test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where not(META_CATEG_NAME not like '%ab%')
+group by META_CATEG_NAME


Mime
View raw message