kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject kylin git commit: KYLIN-1954 refind and fix CI
Date Tue, 30 Aug 2016 03:05:25 GMT
Repository: kylin
Updated Branches:
  refs/heads/master cc9acbc2e -> fdad2d8d3


KYLIN-1954 refind and fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdad2d8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdad2d8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdad2d8d

Branch: refs/heads/master
Commit: fdad2d8d3ff34b069f646794226bcb73543a9474
Parents: cc9acbc
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Tue Aug 30 09:55:57 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Tue Aug 30 11:05:26 2016 +0800

----------------------------------------------------------------------
 .../kylin/job/dataGen/FactTableGenerator.java   | 10 +++--
 .../kylin/dict/BuildInFunctionTransformer.java  |  5 ++-
 .../localmeta/data/data_gen_config.json         |  2 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |  4 +-
 .../apache/kylin/query/ITCombinationTest.java   | 14 ++++--
 .../apache/kylin/query/ITKylinQueryTest.java    | 13 +++++-
 .../test/resources/query/sql_like/query01.sql   | 31 +++++++++++++
 .../test/resources/query/sql_like/query02.sql   | 22 +++++++++
 .../apache/kylin/query/routing/RoutingRule.java |  2 +
 .../rules/RemoveBlackoutRealizationsRule.java   | 47 ++++++++++++++++++++
 10 files changed, 136 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 9373a02..5aac4de 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -586,10 +586,12 @@ public class FactTableGenerator {
         }
 
         for (Integer index : differentiateColumns) {
-            if (currentRowTime >= differentiateBoundary) {
-                columnValues.set(index, columnValues.get(index) + "_B");
-            } else {
-                columnValues.set(index, columnValues.get(index) + "_A");
+            if (r.nextBoolean()) {//only change half of data
+                if (currentRowTime >= differentiateBoundary) {
+                    columnValues.set(index, columnValues.get(index) + "_B");
+                } else {
+                    columnValues.set(index, columnValues.get(index) + "_A");
+                }
             }
         }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
index 9156b67..665a7e6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
@@ -70,8 +70,11 @@ public class BuildInFunctionTransformer implements ITupleFilterTransformer
{
             ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>)
tupleFilter.getChildren().listIterator();
             while (childIterator.hasNext()) {
                 TupleFilter transformed = transform(childIterator.next());
-                if (transformed != null)
+                if (transformed != null) {
                     childIterator.set(transformed);
+                } else {
+                    throw new IllegalStateException("Should not be null");
+                }
             }
         }
         return translated == null ? tupleFilter : translated;

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/examples/test_case_data/localmeta/data/data_gen_config.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json b/examples/test_case_data/localmeta/data/data_gen_config.json
index f730058..16ba3d5 100644
--- a/examples/test_case_data/localmeta/data/data_gen_config.json
+++ b/examples/test_case_data/localmeta/data/data_gen_config.json
@@ -1,5 +1,5 @@
 {
-  "differentiateBoundary": "2013-01-01",
+  "differentiateBoundary": "2013-07-01",
   "columnConfigs": [
     {
       "columnName": "lstg_format_name",

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index ca3a9bc..4c847bf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -217,7 +217,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
 
         PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price)
as GMV, count(1) as TRANS_CNT from test_kylin_fact " + "where LSTG_FORMAT_NAME = ? group by
LSTG_FORMAT_NAME");
 
-        statement.setString(1, "FP-GTC");
+        statement.setString(1, "FP-GTC_A");
 
         ResultSet rs = statement.executeQuery();
 
@@ -225,7 +225,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
 
         String format_name = rs.getString(1);
 
-        Assert.assertTrue("FP-GTC".equals(format_name));
+        Assert.assertTrue("FP-GTC_A".equals(format_name));
 
         rs.close();
         statement.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index cf18b20..cbd4e44 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -25,6 +25,7 @@ import java.util.Map;
 
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.storage.hbase.HBaseStorage;
 import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
 import org.junit.AfterClass;
@@ -64,13 +65,12 @@ public class ITCombinationTest extends ITKylinQueryTest {
      */
     @Parameterized.Parameters
     public static Collection<Object[]> configs() {
-        //       return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset"
}, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, });
-        return Arrays.asList(new Object[][] { { "inner", "on", "v2" }, { "left", "on", "v1"
}, { "left", "on", "v2" } });
+        return Arrays.asList(new Object[][] { { "inner", "on", "v2", false }, { "left", "on",
"v1", false }, { "left", "on", "v2", false }, { "inner", "on", "v2", true }, { "left", "on",
"v2", true } });
     }
 
-    public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine)
throws Exception {
+    public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine,
boolean excludeViewCubes) throws Exception {
 
-        printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " +
coprocessorToggle + ", query engine: " + queryEngine);
+        printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " +
coprocessorToggle + ", query engine: " + queryEngine + ", excludeViewCubes: " + excludeViewCubes);
 
         ITKylinQueryTest.clean();
 
@@ -85,6 +85,12 @@ public class ITCombinationTest extends ITKylinQueryTest {
             // unset
         }
 
+        RemoveBlackoutRealizationsRule.blackouts.clear();
+        if (excludeViewCubes) {
+            RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
+            RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
+        }
+
         if ("v1".equalsIgnoreCase(queryEngine))
             HBaseStorage.overwriteStorageQuery = HBaseStorage.v1CubeStorageQuery;
         else

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index dcacb06..4657a5c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -34,6 +34,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.enumerator.OLAPQuery;
 import org.apache.kylin.query.relnode.OLAPContext;
 import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.query.schema.OLAPSchemaFactory;
 import org.apache.kylin.storage.hbase.HBaseStorage;
 import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
@@ -46,7 +47,6 @@ import org.junit.Test;
 
 import com.google.common.collect.Maps;
 
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
     @BeforeClass
@@ -57,9 +57,12 @@ public class ITKylinQueryTest extends KylinTestBase {
         priorities.put(RealizationType.CUBE, 0);
         Candidate.setPriorities(priorities);
 
-        joinType = "left";
+        joinType = "inner";
 
         setupAll();
+        
+        RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
+        RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
     }
 
     @AfterClass
@@ -96,6 +99,7 @@ public class ITKylinQueryTest extends KylinTestBase {
 
         ObserverEnabler.forceCoprocessorUnset();
         HBaseMetadataTestCase.staticCleanupTestMetadata();
+        RemoveBlackoutRealizationsRule.blackouts.clear();
 
     }
 
@@ -147,6 +151,11 @@ public class ITKylinQueryTest extends KylinTestBase {
     }
 
     @Test
+    public void testLikeQuery() throws Exception {
+        execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_like", null,
true);
+    }
+
+    @Test
     public void testVerifyQuery() throws Exception {
         verifyResultRowCount(getQueryFolderPrefix() + "src/test/resources/query/sql_verifyCount");
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql b/kylin-it/src/test/resources/query/sql_like/query01.sql
new file mode 100644
index 0000000..e1f131d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query01.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name as lstg_format_name, count(*) as cnt 
+ 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id
= test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where lstg_format_name like '%BIN%'
+group by lstg_format_name
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query02.sql b/kylin-it/src/test/resources/query/sql_like/query02.sql
new file mode 100644
index 0000000..ef0b4a0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query02.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact
+where lower(lstg_format_name)='abin_a' and substring(lstg_format_name,1,3) in ('ABI') and
upper(lstg_format_name) > 'AAAA' and
+lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name)
> 3 and lstg_format_name||'a'='ABIN_Aa'
+group by lstg_format_name
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index 11498d6..53f871d 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -24,6 +24,7 @@ import java.util.List;
 import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.routing.rules.RealizationSortRule;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.query.routing.rules.RemoveUncapableRealizationsRule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -37,6 +38,7 @@ public abstract class RoutingRule {
     private static List<RoutingRule> rules = Lists.newLinkedList();
 
     static {
+        rules.add(new RemoveBlackoutRealizationsRule());
         rules.add(new RemoveUncapableRealizationsRule());
         rules.add(new RealizationSortRule());
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
new file mode 100644
index 0000000..9c3d7c9
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.routing.rules;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.RoutingRule;
+
+import com.google.common.collect.Sets;
+
+/**
+ * for IT use, exclude some cubes 
+ */
+public class RemoveBlackoutRealizationsRule extends RoutingRule {
+    public static Set<String> blackouts = Sets.newHashSet();
+
+    @Override
+    public void apply(List<Candidate> candidates) {
+        for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();)
{
+            Candidate candidate = iterator.next();
+
+            if (blackouts.contains(candidate.getRealization().getCanonicalName())) {
+                iterator.remove();
+            }
+        }
+    }
+
+}


Mime
View raw message