kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [06/26] kylin git commit: KYLIN-2030 enhancement
Date Sun, 30 Oct 2016 10:43:47 GMT
KYLIN-2030 enhancement


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46284bdc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46284bdc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46284bdc

Branch: refs/heads/KYLIN-1971
Commit: 46284bdca211d59f4a3274f5579dc8fe8a48cc6e
Parents: 1429da5
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Sat Oct 8 16:14:26 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800

----------------------------------------------------------------------
 .../kylin/cube/CubeCapabilityChecker.java       | 43 +++++++++++++++-----
 .../org/apache/kylin/cube/model/CubeDesc.java   |  6 +--
 .../kylin/metadata/model/FunctionDesc.java      |  1 +
 .../test/resources/query/sql_lookup/query06.sql |  1 +
 .../test/resources/query/sql_lookup/query07.sql |  1 +
 5 files changed, 39 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c8460f0..ee21b1c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,18 +25,22 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,20 +70,39 @@ public class CubeCapabilityChecker {
         Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions,
cube);
 
         // try custom measure types
+        // in RAW query, unmatchedDimensions and unmatchedAggregations will null, so can't
chose RAW cube well!
+        //        if (!unmatchedDimensions.isEmpty() || !unmatchedAggregations.isEmpty())
{
         tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result);
+        //        }
 
-        // try dimension-as-measure
-        if (!unmatchedAggregations.isEmpty()) {
-            if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+        //more tricks
+        if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+            //for query-on-facttable
+            //1. dimension as measure
+
+            if (!unmatchedAggregations.isEmpty()) {
                 tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
-            } else {
-                //deal with query on lookup table, like https://issues.apache.org/jira/browse/KYLIN-2030
-                if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable))
{
-                    Set<TblColRef> dimCols = Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
+            }
+        } else {
+            //for non query-on-facttable 
+            if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
+                TableDesc tableDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
+                Set<TblColRef> dimCols = Sets.newHashSet();
+                for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+                    dimCols.add(columnDesc.getRef());
+                }
+
+                //1. dimension as measure, like max(cal_dt) or count( distinct col) from
lookup
+                if (!unmatchedAggregations.isEmpty()) {
                     tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, dimCols);
-                } else {
-                    logger.info("Skip tryDimensionAsMeasures because current cube {} does
not touch lookup table {} at all", cube.getName(), digest.factTable);
                 }
+
+                //2. more "dimensions" contributed by snapshot
+                if (!unmatchedDimensions.isEmpty()) {
+                    unmatchedDimensions.removeAll(dimCols);
+                }
+            } else {
+                logger.info("cube {} does not touch lookup table {} at all", cube.getName(),
digest.factTable);
             }
         }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0c0b4f3..159042d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -33,9 +33,9 @@ import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.Map.Entry;
 
 import javax.annotation.Nullable;
 
@@ -67,9 +67,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.base.Function;
 import com.google.common.collect.Collections2;
@@ -674,7 +674,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
     private boolean hasSingle(ArrayList<Set<String>> dimsList) {
         boolean hasSingle = false;
         for (Set<String> dims : dimsList) {
-            if (dims.size() < 2)
+            if (dims.size() == 1)
                 hasSingle = true;
         }
         return hasSingle;

http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index e7d5186..ae7f805 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -102,6 +102,7 @@ public class FunctionDesc {
     }
 
     public MeasureType<?> getMeasureType() {
+        //like max(cal_dt)
         if (isDimensionAsMetric && !isCountDistinct()) {
             return null;
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query06.sql b/kylin-it/src/test/resources/query/sql_lookup/query06.sql
new file mode 100644
index 0000000..e2b8469
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query06.sql
@@ -0,0 +1 @@
+select max(cal_dt) as x from edw.test_cal_dt
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query07.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query07.sql b/kylin-it/src/test/resources/query/sql_lookup/query07.sql
new file mode 100644
index 0000000..6be3439
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query07.sql
@@ -0,0 +1 @@
+select site_id,count(*) as y,count(DISTINCT site_name) as x from edw.test_sites group by
site_id


Mime
View raw message