kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject incubator-kylin git commit: KYLIN-943 add topn measure to existing test cubes
Date Tue, 22 Sep 2015 09:24:19 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/KYLIN-943 ba8fcd851 -> 167386858


KYLIN-943 add topn measure to existing test cubes

Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/16738685
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/16738685
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/16738685

Branch: refs/heads/KYLIN-943
Commit: 167386858fa251222888c5bc5d3b1eb5a2ff232d
Parents: ba8fcd8
Author: shaofengshi <shaofengshi@apache.org>
Authored: Tue Sep 22 17:24:05 2015 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Tue Sep 22 17:24:05 2015 +0800

----------------------------------------------------------------------
 .../kylin/job/BuildCubeWithEngineTest.java      | 15 +-----------
 .../kylin/cube/CubeCapabilityChecker.java       |  2 +-
 .../org/apache/kylin/cube/CubeInstance.java     |  6 +++++
 .../kylin/gridtable/GTAggregateScanner.java     |  3 ++-
 .../test_kylin_cube_without_slr_desc.json       | 25 ++++++++++++++++++++
 ...t_kylin_cube_without_slr_left_join_desc.json | 25 ++++++++++++++++++++
 .../localmeta/project/default.json              |  5 ----
 examples/test_case_data/sandbox/mapred-site.xml | 18 +++++++-------
 query/src/test/resources/query/sql/query81.sql  |  6 ++++-
 query/src/test/resources/query/sql/query82.sql  | 13 ++++++----
 10 files changed, 83 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 4c17630..5aa108d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -133,7 +133,7 @@ public class BuildCubeWithEngineTest {
     }
 
     private void testLeft() throws Exception {
-        String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2", "testLeftJoinTopNCube"
};
+        String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2" };
         runTestAndAssertSucceed(testCase);
     }
 
@@ -186,19 +186,6 @@ public class BuildCubeWithEngineTest {
         }
     }
 
-
-    @SuppressWarnings("unused")
-    // called by reflection
-    private List<String> testLeftJoinTopNCube() throws Exception {
-        SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
-        f.setTimeZone(TimeZone.getTimeZone("GMT"));
-        long date1 = 0;
-        long date2 = f.parse("2050-01-01").getTime();
-        List<String> result = Lists.newArrayList();
-        result.add(buildSegment("test_kylin_cube_topn", date1, date2));
-        return result;
-    }
-
     @SuppressWarnings("unused")
     // called by reflection
     private List<String> testInnerJoinCube2() throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 628340e..df0d9c6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -79,7 +79,7 @@ public class CubeCapabilityChecker {
         return true;
     }
 
-    private static boolean isMatchedWithTopN(Collection<TblColRef> dimensionColumns,
CubeInstance cube, SQLDigest digest) {
+    public static boolean isMatchedWithTopN(Collection<TblColRef> dimensionColumns,
CubeInstance cube, SQLDigest digest) {
 
         CubeDesc cubeDesc = cube.getDescriptor();
         List<FunctionDesc> cubeFunctions = cubeDesc.listAllFunctions();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 4bfdb18..0811936 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -363,6 +363,12 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
                 calculatedCost += COST_WEIGHT_INNER_JOIN;
             }
         }
+        
+        if (CubeCapabilityChecker.isMatchedWithTopN(CubeDimensionDeriver.getDimensionColumns(digest),
this, digest)) {
+            // this is topN query
+            calculatedCost = calculatedCost / 3;
+        }
+
         return calculatedCost;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 6950e98..9050c49 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -12,6 +12,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.metadata.measure.HLLCAggregator;
 import org.apache.kylin.metadata.measure.LDCAggregator;
 import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.metadata.measure.TopNAggregator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -166,7 +167,7 @@ public class GTAggregateScanner implements IGTScanner {
 
             // skip expensive aggregation
             for (int i = 0; i < totalSum.length; i++) {
-                if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator)
+                if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator
|| totalSum[i] instanceof TopNAggregator )
                     totalSum[i] = null;
             }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 76675f5..0069d42 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -162,6 +162,20 @@
         "returntype": "hllc(10)"
       },
       "dependent_measure_ref": null
+    },
+    {
+      "id": 8,
+      "name": "TOP_SELLER",
+      "function": {
+        "expression": "TOP_N",
+        "parameter": {
+          "type": "column",
+          "value": "PRICE",
+          "displaycolumn": "seller_id"
+        },
+        "returntype": "topn(100)"
+      },
+      "dependent_measure_ref": null
     }
   ],
   "rowkey": {
@@ -261,6 +275,17 @@
             ]
           }
         ]
+      },  
+      {
+        "name": "f3",
+        "columns": [
+          {
+            "qualifier": "m",
+            "measure_refs": [
+              "top_seller"
+            ]
+          }
+        ]
       }
     ]
   },

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index d3be1c9..4cc9f5f 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -162,6 +162,20 @@
         "returntype": "hllc(10)"
       },
       "dependent_measure_ref": null
+    },
+    {
+      "id": 8,
+      "name": "TOP_SELLER",
+      "function": {
+        "expression": "TOP_N",
+        "parameter": {
+          "type": "column",
+          "value": "PRICE",
+          "displaycolumn": "seller_id"
+        },
+        "returntype": "topn(100)"
+      },
+      "dependent_measure_ref": null
     }
   ],
   "rowkey": {
@@ -261,6 +275,17 @@
             ]
           }
         ]
+      },  
+      {
+        "name": "f3",
+        "columns": [
+          {
+            "qualifier": "m",
+            "measure_refs": [
+              "top_seller"
+            ]
+          }
+        ]
       }
     ]
   },

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 71eb1fa..e4118ca 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -23,11 +23,6 @@
       "realization": "test_kylin_cube_without_slr_left_join_empty"
     },
     {
-      "name": "test_kylin_cube_topn",
-      "type": "CUBE",
-      "realization": "test_kylin_cube_topn"
-    },
-    {
       "name": "test_kylin_ii_left_join",
       "type": "INVERTED_INDEX",
       "realization": "test_kylin_ii_left_join"

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 611bf8b..cba5aaa 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -3,7 +3,7 @@
 
     <property>
         <name>io.sort.mb</name>
-        <value>64</value>
+        <value>128</value>
     </property>
 
     <property>
@@ -13,12 +13,12 @@
 
     <property>
         <name>mapred.job.map.memory.mb</name>
-        <value>250</value>
+        <value>512</value>
     </property>
 
     <property>
         <name>mapred.job.reduce.memory.mb</name>
-        <value>250</value>
+        <value>512</value>
     </property>
 
     <property>
@@ -98,7 +98,7 @@
 
     <property>
         <name>mapreduce.map.java.opts</name>
-        <value>-Xmx200m</value>
+        <value>-Xmx512m</value>
     </property>
 
     <property>
@@ -108,7 +108,7 @@
 
     <property>
         <name>mapreduce.map.memory.mb</name>
-        <value>250</value>
+        <value>512</value>
     </property>
 
     <property>
@@ -153,7 +153,7 @@
 
     <property>
         <name>mapreduce.reduce.memory.mb</name>
-        <value>250</value>
+        <value>512</value>
     </property>
 
     <property>
@@ -203,7 +203,7 @@
 
     <property>
         <name>mapreduce.task.io.sort.mb</name>
-        <value>64</value>
+        <value>128</value>
     </property>
 
     <property>
@@ -218,7 +218,7 @@
 
     <property>
         <name>yarn.app.mapreduce.am.command-opts</name>
-        <value>-Xmx200m</value>
+        <value>-Xmx512m</value>
     </property>
 
     <property>
@@ -228,7 +228,7 @@
 
     <property>
         <name>yarn.app.mapreduce.am.resource.mb</name>
-        <value>250</value>
+        <value>512</value>
     </property>
 
     <property>

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/query/src/test/resources/query/sql/query81.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query81.sql b/query/src/test/resources/query/sql/query81.sql
index 39a14a5..d7cc036 100644
--- a/query/src/test/resources/query/sql/query81.sql
+++ b/query/src/test/resources/query/sql/query81.sql
@@ -18,8 +18,12 @@
 
 SELECT 
  seller_id 
- FROM test_kylin_fact left join edw.test_cal_dt as test_cal_dt
+  FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
  ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id 
  where test_kylin_fact.cal_dt < DATE '2013-02-01' 
  group by 
  test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 100

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/16738685/query/src/test/resources/query/sql/query82.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query82.sql b/query/src/test/resources/query/sql/query82.sql
index 29d67b8..82c64d6 100644
--- a/query/src/test/resources/query/sql/query82.sql
+++ b/query/src/test/resources/query/sql/query82.sql
@@ -17,9 +17,14 @@
 --
 
 SELECT 
- test_kylin_fact.cal_dt, seller_id 
- FROM test_kylin_fact 
-left JOIN edw.test_cal_dt as test_cal_dt
+ cal_dt, seller_id from (select
+  test_kylin_fact.cal_dt, seller_id, sum(test_kylin_fact.price) as gmv
+  FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
  ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id 
  group by 
- test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc
limit 100
+ test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by gmv desc limit 100
+ ) as abc


Mime
View raw message