kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject incubator-kylin git commit: KYLIN-660 Make configurable of dictionary cardinality cap
Date Thu, 11 Jun 2015 08:05:19 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.6 a4692dba6 -> 498b98450


KYLIN-660 Make configurable of dictionary cardinality cap


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/498b9845
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/498b9845
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/498b9845

Branch: refs/heads/0.6
Commit: 498b9845093af7bd9629a1e622bccd41f8400aaf
Parents: a4692db
Author: Li, Yang <yangli9@ebay.com>
Authored: Thu Jun 11 16:04:54 2015 +0800
Committer: Li, Yang <yangli9@ebay.com>
Committed: Thu Jun 11 16:04:54 2015 +0800

----------------------------------------------------------------------
 .../java/com/kylinolap/common/KylinConfig.java  | 24 +++++++-------------
 .../com/kylinolap/dict/DictionaryGenerator.java |  3 ++-
 2 files changed, 10 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/498b9845/common/src/main/java/com/kylinolap/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/KylinConfig.java b/common/src/main/java/com/kylinolap/common/KylinConfig.java
index 2112fe8..9dd64b4 100644
--- a/common/src/main/java/com/kylinolap/common/KylinConfig.java
+++ b/common/src/main/java/com/kylinolap/common/KylinConfig.java
@@ -392,16 +392,10 @@ public class KylinConfig {
         return Integer.parseInt(getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS,
"60"));
     }
 
-    /**
-     * @return
-     */
     public int getMaxConcurrentJobLimit() {
         return Integer.parseInt(getOptional(KYLIN_JOB_CONCURRENT_MAX_LIMIT, "10"));
     }
 
-    /**
-     * @return
-     */
     public String getTimeZone() {
         return getOptional(KYLIN_REST_TIMEZONE, "PST");
     }
@@ -413,20 +407,10 @@ public class KylinConfig {
         return nodes.split("\\s*,\\s*");
     }
 
-    /**
-     * @return
-     */
     public String getAdminDls() {
         return getOptional("kylin.job.admin.dls", null);
     }
 
-    public int getScanThreshold() {
-        return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
-    }
-
-    /**
-     * @return
-     */
     public long getJobStepTimeout() {
         return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60
* 60)));
     }
@@ -434,6 +418,14 @@ public class KylinConfig {
     public String getServerMode() {
         return this.getOptional("kylin.server.mode", "all");
     }
+    
+    public int getDictionaryMaxCardinality() {
+        return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
+    }
+
+    public int getScanThreshold() {
+        return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
+    }
 
     public Long getQueryDurationCacheThreshold() {
         return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/498b9845/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
index f474035..e1c2b07 100644
--- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
+import com.kylinolap.common.KylinConfig;
 import com.kylinolap.common.util.JsonUtil;
 import com.kylinolap.dict.lookup.ReadableTable;
 import com.kylinolap.dict.lookup.TableReader;
@@ -40,7 +41,7 @@ import com.kylinolap.metadata.model.schema.DataType;
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class DictionaryGenerator {
 
-    private static final int DICT_MAX_CARDINALITY = 2000000; // 2 million
+    private static final int DICT_MAX_CARDINALITY = KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
 
     private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
 


Mime
View raw message