kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject incubator-kylin git commit: KYLIN-1068 small changes
Date Thu, 15 Oct 2015 07:26:50 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/KYLIN-1068 3b2a12faa -> 58dd59130


KYLIN-1068 small changes

Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/58dd5913
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/58dd5913
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/58dd5913

Branch: refs/heads/KYLIN-1068
Commit: 58dd59130206d1d47f1cf650a0b491cc328a930e
Parents: 3b2a12f
Author: shaofengshi <shaofengshi@apache.org>
Authored: Thu Oct 15 15:26:24 2015 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Thu Oct 15 15:26:24 2015 +0800

----------------------------------------------------------------------
 .../main/java/org/apache/kylin/common/topn/TopNCounter.java   | 7 +------
 examples/test_case_data/sandbox/kylin.properties              | 7 ++-----
 .../kylin/storage/hbase/steps/HBaseMROutput2Transition.java   | 5 +++--
 3 files changed, 6 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58dd5913/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
index 6093a43..593058b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
@@ -102,7 +102,6 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>>
{
                 droppedItem = counter.item;
                 counterMap.remove(droppedItem);
                 counter.item = item;
-//                counter.error = counter.count;
                 counter.count = 0.0;
             }
             counterMap.put(item, counterNode);
@@ -217,27 +216,23 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>>
{
         if (another.size() >= another.capacity) {
             m2 = another.counterList.tail().getValue().count;
         }
-
+        
         for (Map.Entry<T, ListNode2<Counter<T>>> entry : this.counterMap.entrySet())
{
             T item = entry.getKey();
             ListNode2<Counter<T>> existing = another.counterMap.get(item);
             if (existing != null) {
                 this.offer(item, another.counterMap.get(item).getValue().count);
-//                entry.getValue().getValue().error += another.counterMap.get(item).getValue().error;
 
                 another.counterMap.remove(item);
             } else {
                 this.offer(item, m2);
-//                this.counterMap.get(item).getValue().error = entry.getValue().getValue().error
+ m2;
             }
         }
 
         for (Map.Entry<T, ListNode2<Counter<T>>> entry : another.counterMap.entrySet())
{
             T item = entry.getKey();
             double counter = entry.getValue().getValue().count;
-//            double error = entry.getValue().getValue().error;
             this.offer(item, counter + m1);
-//            this.counterMap.get(item).getValue().error = error + m1;
         }
 
         return this;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58dd5913/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index c13fcfb..35e2927 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -36,11 +36,8 @@ kylin.job.concurrent.max.limit=10
 # Max reducer number
 kylin.job.mapreduce.max.reducer.number=5
 
-# Whether calculate cube in mem in each mapper;
-kylin.job.cubing.inMem=true
-
-#the percentage of the sampling, default 25%
-kylin.job.cubing.inMem.sampling.percent=25
+#the percentage of the sampling, default 100%
+kylin.job.cubing.inMem.sampling.percent=100
 
 # The cut size for hbase region, in GB.
 # E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58dd5913/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index 6ccb60b..b3bd36e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -366,8 +366,9 @@ public class HBaseMROutput2Transition implements IMROutput2 {
             Configuration conf = HBaseConfiguration.create(job.getConfiguration());
             HTable htable = new HTable(conf, htableName);
             
-            int regions = htable.getStartKeys().length + 1;
-            int reducerNum = regions * 10;
+            int regions = htable.getStartKeys().length;
+            int reducerNum = regions * 3;
+            reducerNum = Math.max(1, reducerNum);
             KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
             reducerNum = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), reducerNum);
 


Mime
View raw message