kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [1/8] kylin git commit: minor, trim build log [Forced Update!]
Date Fri, 25 Nov 2016 07:09:45 GMT
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 84a0726a5 -> 99197dafc (forced update)


minor, trim build log

Revert "KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep"

This reverts commit 3ab966b650debb83eb219c9ed8d357d141466776.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8530ebd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8530ebd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8530ebd6

Branch: refs/heads/KYLIN-2217-2
Commit: 8530ebd692b3362f7fb9eb07160c8c069d467fc4
Parents: 6afcb26
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Fri Nov 25 09:52:47 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Fri Nov 25 09:58:10 2016 +0800

----------------------------------------------------------------------
 .../kylin/engine/mr/common/CubeStatsReader.java    |  2 +-
 .../kylin/engine/mr/steps/SaveStatisticsStep.java  | 17 +++--------------
 2 files changed, 4 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c6839d6..c917cfb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -227,7 +227,7 @@ public class CubeStatsReader {
         return ret;
     }
 
-    public void print(PrintWriter out) {
+    private void print(PrintWriter out) {
         Map<Long, Long> cuboidRows = getCuboidRowEstimatesHLL();
         Map<Long, Double> cuboidSizes = getCuboidSizeMap();
         List<Long> cuboids = new ArrayList<Long>(cuboidRows.keySet());

http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 79346a5..020c62c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,8 +19,6 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
 import java.util.Random;
 
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -31,8 +29,8 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
 import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.engine.mr.common.CubeStatsReader;
 import org.apache.kylin.job.exception.ExecuteException;
@@ -86,16 +84,6 @@ public class SaveStatisticsStep extends AbstractExecutable {
 
     private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) throws IOException
{
         String algPref = kylinConf.getCubeAlgorithm();
-
-        CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
-        StringWriter sw = new StringWriter();
-        PrintWriter pw = new PrintWriter(sw);
-        cubeStats.print(pw);
-        pw.flush();
-        pw.close();
-        logger.info("Cube Stats Estimation for segment {} :", seg.toString());
-        logger.info(sw.toString());
-
         AlgorithmEnum alg;
         if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
             alg = AlgorithmEnum.INMEM;
@@ -115,13 +103,14 @@ public class SaveStatisticsStep extends AbstractExecutable {
             } else if ("random".equalsIgnoreCase(algPref)) { // for testing
                 alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
             } else { // the default
+                CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
                 int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
                 int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
                 double mapperOverlapRatio = cubeStats.getMapperOverlapRatioOfFirstBuild();
                 double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
                 logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold
is " + mapperNumLimit);
                 logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio
+ " and threshold is " + overlapThreshold);
-
+ 
                 // in-mem cubing is good when
                 // 1) the cluster has enough mapper slots to run in parallel
                 // 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR
has advantage


Mime
View raw message