kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1245 bug fix and better log
Date Fri, 25 Dec 2015 10:15:47 GMT
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 07b1c6aaa -> ed63365fc


KYLIN-1245 bug fix and better log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed63365f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed63365f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed63365f

Branch: refs/heads/2.0-rc
Commit: ed63365fc3ca2c9745fa51126fc10cf7a2477823
Parents: 07b1c6a
Author: Li, Yang <yangli9@ebay.com>
Authored: Fri Dec 25 18:12:38 2015 +0800
Committer: Li, Yang <yangli9@ebay.com>
Committed: Fri Dec 25 18:14:56 2015 +0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/kylin/cube/CubeSegment.java    | 9 +++------
 .../java/org/apache/kylin/engine/mr/steps/CuboidJob.java    | 2 +-
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java   | 2 +-
 .../org/apache/kylin/engine/mr/steps/InMemCuboidJob.java    | 2 +-
 .../apache/kylin/engine/mr/steps/SaveStatisticsStep.java    | 4 ++--
 5 files changed, 8 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/ed63365f/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 62df1e9..8d14a6f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -31,20 +31,17 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
 import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.IRealizationSegment;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
 import com.google.common.collect.Maps;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationSegment;
 
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility
= Visibility.NONE, setterVisibility = Visibility.NONE)
 public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, IRealizationSegment
{
@@ -342,7 +339,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware,
I
 
     @Override
     public String toString() {
-        return Objects.toStringHelper(this).add("cube", cubeInstance.getName()).add("name",
name).add("status", status).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("last_build_job_id",
lastBuildJobID).toString();
+        return cubeInstance.getName() + "[" + name + "]";
     }
 
     public void setDictionaries(ConcurrentHashMap<String, String> dictionaries) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed63365f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 1647480..ab5693b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -105,7 +105,7 @@ public class CuboidJob extends AbstractHadoopJob {
             CubeInstance cube = cubeMgr.getCube(cubeName);
             
             if (checkSkip(cubingJobId)) {
-                logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + segmentName);
+                logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeName
+ "[" + segmentName + "]");
                 return 0;
             }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed63365f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 0416c3a..17ed7d6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -143,7 +143,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable,
Text,
             for (HyperLogLogPlusCounter hll : cuboidHLLMap.values()) {
                 grandTotal += hll.getCountEstimate();
             }
-            double mapperOverlapRatio = (double) totalRowsBeforeMerge / grandTotal;
+            double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge
/ grandTotal;
             
             writeMapperAndCuboidStatistics(context); // for human check
             CuboidStatsUtil.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput),
//

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed63365f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index 6fec03c..d197252 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -91,7 +91,7 @@ public class InMemCuboidJob extends AbstractHadoopJob {
             String cubingJobId = getOptionValue(OPTION_CUBING_JOB_ID);
 
             if (checkSkip(cubingJobId)) {
-                logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeSeg.getName());
+                logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeSeg);
                 return 0;
             }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed63365f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index b12063f..396955b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -100,11 +100,11 @@ public class SaveStatisticsStep extends AbstractExecutable {
         } else { // the default
             double threshold = kylinConf.getCubeAlgorithmAutoThreshold();
             double mapperOverlapRatio = new CubeStatsReader(seg, kylinConf).getMapperOverlapRatioOfFirstBuild();
-            logger.info("mapperOverlapRatio for " + seg.getName() + " is " + mapperOverlapRatio
+ " and threshold is " + threshold);
+            logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + "
and threshold is " + threshold);
             alg = mapperOverlapRatio < threshold ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
         }
 
-        logger.info("The cube algorithm for " + seg.getName() + " is " + alg);
+        logger.info("The cube algorithm for " + seg + " is " + alg);
 
         CubingJob cubingJob = (CubingJob) executableManager.getJob(getCubingJobId());
         cubingJob.setAlgorithm(alg);


Mime
View raw message