kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [19/48] kylin git commit: KYLIN-2851, inspect available memory during MR job
Date Fri, 15 Sep 2017 14:42:39 GMT
KYLIN-2851, inspect available memory during MR job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55c6ee78
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55c6ee78
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55c6ee78

Branch: refs/heads/ranger
Commit: 55c6ee780a5cabc1618fc5f078c776f73123059b
Parents: 9230663
Author: Cheng Wang <cheng.wang@kyligence.io>
Authored: Thu Sep 7 16:08:26 2017 +0800
Committer: 成 <cheng.wang@kyligence.io>
Committed: Thu Sep 7 17:27:50 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/engine/mr/KylinMapper.java | 42 ++++++++++++++++++--
 .../apache/kylin/engine/mr/KylinReducer.java    | 42 ++++++++++++++++++--
 .../engine/mr/steps/BaseCuboidMapperBase.java   |  2 +-
 .../kylin/engine/mr/steps/CuboidReducer.java    |  2 +-
 .../mr/steps/FactDistinctColumnsCombiner.java   |  2 +-
 .../mr/steps/FactDistinctColumnsMapper.java     |  4 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |  2 +-
 .../mr/steps/FactDistinctColumnsReducer.java    |  2 +-
 .../engine/mr/steps/HiveToBaseCuboidMapper.java |  4 +-
 .../engine/mr/steps/InMemCuboidMapper.java      |  2 +-
 .../engine/mr/steps/InMemCuboidReducer.java     |  2 +-
 .../engine/mr/steps/MergeCuboidMapper.java      |  2 +-
 .../kylin/engine/mr/steps/NDCuboidMapper.java   |  2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |  2 +-
 .../steps/RowKeyDistributionCheckerReducer.java |  2 +-
 .../HiveToBaseCuboidMapperPerformanceTest.java  |  2 +-
 .../cardinality/ColumnCardinalityMapper.java    |  2 +-
 .../cardinality/ColumnCardinalityReducer.java   |  2 +-
 .../kafka/hadoop/KafkaFlatTableMapper.java      |  2 +-
 .../storage/hbase/steps/CubeHFileMapper.java    |  2 +-
 .../hbase/steps/RangeKeyDistributionMapper.java |  2 +-
 .../steps/RangeKeyDistributionReducer.java      |  2 +-
 .../hbase/steps/CubeHFileMapper2Test.java       |  2 +-
 23 files changed, 99 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index 1595bdd..81ca5d5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -23,6 +23,7 @@ import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,10 +41,38 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Mapper<KEYIN,
     }
 
     @Override
-    final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context) throws IOException, InterruptedException {
+    final protected void setup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+            throws IOException, InterruptedException {
+        try {
+            logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
+            doSetup(context);
+        } catch (IOException ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        } catch (InterruptedException ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        } catch (RuntimeException ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        } catch (Error ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        }
+    }
+
+    protected void doSetup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+            throws IOException, InterruptedException {
+        // NOTHING
+    }
+
+    @Override
+    final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context)
+            throws IOException, InterruptedException {
         try {
             if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
                 logger.info("Accepting Mapper Key with ordinal: " + mapCounter);
+                logger.info("Do map, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
             }
             doMap(key, value, context);
         } catch (IOException ex) { // KYLIN-2170
@@ -61,14 +90,18 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Mapper<KEYIN,
         }
     }
 
-    protected void doMap(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context) throws IOException, InterruptedException {
+    protected void doMap(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context)
+            throws IOException, InterruptedException {
         super.map(key, value, context);
     }
 
     @Override
-    final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
throws IOException, InterruptedException {
+    final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+            throws IOException, InterruptedException {
         try {
+            logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
             doCleanup(context);
+            logger.info("Total rows: {}", mapCounter);
         } catch (IOException ex) { // KYLIN-2170
             logger.error("", ex);
             throw ex;
@@ -84,6 +117,7 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Mapper<KEYIN,
         }
     }
 
-    protected void doCleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
throws IOException, InterruptedException {
+    protected void doCleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+            throws IOException, InterruptedException {
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index e47af9c..28c62db 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -23,6 +23,7 @@ import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,10 +40,38 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Reducer<KEYI
     }
 
     @Override
-    final public void reduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN,
VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
+    final protected void setup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+            throws IOException, InterruptedException {
+        try {
+            logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
+            doSetup(context);
+        } catch (IOException ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        } catch (InterruptedException ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        } catch (RuntimeException ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        } catch (Error ex) { // KYLIN-2170
+            logger.error("", ex);
+            throw ex;
+        }
+    }
+
+    protected void doSetup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+            throws IOException, InterruptedException {
+        // NOTHING
+    }
+
+    @Override
+    final public void reduce(KEYIN key, Iterable<VALUEIN> values,
+            Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException,
InterruptedException {
         try {
             if (reduceCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
                 logger.info("Accepting Reducer Key with ordinal: " + reduceCounter);
+                logger.info("Do reduce, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
             }
 
             doReduce(key, values, context);
@@ -61,14 +90,18 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Reducer<KEYI
         }
     }
 
-    protected void doReduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN,
VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
+    protected void doReduce(KEYIN key, Iterable<VALUEIN> values,
+            Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException,
InterruptedException {
         super.reduce(key, values, context);
     }
 
     @Override
-    final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context) throws IOException, InterruptedException {
+    final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context)
+            throws IOException, InterruptedException {
         try {
+            logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
             doCleanup(context);
+            logger.info("Total rows: " + reduceCounter);
         } catch (IOException ex) { // KYLIN-2170
             logger.error("", ex);
             throw ex;
@@ -84,6 +117,7 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Reducer<KEYI
         }
     }
 
-    protected void doCleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
throws IOException, InterruptedException {
+    protected void doCleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+            throws IOException, InterruptedException {
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 93e413b..1ae6cd0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -58,7 +58,7 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends
KylinMapper<K
     private BaseCuboidBuilder baseCuboidBuilder;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
         cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
         segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index 495be77..2dd9f00 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -60,7 +60,7 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text>
{
     private Text outputValue = new Text();
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
         cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
index a367bc6..8910024 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
@@ -29,7 +29,7 @@ import org.apache.kylin.engine.mr.KylinReducer;
 public class FactDistinctColumnsCombiner extends KylinReducer<SelfDefineSortableKey, Text,
SelfDefineSortableKey, Text> {
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 8281759..6e1a5dc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -82,8 +82,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
     private boolean isUsePutRowKeyToHllNewAlgorithm;
 
     @Override
-    protected void setup(Context context) throws IOException {
-        super.setup(context);
+    protected void doSetup(Context context) throws IOException {
+        super.doSetup(context);
         tmpbuf = ByteBuffer.allocate(4096);
         collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
         if (collectStatistics) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 458af69..9952def 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -66,7 +66,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN>
extends Kyli
     protected Map<Integer, Integer> columnIndexToReducerBeginId = new HashMap<>();
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         Configuration conf = context.getConfiguration();
         bindCurrentConfiguration(conf);
         KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 7f01c3a..20fb9e2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -88,7 +88,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
     private MultipleOutputs mos;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
         Configuration conf = context.getConfiguration();
         mos = new MultipleOutputs(context);

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
index a04fb43..a5f267f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
@@ -32,8 +32,8 @@ public class HiveToBaseCuboidMapper<KEYIN> extends BaseCuboidMapperBase<KEYIN,
O
     private IMRTableInputFormat flatTableInputFormat;
 
     @Override
-    protected void setup(Context context) throws IOException {
-        super.setup(context);
+    protected void doSetup(Context context) throws IOException {
+        super.doSetup(context);
         flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSegment).getFlatTableInputFormat();
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index eee189c..f8ac7d2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -69,7 +69,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN,
Object, ByteArr
     private Future<?> future;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
 
         Configuration conf = context.getConfiguration();

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index 244889f..5aa2c69 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -55,7 +55,7 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable,
ByteArra
     private Text outputValue;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
         KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index a01a928..c2ef2d2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -91,7 +91,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text,
Text> {
     private Text outputValue;
 
     @Override
-    protected void setup(Context context) throws IOException, InterruptedException {
+    protected void doSetup(Context context) throws IOException, InterruptedException {
         super.bindCurrentConfiguration(context.getConfiguration());
 
         cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 782ce72..e1a68d3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -62,7 +62,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text>
{
     private NDCuboidBuilder ndCuboidBuilder;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
 
         cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
index eab57d1..d428dcb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
@@ -48,7 +48,7 @@ public class RowKeyDistributionCheckerMapper extends KylinMapper<Text,
Text, Tex
     List<Text> keyList;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
 
         rowKeyStatsFilePath = context.getConfiguration().get("rowKeyStatsFilePath");

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
index d203e8c..af4a7df 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
@@ -33,7 +33,7 @@ public class RowKeyDistributionCheckerReducer extends KylinReducer<Text,
LongWri
     LongWritable outputKey = new LongWritable(0L);
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
index 9e03493..000de9b 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
@@ -50,7 +50,7 @@ public class HiveToBaseCuboidMapperPerformanceTest {
         HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
         Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
 
-        mapper.setup(context);
+        mapper.doSetup(context);
 
         Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
         Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index 09602ec..1cadec4 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -56,7 +56,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T,
Object, IntWritab
     private IMRTableInputFormat tableInputFormat;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         Configuration conf = context.getConfiguration();
         bindCurrentConfiguration(conf);
         KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
index 0648960..53a2294 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
@@ -44,7 +44,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable,
BytesWri
     private Map<Integer, HLLCounter> hllcMap = new HashMap<Integer, HLLCounter>();
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
index e94c30e..39df3fc 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
@@ -33,7 +33,7 @@ public class KafkaFlatTableMapper extends KylinMapper<LongWritable, BytesWritabl
     private Text outValue = new Text();
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         Configuration conf = context.getConfiguration();
         bindCurrentConfiguration(conf);
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
index 371a83b..f3d9f69 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
@@ -53,7 +53,7 @@ public class CubeHFileMapper extends KylinMapper<Text, Text, ImmutableBytesWrita
     List<KeyValueCreator> keyValueCreators;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
         cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
index c66ccb3..30585df 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
@@ -39,7 +39,7 @@ public class RangeKeyDistributionMapper extends KylinMapper<Text, Text,
Text, Lo
     private Text lastKey;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
index 63433dd..3899b4c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
@@ -54,7 +54,7 @@ public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable
     private String output = null;
 
     @Override
-    protected void setup(Context context) throws IOException {
+    protected void doSetup(Context context) throws IOException {
         super.bindCurrentConfiguration(context.getConfiguration());
 
         if (context.getConfiguration().get(BatchConstants.CFG_OUTPUT_PATH) != null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
index d47f393..b43e4a8 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
@@ -73,7 +73,7 @@ public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
         Context context = MockupMapContext.create(hconf, cubeName, outKV);
 
         CubeHFileMapper mapper = new CubeHFileMapper();
-        mapper.setup(context);
+        mapper.doSetup(context);
 
         Text key = new Text("not important");
         Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });


Mime
View raw message