KYLIN-2851, inspect available memory during MR job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55c6ee78
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55c6ee78
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55c6ee78
Branch: refs/heads/ranger
Commit: 55c6ee780a5cabc1618fc5f078c776f73123059b
Parents: 9230663
Author: Cheng Wang <cheng.wang@kyligence.io>
Authored: Thu Sep 7 16:08:26 2017 +0800
Committer: 成 <cheng.wang@kyligence.io>
Committed: Thu Sep 7 17:27:50 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/KylinMapper.java | 42 ++++++++++++++++++--
.../apache/kylin/engine/mr/KylinReducer.java | 42 ++++++++++++++++++--
.../engine/mr/steps/BaseCuboidMapperBase.java | 2 +-
.../kylin/engine/mr/steps/CuboidReducer.java | 2 +-
.../mr/steps/FactDistinctColumnsCombiner.java | 2 +-
.../mr/steps/FactDistinctColumnsMapper.java | 4 +-
.../mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
.../mr/steps/FactDistinctColumnsReducer.java | 2 +-
.../engine/mr/steps/HiveToBaseCuboidMapper.java | 4 +-
.../engine/mr/steps/InMemCuboidMapper.java | 2 +-
.../engine/mr/steps/InMemCuboidReducer.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../kylin/engine/mr/steps/NDCuboidMapper.java | 2 +-
.../steps/RowKeyDistributionCheckerMapper.java | 2 +-
.../steps/RowKeyDistributionCheckerReducer.java | 2 +-
.../HiveToBaseCuboidMapperPerformanceTest.java | 2 +-
.../cardinality/ColumnCardinalityMapper.java | 2 +-
.../cardinality/ColumnCardinalityReducer.java | 2 +-
.../kafka/hadoop/KafkaFlatTableMapper.java | 2 +-
.../storage/hbase/steps/CubeHFileMapper.java | 2 +-
.../hbase/steps/RangeKeyDistributionMapper.java | 2 +-
.../steps/RangeKeyDistributionReducer.java | 2 +-
.../hbase/steps/CubeHFileMapper2Test.java | 2 +-
23 files changed, 99 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index 1595bdd..81ca5d5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,10 +41,38 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Mapper<KEYIN,
}
@Override
- final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context) throws IOException, InterruptedException {
+ final protected void setup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+ throws IOException, InterruptedException {
+ try {
+ logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
+ doSetup(context);
+ } catch (IOException ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ } catch (InterruptedException ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ } catch (RuntimeException ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ } catch (Error ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ }
+ }
+
+ protected void doSetup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+ throws IOException, InterruptedException {
+ // NOTHING
+ }
+
+ @Override
+ final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context)
+ throws IOException, InterruptedException {
try {
if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Accepting Mapper Key with ordinal: " + mapCounter);
+ logger.info("Do map, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
}
doMap(key, value, context);
} catch (IOException ex) { // KYLIN-2170
@@ -61,14 +90,18 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Mapper<KEYIN,
}
}
- protected void doMap(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context) throws IOException, InterruptedException {
+ protected void doMap(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context)
+ throws IOException, InterruptedException {
super.map(key, value, context);
}
@Override
- final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
throws IOException, InterruptedException {
+ final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+ throws IOException, InterruptedException {
try {
+ logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
doCleanup(context);
+ logger.info("Total rows: {}", mapCounter);
} catch (IOException ex) { // KYLIN-2170
logger.error("", ex);
throw ex;
@@ -84,6 +117,7 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Mapper<KEYIN,
}
}
- protected void doCleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
throws IOException, InterruptedException {
+ protected void doCleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+ throws IOException, InterruptedException {
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index e47af9c..28c62db 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,10 +40,38 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Reducer<KEYI
}
@Override
- final public void reduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN,
VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
+ final protected void setup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+ throws IOException, InterruptedException {
+ try {
+ logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
+ doSetup(context);
+ } catch (IOException ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ } catch (InterruptedException ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ } catch (RuntimeException ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ } catch (Error ex) { // KYLIN-2170
+ logger.error("", ex);
+ throw ex;
+ }
+ }
+
+ protected void doSetup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+ throws IOException, InterruptedException {
+ // NOTHING
+ }
+
+ @Override
+ final public void reduce(KEYIN key, Iterable<VALUEIN> values,
+ Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException,
InterruptedException {
try {
if (reduceCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Accepting Reducer Key with ordinal: " + reduceCounter);
+ logger.info("Do reduce, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
}
doReduce(key, values, context);
@@ -61,14 +90,18 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Reducer<KEYI
}
}
- protected void doReduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN,
VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
+ protected void doReduce(KEYIN key, Iterable<VALUEIN> values,
+ Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException,
InterruptedException {
super.reduce(key, values, context);
}
@Override
- final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context) throws IOException, InterruptedException {
+ final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context
context)
+ throws IOException, InterruptedException {
try {
+ logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB());
doCleanup(context);
+ logger.info("Total rows: " + reduceCounter);
} catch (IOException ex) { // KYLIN-2170
logger.error("", ex);
throw ex;
@@ -84,6 +117,7 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Reducer<KEYI
}
}
- protected void doCleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
throws IOException, InterruptedException {
+ protected void doCleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context)
+ throws IOException, InterruptedException {
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 93e413b..1ae6cd0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -58,7 +58,7 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends
KylinMapper<K
private BaseCuboidBuilder baseCuboidBuilder;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index 495be77..2dd9f00 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -60,7 +60,7 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text>
{
private Text outputValue = new Text();
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
index a367bc6..8910024 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
@@ -29,7 +29,7 @@ import org.apache.kylin.engine.mr.KylinReducer;
public class FactDistinctColumnsCombiner extends KylinReducer<SelfDefineSortableKey, Text,
SelfDefineSortableKey, Text> {
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 8281759..6e1a5dc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -82,8 +82,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
private boolean isUsePutRowKeyToHllNewAlgorithm;
@Override
- protected void setup(Context context) throws IOException {
- super.setup(context);
+ protected void doSetup(Context context) throws IOException {
+ super.doSetup(context);
tmpbuf = ByteBuffer.allocate(4096);
collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
if (collectStatistics) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 458af69..9952def 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -66,7 +66,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN>
extends Kyli
protected Map<Integer, Integer> columnIndexToReducerBeginId = new HashMap<>();
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
bindCurrentConfiguration(conf);
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 7f01c3a..20fb9e2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -88,7 +88,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private MultipleOutputs mos;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
Configuration conf = context.getConfiguration();
mos = new MultipleOutputs(context);
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
index a04fb43..a5f267f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
@@ -32,8 +32,8 @@ public class HiveToBaseCuboidMapper<KEYIN> extends BaseCuboidMapperBase<KEYIN,
O
private IMRTableInputFormat flatTableInputFormat;
@Override
- protected void setup(Context context) throws IOException {
- super.setup(context);
+ protected void doSetup(Context context) throws IOException {
+ super.doSetup(context);
flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSegment).getFlatTableInputFormat();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index eee189c..f8ac7d2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -69,7 +69,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN,
Object, ByteArr
private Future<?> future;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
Configuration conf = context.getConfiguration();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index 244889f..5aa2c69 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -55,7 +55,7 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable,
ByteArra
private Text outputValue;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index a01a928..c2ef2d2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -91,7 +91,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text,
Text> {
private Text outputValue;
@Override
- protected void setup(Context context) throws IOException, InterruptedException {
+ protected void doSetup(Context context) throws IOException, InterruptedException {
super.bindCurrentConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 782ce72..e1a68d3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -62,7 +62,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text>
{
private NDCuboidBuilder ndCuboidBuilder;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
index eab57d1..d428dcb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
@@ -48,7 +48,7 @@ public class RowKeyDistributionCheckerMapper extends KylinMapper<Text,
Text, Tex
List<Text> keyList;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
rowKeyStatsFilePath = context.getConfiguration().get("rowKeyStatsFilePath");
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
index d203e8c..af4a7df 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java
@@ -33,7 +33,7 @@ public class RowKeyDistributionCheckerReducer extends KylinReducer<Text,
LongWri
LongWritable outputKey = new LongWritable(0L);
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
index 9e03493..000de9b 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
@@ -50,7 +50,7 @@ public class HiveToBaseCuboidMapperPerformanceTest {
HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
- mapper.setup(context);
+ mapper.doSetup(context);
Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index 09602ec..1cadec4 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -56,7 +56,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T,
Object, IntWritab
private IMRTableInputFormat tableInputFormat;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
bindCurrentConfiguration(conf);
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
index 0648960..53a2294 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
@@ -44,7 +44,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable,
BytesWri
private Map<Integer, HLLCounter> hllcMap = new HashMap<Integer, HLLCounter>();
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
index e94c30e..39df3fc 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java
@@ -33,7 +33,7 @@ public class KafkaFlatTableMapper extends KylinMapper<LongWritable, BytesWritabl
private Text outValue = new Text();
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
bindCurrentConfiguration(conf);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
index 371a83b..f3d9f69 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
@@ -53,7 +53,7 @@ public class CubeHFileMapper extends KylinMapper<Text, Text, ImmutableBytesWrita
List<KeyValueCreator> keyValueCreators;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
index c66ccb3..30585df 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
@@ -39,7 +39,7 @@ public class RangeKeyDistributionMapper extends KylinMapper<Text, Text,
Text, Lo
private Text lastKey;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
index 63433dd..3899b4c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
@@ -54,7 +54,7 @@ public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable
private String output = null;
@Override
- protected void setup(Context context) throws IOException {
+ protected void doSetup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
if (context.getConfiguration().get(BatchConstants.CFG_OUTPUT_PATH) != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
index d47f393..b43e4a8 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
@@ -73,7 +73,7 @@ public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
Context context = MockupMapContext.create(hconf, cubeName, outKV);
CubeHFileMapper mapper = new CubeHFileMapper();
- mapper.setup(context);
+ mapper.doSetup(context);
Text key = new Text("not important");
Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });
|