KYLIN-2851, inspect available memory during MR job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55c6ee78 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55c6ee78 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55c6ee78 Branch: refs/heads/ranger Commit: 55c6ee780a5cabc1618fc5f078c776f73123059b Parents: 9230663 Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Thu Sep 7 16:08:26 2017 +0800 Committer: æ <cheng.w...@kyligence.io> Committed: Thu Sep 7 17:27:50 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/engine/mr/KylinMapper.java | 42 ++++++++++++++++++-- .../apache/kylin/engine/mr/KylinReducer.java | 42 ++++++++++++++++++-- .../engine/mr/steps/BaseCuboidMapperBase.java | 2 +- .../kylin/engine/mr/steps/CuboidReducer.java | 2 +- .../mr/steps/FactDistinctColumnsCombiner.java | 2 +- .../mr/steps/FactDistinctColumnsMapper.java | 4 +- .../mr/steps/FactDistinctColumnsMapperBase.java | 2 +- .../mr/steps/FactDistinctColumnsReducer.java | 2 +- .../engine/mr/steps/HiveToBaseCuboidMapper.java | 4 +- .../engine/mr/steps/InMemCuboidMapper.java | 2 +- .../engine/mr/steps/InMemCuboidReducer.java | 2 +- .../engine/mr/steps/MergeCuboidMapper.java | 2 +- .../kylin/engine/mr/steps/NDCuboidMapper.java | 2 +- .../steps/RowKeyDistributionCheckerMapper.java | 2 +- .../steps/RowKeyDistributionCheckerReducer.java | 2 +- .../HiveToBaseCuboidMapperPerformanceTest.java | 2 +- .../cardinality/ColumnCardinalityMapper.java | 2 +- .../cardinality/ColumnCardinalityReducer.java | 2 +- .../kafka/hadoop/KafkaFlatTableMapper.java | 2 +- .../storage/hbase/steps/CubeHFileMapper.java | 2 +- .../hbase/steps/RangeKeyDistributionMapper.java | 2 +- .../steps/RangeKeyDistributionReducer.java | 2 +- .../hbase/steps/CubeHFileMapper2Test.java | 2 +- 23 files changed, 99 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java index 1595bdd..81ca5d5 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java @@ -23,6 +23,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Mapper; import org.apache.kylin.common.util.HadoopUtil; +import org.apache.kylin.common.util.MemoryBudgetController; import org.apache.kylin.engine.mr.common.BatchConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,10 +41,38 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, } @Override - final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + final protected void setup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { + try { + logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); + doSetup(context); + } catch (IOException ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } catch (InterruptedException ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } catch (RuntimeException ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } catch (Error ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } + } + + protected void doSetup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { + // NOTHING + } + + @Override + final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { try { if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Accepting Mapper Key with ordinal: " + mapCounter); + logger.info("Do map, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); } doMap(key, value, context); } catch (IOException ex) { // KYLIN-2170 @@ -61,14 +90,18 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, } } - protected void doMap(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + protected void doMap(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { super.map(key, value, context); } @Override - final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { try { + logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); doCleanup(context); + logger.info("Total rows: {}", mapCounter); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; @@ -84,6 +117,7 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, } } - protected void doCleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + protected void doCleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java index e47af9c..28c62db 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java @@ -23,6 +23,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Reducer; import org.apache.kylin.common.util.HadoopUtil; +import org.apache.kylin.common.util.MemoryBudgetController; import org.apache.kylin.engine.mr.common.BatchConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,10 +40,38 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYI } @Override - final public void reduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + final protected void setup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { + try { + logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); + doSetup(context); + } catch (IOException ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } catch (InterruptedException ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } catch (RuntimeException ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } catch (Error ex) { // KYLIN-2170 + logger.error("", ex); + throw ex; + } + } + + protected void doSetup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { + // NOTHING + } + + @Override + final public void reduce(KEYIN key, Iterable<VALUEIN> values, + Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { try { if (reduceCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Accepting Reducer Key with ordinal: " + reduceCounter); + logger.info("Do reduce, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); } doReduce(key, values, context); @@ -61,14 +90,18 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYI } } - protected void doReduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + protected void doReduce(KEYIN key, Iterable<VALUEIN> values, + Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { super.reduce(key, values, context); } @Override - final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { try { + logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); doCleanup(context); + logger.info("Total rows: " + reduceCounter); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; @@ -84,6 +117,7 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYI } } - protected void doCleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { + protected void doCleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) + throws IOException, InterruptedException { } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java index 93e413b..1ae6cd0 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java @@ -58,7 +58,7 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<K private BaseCuboidBuilder baseCuboidBuilder; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(); segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java index 495be77..2dd9f00 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java @@ -60,7 +60,7 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> { private Text outputValue = new Text(); @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java index a367bc6..8910024 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java @@ -29,7 +29,7 @@ import org.apache.kylin.engine.mr.KylinReducer; public class FactDistinctColumnsCombiner extends KylinReducer<SelfDefineSortableKey, Text, SelfDefineSortableKey, Text> { @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java index 8281759..6e1a5dc 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java @@ -82,8 +82,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB private boolean isUsePutRowKeyToHllNewAlgorithm; @Override - protected void setup(Context context) throws IOException { - super.setup(context); + protected void doSetup(Context context) throws IOException { + super.doSetup(context); tmpbuf = ByteBuffer.allocate(4096); collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED)); if (collectStatistics) { http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java index 458af69..9952def 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java @@ -66,7 +66,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends Kyli protected Map<Integer, Integer> columnIndexToReducerBeginId = new HashMap<>(); @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java index 7f01c3a..20fb9e2 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java @@ -88,7 +88,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK private MultipleOutputs mos; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); mos = new MultipleOutputs(context); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java index a04fb43..a5f267f 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java @@ -32,8 +32,8 @@ public class HiveToBaseCuboidMapper<KEYIN> extends BaseCuboidMapperBase<KEYIN, O private IMRTableInputFormat flatTableInputFormat; @Override - protected void setup(Context context) throws IOException { - super.setup(context); + protected void doSetup(Context context) throws IOException { + super.doSetup(context); flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSegment).getFlatTableInputFormat(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java index eee189c..f8ac7d2 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java @@ -69,7 +69,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr private Future<?> future; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java index 244889f..5aa2c69 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java @@ -55,7 +55,7 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable, ByteArra private Text outputValue; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java index a01a928..c2ef2d2 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java @@ -91,7 +91,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> { private Text outputValue; @Override - protected void setup(Context context) throws IOException, InterruptedException { + protected void doSetup(Context context) throws IOException, InterruptedException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java index 782ce72..e1a68d3 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java @@ -62,7 +62,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> { private NDCuboidBuilder ndCuboidBuilder; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java index eab57d1..d428dcb 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java @@ -48,7 +48,7 @@ public class RowKeyDistributionCheckerMapper extends KylinMapper<Text, Text, Tex List<Text> keyList; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); rowKeyStatsFilePath = context.getConfiguration().get("rowKeyStatsFilePath"); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java index d203e8c..af4a7df 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerReducer.java @@ -33,7 +33,7 @@ public class RowKeyDistributionCheckerReducer extends KylinReducer<Text, LongWri LongWritable outputKey = new LongWritable(0L); @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java index 9e03493..000de9b 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java @@ -50,7 +50,7 @@ public class HiveToBaseCuboidMapperPerformanceTest { HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper(); Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null); - mapper.setup(context); + mapper.doSetup(context); Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath)); Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java index 09602ec..1cadec4 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java @@ -56,7 +56,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab private IMRTableInputFormat tableInputFormat; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java index 0648960..53a2294 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java @@ -44,7 +44,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri private Map<Integer, HLLCounter> hllcMap = new HashMap<Integer, HLLCounter>(); @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java index e94c30e..39df3fc 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java @@ -33,7 +33,7 @@ public class KafkaFlatTableMapper extends KylinMapper<LongWritable, BytesWritabl private Text outValue = new Text(); @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java index 371a83b..f3d9f69 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java @@ -53,7 +53,7 @@ public class CubeHFileMapper extends KylinMapper<Text, Text, ImmutableBytesWrita List<KeyValueCreator> keyValueCreators; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java index c66ccb3..30585df 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java @@ -39,7 +39,7 @@ public class RangeKeyDistributionMapper extends KylinMapper<Text, Text, Text, Lo private Text lastKey; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java index 63433dd..3899b4c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java @@ -54,7 +54,7 @@ public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable private String output = null; @Override - protected void setup(Context context) throws IOException { + protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); if (context.getConfiguration().get(BatchConstants.CFG_OUTPUT_PATH) != null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/55c6ee78/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java index d47f393..b43e4a8 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java @@ -73,7 +73,7 @@ public class CubeHFileMapper2Test extends LocalFileMetadataTestCase { Context context = MockupMapContext.create(hconf, cubeName, outKV); CubeHFileMapper mapper = new CubeHFileMapper(); - mapper.setup(context); + mapper.doSetup(context); Text key = new Text("not important"); Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });
