Repository: kylin Updated Branches: refs/heads/master 5ea095a32 -> 6b068e36e
KYLIN-2170 refactor doMap()/doRecuce()/doCleanup() Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b068e36 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b068e36 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b068e36 Branch: refs/heads/master Commit: 6b068e36e2a1fb7b18bddcea2d73ee93c387ab99 Parents: 5ea095a Author: Li Yang <liy...@apache.org> Authored: Thu Nov 10 10:50:24 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Thu Nov 10 10:50:36 2016 +0800 ---------------------------------------------------------------------- .../mr/steps/fdc2/FactDistinctColumnsCombiner2.java | 2 +- .../mr/steps/fdc2/FactDistinctColumnsMapperBase2.java | 2 +- .../mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java | 4 ++-- .../kylin/source/kafka/hadoop/KafkaFlatTableMapper.java | 12 ++++-------- 4 files changed, 8 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6b068e36/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java index 6ff07f0..6652f4e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java @@ -34,7 +34,7 @@ public class FactDistinctColumnsCombiner2 extends KylinReducer<SelfDefineSortabl } @Override - public void reduce(SelfDefineSortableKey key, Iterable<Text> values, Context context) throws IOException, InterruptedException { + public void doReduce(SelfDefineSortableKey key, Iterable<Text> values, Context context) throws IOException, InterruptedException { // for hll, each key only has one output, no need to do local combine; // for normal col, values are empty text http://git-wip-us.apache.org/repos/asf/kylin/blob/6b068e36/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java index 2e9a2dc..6238d22 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java @@ -41,7 +41,7 @@ import java.util.List; /** */ -public class FactDistinctColumnsMapperBase2<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VALUEIN, SelfDefineSortableKey, Text> { +abstract public class FactDistinctColumnsMapperBase2<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VALUEIN, SelfDefineSortableKey, Text> { protected String cubeName; protected CubeInstance cube; http://git-wip-us.apache.org/repos/asf/kylin/blob/6b068e36/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java index dfc46b6..bdf631d 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java @@ -131,7 +131,7 @@ public class FactDistinctHiveColumnsMapper2<KEYIN> extends FactDistinctColumnsMa } @Override - public void map(KEYIN key, Object record, Context context) throws IOException, InterruptedException { + public void doMap(KEYIN key, Object record, Context context) throws IOException, InterruptedException { String[] row = flatTableInputFormat.parseMapperInput(record); keyBuffer.clear(); @@ -208,7 +208,7 @@ public class FactDistinctHiveColumnsMapper2<KEYIN> extends FactDistinctColumnsMa } @Override - protected void cleanup(Context context) throws IOException, InterruptedException { + protected void doCleanup(Context context) throws IOException, InterruptedException { if (collectStatistics) { ByteBuffer hllBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE); // output each cuboid's hll to reducer, key is 0 - cuboidId http://git-wip-us.apache.org/repos/asf/kylin/blob/6b068e36/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java index 995b2d4..e94c30e 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableMapper.java @@ -39,13 +39,9 @@ public class KafkaFlatTableMapper extends KylinMapper<LongWritable, BytesWritabl } @Override - public void map(LongWritable key, BytesWritable value, Context context) throws IOException { - try { - outKey.set(Bytes.toBytes(key.get())); - outValue.set(value.getBytes(), 0, value.getLength()); - context.write(outKey, outValue); - } catch (InterruptedException e) { - e.printStackTrace(); - } + public void doMap(LongWritable key, BytesWritable value, Context context) throws IOException, InterruptedException { + outKey.set(Bytes.toBytes(key.get())); + outValue.set(value.getBytes(), 0, value.getLength()); + context.write(outKey, outValue); } }
