Repository: kylin Updated Branches: refs/heads/master e6a84c569 -> 809b78309
KYLIN-2282 Step name Build N-Dimension Cuboid Data : N-Dimension is inaccurate Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/809b7830 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/809b7830 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/809b7830 Branch: refs/heads/master Commit: 809b78309d0b85058aad43e6b863271dfebd1255 Parents: e043d5f Author: shaofengshi <shaofeng...@apache.org> Authored: Thu Jan 5 12:00:12 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Thu Jan 5 12:01:36 2017 +0800 ---------------------------------------------------------------------- .../kylin/engine/mr/BatchCubingJobBuilder.java | 27 +++++++++---------- .../kylin/engine/mr/BatchCubingJobBuilder2.java | 28 +++++++++----------- .../kylin/engine/mr/JobBuilderSupport.java | 11 ++++---- 3 files changed, 30 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/809b7830/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java index 6c973eb..8fbecd3 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java @@ -63,16 +63,15 @@ public class BatchCubingJobBuilder extends JobBuilderSupport { result.addTask(createBuildDictionaryStep(jobId)); // Phase 3: Build Cube - RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey(); - final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel(); + RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey(); + final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel(); final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length; - final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); + final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, groupRowkeyColumnsCount); // base cuboid step - result.addTask(createBaseCuboidStep(cuboidOutputTempPath, jobId)); + result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId)); // n dim cuboid steps - for (int i = 1; i <= groupRowkeyColumnsCount; i++) { - int dimNum = totalRowkeyColumnsCount - i; - result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount)); + for (int i = 1; i < groupRowkeyColumnsCount; i++) { + result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i)); } outputSide.addStepPhase3_BuildCube(result, cuboidRootPath); @@ -84,7 +83,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport { return result; } - private MapReduceExecutable createBaseCuboidStep(String[] cuboidOutputTempPath, String jobId) { + private MapReduceExecutable createBaseCuboidStep(String cuboidOutputPath, String jobId) { // base cuboid job MapReduceExecutable baseCuboidStep = new MapReduceExecutable(); @@ -96,7 +95,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport { appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, "FLAT_TABLE"); // marks flat table input - appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[0]); + appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "0"); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId); @@ -107,20 +106,20 @@ public class BatchCubingJobBuilder extends JobBuilderSupport { return baseCuboidStep; } - private MapReduceExecutable createNDimensionCuboidStep(String[] cuboidOutputTempPath, int dimNum, int totalRowkeyColumnCount) { + private MapReduceExecutable createNDimensionCuboidStep(String parentPath, String outputPath, int level) { // ND cuboid job MapReduceExecutable ndCuboidStep = new MapReduceExecutable(); - ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : " + dimNum + "-Dimension"); + ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : level " + level); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); - appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]); - appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]); + appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, parentPath); + appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step"); - appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + (totalRowkeyColumnCount - dimNum)); + appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + level); ndCuboidStep.setMapReduceParams(cmd.toString()); ndCuboidStep.setMapReduceJobClass(NDCuboidJob.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/809b7830/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java index 966bb1b..20540a6 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java @@ -19,7 +19,6 @@ package org.apache.kylin.engine.mr; import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.cube.model.RowKeyDesc; import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide; import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; @@ -77,16 +76,13 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport { } private void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) { - RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey(); - final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel(); - final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length; - final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); + final int maxLevel = seg.getCubeDesc().getBuildLevel(); + final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, maxLevel); // base cuboid step - result.addTask(createBaseCuboidStep(cuboidOutputTempPath, jobId)); + result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId)); // n dim cuboid steps - for (int i = 1; i < groupRowkeyColumnsCount; i++) { - int dimNum = totalRowkeyColumnsCount - i; - result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount, jobId)); + for (int i = 1; i < maxLevel; i++) { + result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i, jobId)); } } @@ -125,7 +121,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport { return InMemCuboidJob.class; } - private MapReduceExecutable createBaseCuboidStep(String[] cuboidOutputTempPath, String jobId) { + private MapReduceExecutable createBaseCuboidStep(String cuboidOutputPath, String jobId) { // base cuboid job MapReduceExecutable baseCuboidStep = new MapReduceExecutable(); @@ -137,7 +133,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport { appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, "FLAT_TABLE"); // marks flat table input - appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[0]); + appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "0"); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId); @@ -152,20 +148,20 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport { return BaseCuboidJob.class; } - private MapReduceExecutable createNDimensionCuboidStep(String[] cuboidOutputTempPath, int dimNum, int totalRowkeyColumnCount, String jobId) { + private MapReduceExecutable createNDimensionCuboidStep(String parentPath, String outputPath, int level, String jobId) { // ND cuboid job MapReduceExecutable ndCuboidStep = new MapReduceExecutable(); - ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : " + dimNum + "-Dimension"); + ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : level " + level); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); - appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]); - appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]); + appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, parentPath); + appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step"); - appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + (totalRowkeyColumnCount - dimNum)); + appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + level); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId); ndCuboidStep.setMapReduceParams(cmd.toString()); http://git-wip-us.apache.org/repos/asf/kylin/blob/809b7830/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java index 5f5814b..23fce2b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java @@ -192,14 +192,13 @@ public class JobBuilderSupport { return buf.append(" -").append(paraName).append(" ").append(paraValue); } - public String[] getCuboidOutputPaths(String cuboidRootPath, int totalRowkeyColumnCount, int groupRowkeyColumnsCount) { - String[] paths = new String[groupRowkeyColumnsCount + 1]; - for (int i = 0; i <= groupRowkeyColumnsCount; i++) { - int dimNum = totalRowkeyColumnCount - i; - if (dimNum == totalRowkeyColumnCount) { + public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) { + String[] paths = new String[levels]; + for (int i = 0; i < levels; i++) { + if (i == 0) { paths[i] = cuboidRootPath + "base_cuboid"; } else { - paths[i] = cuboidRootPath + dimNum + "d_cuboid"; + paths[i] = cuboidRootPath + "level_" + i + "_cuboid"; } } return paths;
