Revert "KYLIN-1415 Cube parallel merge" This reverts commit 8aef3cab02bc670dbabb7d1418b8e3a8a394f8fc.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d0005d3c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d0005d3c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d0005d3c Branch: refs/heads/helix-rebase Commit: d0005d3c03a83a88d37fe990282bf02093be2757 Parents: 0fa8a02 Author: shaofengshi <shaofeng...@apache.org> Authored: Tue Feb 16 10:40:07 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Wed Mar 2 17:29:02 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/cube/CubeManager.java | 48 +++++++------------- .../apache/kylin/rest/service/CubeService.java | 2 +- 2 files changed, 18 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d0005d3c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 16922ac..84dd30a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -18,11 +18,13 @@ package org.apache.kylin.cube; -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.Nullable; + import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; @@ -55,11 +57,11 @@ import org.apache.kylin.source.SourceFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; /** * @author yangli9 @@ -456,14 +458,8 @@ public class CubeManager implements IRealizationProvider { } public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException { - return mergeSegments(cube, startDate, endDate, forceMergeEmptySeg, true); - } - - public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg, boolean strictCheck) throws IOException { + checkNoBuildingSegment(cube); checkCubeIsPartitioned(cube); - - if (strictCheck) - checkNoBuildingSegment(cube); Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate); CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond()); @@ -623,23 +619,13 @@ public class CubeManager implements IRealizationProvider { return null; } - List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY)); - - if (readySegments.size() == 0) { - logger.debug("Cube " + cube.getName() + " has no ready segment to merge"); + if (cube.getBuildingSegments().size() > 0) { + logger.debug("Cube " + cube.getName() + " has bulding segment, will not trigger merge at this moment"); return null; } - List<CubeSegment> buildingSegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.NEW)); - List<CubeSegment> toSkipSegments = Lists.newArrayList(); - for (CubeSegment building : buildingSegments) { - for (CubeSegment ready : readySegments) { - if (ready.getDateRangeStart() >= building.getDateRangeStart() && ready.getDateRangeEnd() <= building.getDateRangeEnd()) { - toSkipSegments.add(ready); - } - } - } - readySegments.removeAll(toSkipSegments); + List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY)); + if (readySegments.size() == 0) { logger.debug("Cube " + cube.getName() + " has no ready segment to merge"); return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/d0005d3c/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index abc613b..b2a278a 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -653,7 +653,7 @@ public class CubeService extends BasicService { cube = getCubeManager().getCube(cubeName); CubeSegment newSeg = getCubeManager().autoMergeCubeSegments(cube); if (newSeg != null) { - newSeg = getCubeManager().mergeSegments(cube, newSeg.getDateRangeStart(), newSeg.getDateRangeEnd(), true, false); + newSeg = getCubeManager().mergeSegments(cube, newSeg.getDateRangeStart(), newSeg.getDateRangeEnd(), true); logger.debug("Will submit merge job on " + newSeg); DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM"); getExecutableManager().addJob(job);