Repository: incubator-kylin Updated Branches: refs/heads/0.8.0 02b0d84a0 -> 247d4f13c
bug fix on long running cube steaming Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/247d4f13 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/247d4f13 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/247d4f13 Branch: refs/heads/0.8.0 Commit: 247d4f13c5bc0d0f910e041bdd34f11e1483fe08 Parents: 02b0d84 Author: honma <ho...@ebay.com> Authored: Tue Jun 9 19:07:51 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Tue Jun 9 19:08:03 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/BasicTest.java | 2 +- .../java/org/apache/kylin/cube/CubeManager.java | 20 ++++++++++++++++---- .../apache/kylin/query/routing/QueryRouter.java | 12 ++++++------ .../apache/kylin/rest/service/CacheService.java | 1 + .../apache/kylin/rest/service/CubeService.java | 2 ++ .../kylin/storage/hbase/CubeStorageEngine.java | 9 +++++++-- 6 files changed, 33 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java index 73f708d..01a22c1 100644 --- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -77,7 +77,7 @@ public class BasicTest { @Test @Ignore("convenient trial tool for dev") public void test1() throws Exception { - System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433496233000L)); + System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433833611000L)); System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433250517000L)); System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-06-01 00:00:00")); System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-05-15 17:00:00")); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java index ac49dfa..29bc9b0 100644 --- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -18,6 +18,8 @@ package org.apache.kylin.cube; +import com.google.common.base.Function; +import com.google.common.collect.Collections2; import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -48,6 +50,7 @@ import org.apache.kylin.metadata.realization.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.io.IOException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -351,7 +354,7 @@ public class CubeManager implements IRealizationProvider { return appendSegments(cube, startDate, endDate, strictChecking, saveChange); } - public CubeSegment appendSegments(CubeInstance cube, long startDate, long endDate, boolean strictChecking, boolean saveChange) throws IOException { + public CubeSegment appendSegments(CubeInstance cube, long startDate, long endDate, boolean strictChecking, boolean saveChange) throws IOException { if (strictChecking) checkNoBuildingSegment(cube); @@ -385,7 +388,7 @@ public class CubeManager implements IRealizationProvider { Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate); CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond()); - validateNewSegments(cube, newSegment); + validateNewSegments(cube, false, newSegment); CubeBuilder cubeBuilder = new CubeBuilder(cube); cubeBuilder.setToAddSegs(newSegment); @@ -553,7 +556,7 @@ public class CubeManager implements IRealizationProvider { for (CubeSegment segment : readySegments) { long thisSegmentRange = segment.getDateRangeEnd() - segment.getDateRangeStart(); - if (thisSegmentRange >= toMergeRange ) { + if (thisSegmentRange >= toMergeRange) { // this segment and its previous segments will not be merged toMergeSegments.clear(); currentRange = 0; @@ -658,7 +661,7 @@ public class CubeManager implements IRealizationProvider { } firstSeg.validate(); - for (int i = 0, j = 1; j < tobe.size();) { + for (int i = 0, j = 1; j < tobe.size(); ) { CubeSegment is = tobe.get(i); CubeSegment js = tobe.get(j); js.validate(); @@ -750,6 +753,15 @@ public class CubeManager implements IRealizationProvider { usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier()); } + logger.info("Reloaded new cube: " + cubeName + " with reference being" + cubeInstance + " having " + cubeInstance.getSegments().size() + " segments:" + + StringUtils.join(Collections2.transform(cubeInstance.getSegments(), new Function<CubeSegment, String>() { + @Nullable + @Override + public String apply(CubeSegment input) { + return input.getStorageLocationIdentifier(); + } + }), ",")); + return cubeInstance; } catch (Exception e) { logger.error("Error during load cube instance " + path, e); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java index e9e7e7b..14db363 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java +++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java @@ -18,16 +18,15 @@ package org.apache.kylin.query.routing; -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.query.relnode.OLAPContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; /** * @author xjiang @@ -39,6 +38,7 @@ public class QueryRouter { public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException { ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig()); + logger.info("The project manager's reference is " + prjMgr); String factTableName = olapContext.firstTableScan.getTableName(); String projectName = olapContext.olapSchema.getProjectName(); List<IRealization> realizations = Lists.newArrayList(prjMgr.getRealizationsByTable(projectName, factTableName)); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/server/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java index e580f3a..ea6d410 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -67,6 +67,7 @@ public class CacheService extends BasicService { public void rebuildCache(Broadcaster.TYPE cacheType, String cacheKey) { final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey; + logger.info(log); try { switch (cacheType) { case CUBE: http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index 2469ebd..5ea3004 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -242,6 +242,7 @@ public class CubeService extends BasicService { try { if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) { + logger.info("Releasing all segments due to cube desc conflict"); this.releaseAllSegments(cube); } @@ -385,6 +386,7 @@ public class CubeService extends BasicService { throw new JobException("Enable is not allowed with a running job."); } if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) { + logger.info("Releasing all segments due to cube desc conflict"); cube = this.releaseAllSegments(cube); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java index 3689877..4d143d5 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java @@ -144,7 +144,6 @@ public class CubeStorageEngine implements ICachableStorageEngine { return this.uuid; } - @Override public boolean isDynamic() { return false; @@ -399,8 +398,12 @@ public class CubeStorageEngine implements ICachableStorageEngine { List<HBaseKeyRange> result = Lists.newArrayList(); + logger.info("Current cubeInstance is " + cubeInstance + " with " + cubeInstance.getSegments().size() + " segs in all"); + List<CubeSegment> segs = cubeInstance.getSegments(SegmentStatusEnum.READY); + logger.info("READY segs count: " + segs.size()); + // build row key range for each cube segment - for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) { + for (CubeSegment cubeSeg : segs) { // consider derived (lookup snapshot), filter on dimension may // differ per segment @@ -420,7 +423,9 @@ public class CubeStorageEngine implements ICachableStorageEngine { result.addAll(mergedRanges); } + logger.info("hbasekeyrange count: " + result.size()); dropUnhitSegments(result); + logger.info("hbasekeyrange count after dropping unhit :" + result.size()); return result; }
