KYLIN-2783 Refactor CuboidScheduler to be extensible Signed-off-by: Li Yang <liy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/465c5070 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/465c5070 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/465c5070 Branch: refs/heads/ranger Commit: 465c5070dcc27c6d42c2590cf0685ab154b40e7b Parents: 084a6fb Author: Zhong <nju_y...@apache.org> Authored: Thu Aug 17 20:17:26 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sun Sep 10 07:47:20 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeDescManager.java | 3 - .../org/apache/kylin/cube/CubeInstance.java | 18 +- .../java/org/apache/kylin/cube/CubeManager.java | 17 +- .../java/org/apache/kylin/cube/CubeSegment.java | 5 + .../kylin/cube/common/RowKeySplitter.java | 4 +- .../org/apache/kylin/cube/cuboid/Cuboid.java | 180 +++-------- .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 20 +- .../kylin/cube/cuboid/CuboidScheduler.java | 25 +- .../cube/cuboid/DefaultCuboidScheduler.java | 162 +++++++++- .../kylin/cube/cuboid/TreeCuboidScheduler.java | 322 +++++++++++++++++++ .../inmemcubing/AbstractInMemCubeBuilder.java | 17 +- .../cube/inmemcubing/DoggedCubeBuilder.java | 9 +- .../cube/inmemcubing/InMemCubeBuilder.java | 11 +- .../org/apache/kylin/cube/kv/RowKeyDecoder.java | 4 +- .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 11 +- .../kylin/cube/model/AggregationGroup.java | 4 +- .../org/apache/kylin/cube/model/CubeDesc.java | 33 +- .../cube/model/CubeJoinedFlatTableEnrich.java | 3 +- .../org/apache/kylin/cube/util/CubingUtils.java | 2 +- .../kylin/cube/cuboid/CuboidSchedulerTest.java | 26 +- .../apache/kylin/cube/cuboid/CuboidTest.java | 60 ++-- .../apache/kylin/cube/kv/RowKeyDecoderTest.java | 2 +- .../apache/kylin/cube/kv/RowKeyEncoderTest.java | 6 +- .../gtrecord/GTCubeStorageQueryBase.java | 6 +- .../kylin/storage/translate/HBaseKeyRange.java | 2 +- .../kylin/engine/mr/BatchCubingJobBuilder.java | 2 +- .../kylin/engine/mr/BatchCubingJobBuilder2.java | 2 +- .../engine/mr/common/BaseCuboidBuilder.java | 14 +- .../kylin/engine/mr/common/CubeStatsReader.java | 7 +- .../mr/steps/FactDistinctColumnsMapper.java | 2 +- .../engine/mr/steps/InMemCuboidMapper.java | 3 +- .../kylin/engine/mr/steps/KVGTRecordWriter.java | 2 +- .../engine/mr/steps/MergeCuboidMapper.java | 2 +- .../kylin/engine/mr/steps/NDCuboidMapper.java | 6 +- .../apache/kylin/engine/spark/SparkCubing.java | 9 +- .../kylin/engine/spark/SparkCubingByLayer.java | 12 +- .../spark/cube/DefaultTupleConverter.java | 2 +- .../ITDoggedCubeBuilderStressTest.java | 2 +- .../inmemcubing/ITDoggedCubeBuilderTest.java | 4 +- .../inmemcubing/ITInMemCubeBuilderTest.java | 2 +- .../storage/hbase/steps/HBaseCuboidWriter.java | 4 +- .../cube/MeasureTypeOnlyAggrInBaseTest.java | 4 +- 42 files changed, 738 insertions(+), 293 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java index 6635366..f49c9be 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java @@ -171,7 +171,6 @@ public class CubeDescManager { CubeDesc ndesc = loadCubeDesc(CubeDesc.concatResourcePath(name), false); cubeDescMap.putLocal(ndesc.getName(), ndesc); - Cuboid.reloadCache(name); // if related cube is in DESCBROKEN state before, change it back to DISABLED CubeManager cubeManager = CubeManager.getInstance(config); @@ -297,13 +296,11 @@ public class CubeDescManager { String path = cubeDesc.getResourcePath(); getStore().deleteResource(path); cubeDescMap.remove(cubeDesc.getName()); - Cuboid.reloadCache(cubeDesc.getName()); } // remove cubeDesc public void removeLocalCubeDesc(String name) throws IOException { cubeDescMap.removeLocal(name); - Cuboid.reloadCache(name); } private void reloadAllCubeDesc() throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index fad942c..246cbf6 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -26,6 +26,7 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigExt; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; @@ -95,11 +96,26 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, @JsonProperty("create_time_utc") private long createTimeUTC; + // cuboid scheduler lazy built + transient private CuboidScheduler cuboidScheduler; + // default constructor for jackson public CubeInstance() { } - public Segments<CubeSegment> getBuildingSegments() { + public CuboidScheduler getCuboidScheduler() { + if (cuboidScheduler != null) + return cuboidScheduler; + + synchronized (this) { + if (cuboidScheduler == null) { + cuboidScheduler = getDescriptor().getInitialCuboidScheduler(); + } + } + return cuboidScheduler; + } + + public List<CubeSegment> getBuildingSegments() { return segments.getBuildingSegments(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index b782a5e..6ebfd99 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -43,6 +43,7 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.Pair; +import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.DictionaryDesc; import org.apache.kylin.dict.DictionaryInfo; @@ -306,13 +307,14 @@ public class CubeManager implements IRealizationProvider { // delete cube instance and cube desc CubeInstance cube = getCube(cubeName); - if (deleteDesc && cube.getDescriptor() != null) { - CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor()); - } - // remove cube and update cache getStore().deleteResource(cube.getResourcePath()); cubeMap.remove(cube.getName()); + Cuboid.reloadCache(cube); + + if (deleteDesc && cube.getDescriptor() != null) { + CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor()); + } // delete cube from project ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName); @@ -629,17 +631,20 @@ public class CubeManager implements IRealizationProvider { * @param cubeName */ public CubeInstance reloadCubeLocal(String cubeName) { - return reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName)); + CubeInstance cubeInstance = reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName)); + Cuboid.reloadCache(cubeInstance); + return cubeInstance; } public void removeCubeLocal(String cubeName) { CubeInstance cube = cubeMap.get(cubeName); if (cube != null) { + cubeMap.removeLocal(cubeName); for (CubeSegment segment : cube.getSegments()) { usedStorageLocation.remove(segment.getUuid()); } + Cuboid.reloadCache(cube); } - cubeMap.removeLocal(cubeName); } public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) { http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index 358183e..495f9c0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -32,6 +32,7 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.ShardingHash; +import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.kv.CubeDimEncMap; import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.cube.model.CubeDesc; @@ -127,6 +128,10 @@ public class CubeSegment implements IBuildable, ISegment, Serializable { return getCubeInstance().getDescriptor(); } + public CuboidScheduler getCuboidScheduler() { + return getCubeInstance().getCuboidScheduler(); + } + public static String makeSegmentName(TSRange tsRange, SegmentRange segRange) { if (tsRange == null && segRange == null) { return "FULL_BUILD"; http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java index acebce4..cd26347 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java @@ -32,6 +32,7 @@ import org.apache.kylin.metadata.model.TblColRef; public class RowKeySplitter implements java.io.Serializable { + private CubeSegment cubeSegment; private CubeDesc cubeDesc; private RowKeyColumnIO colIO; @@ -63,6 +64,7 @@ public class RowKeySplitter implements java.io.Serializable { } public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { + this.cubeSegment = cubeSeg; this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); @@ -113,7 +115,7 @@ public class RowKeySplitter implements java.io.Serializable { offset += RowConstants.ROWKEY_CUBOIDID_LEN; long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length); - Cuboid cuboid = Cuboid.findById(cubeDesc, lastSplittedCuboidId); + Cuboid cuboid = Cuboid.findById(cubeSegment, lastSplittedCuboidId); // rowkey columns for (int i = 0; i < cuboid.getColumns().size(); i++) { http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java index b71608c..fa83775 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java @@ -21,15 +21,15 @@ package org.apache.kylin.cube.cuboid; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; import org.apache.kylin.cube.model.AggregationGroup; import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask; @@ -38,16 +38,15 @@ import org.apache.kylin.cube.model.RowKeyColDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import com.google.common.collect.Collections2; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Lists; +import com.google.common.collect.Maps; @SuppressWarnings("serial") public class Cuboid implements Comparable<Cuboid>, Serializable { - private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>(); + //TODO Guava cache may be better + private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = Maps.newConcurrentMap(); // smaller is better public final static Comparator<Long> cuboidSelectComparator = new Comparator<Long>() { @@ -57,10 +56,22 @@ public class Cuboid implements Comparable<Cuboid>, Serializable { } }; - // this is the only entry point for query to find the right cuboid - public static Cuboid identifyCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) { - long cuboidID = identifyCuboidId(cubeDesc, dimensions, metrics); - return Cuboid.findById(cubeDesc, cuboidID); + // this is the only entry point for query to find the right cuboid for a segment + public static Cuboid identifyCuboid(CubeSegment cubeSegment, Set<TblColRef> dimensions, + Collection<FunctionDesc> metrics) { + return identifyCuboid(cubeSegment.getCuboidScheduler(), dimensions, metrics); + } + + // this is the only entry point for query to find the right cuboid for a cube instance + public static Cuboid identifyCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensions, + Collection<FunctionDesc> metrics) { + return identifyCuboid(cubeInstance.getCuboidScheduler(), dimensions, metrics); + } + + public static Cuboid identifyCuboid(CuboidScheduler cuboidScheduler, Set<TblColRef> dimensions, + Collection<FunctionDesc> metrics) { + long cuboidID = identifyCuboidId(cuboidScheduler.getCubeDesc(), dimensions, metrics); + return Cuboid.findById(cuboidScheduler, cuboidID); } public static long identifyCuboidId(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) { @@ -77,140 +88,49 @@ public class Cuboid implements Comparable<Cuboid>, Serializable { return cuboidID; } - // for full cube, no need to translate cuboid - public static Cuboid findForFullCube(CubeDesc cube, long cuboidID) { + // for mandatory cuboid, no need to translate cuboid + public static Cuboid findForMandatory(CubeDesc cube, long cuboidID) { return new Cuboid(cube, cuboidID, cuboidID); } - public static Cuboid findById(CubeDesc cube, byte[] cuboidID) { - return findById(cube, Bytes.toLong(cuboidID)); + public static Cuboid findById(CuboidScheduler cuboidScheduler, byte[] cuboidID) { + return findById(cuboidScheduler, Bytes.toLong(cuboidID)); } - public static Cuboid findById(CubeDesc cube, long cuboidID) { - Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cube.getName()); + public static Cuboid findById(CubeSegment cubeSegment, long cuboidID) { + return findById(cubeSegment.getCuboidScheduler(), cuboidID); + } + + public static Cuboid findById(CubeInstance cubeInstance, long cuboidID) { + return findById(cubeInstance.getCuboidScheduler(), cuboidID); + } + + @VisibleForTesting + static Cuboid findById(CubeDesc cubeDesc, long cuboidID) { + return findById(cubeDesc.getInitialCuboidScheduler(), cuboidID); + } + + public static Cuboid findById(CuboidScheduler cuboidScheduler, long cuboidID) { + Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cuboidScheduler.getResponsibleKey()); if (cubeCache == null) { - cubeCache = new ConcurrentHashMap<Long, Cuboid>(); - CUBOID_CACHE.put(cube.getName(), cubeCache); + cubeCache = Maps.newConcurrentMap(); + CUBOID_CACHE.put(cuboidScheduler.getResponsibleKey(), cubeCache); } Cuboid cuboid = cubeCache.get(cuboidID); if (cuboid == null) { - long validCuboidID = translateToValidCuboid(cube, cuboidID); - cuboid = new Cuboid(cube, cuboidID, validCuboidID); + long validCuboidID = cuboidScheduler.findBestMatchCuboid(cuboidID); + cuboid = new Cuboid(cuboidScheduler.getCubeDesc(), cuboidID, validCuboidID); cubeCache.put(cuboidID, cuboid); } return cuboid; } - public static boolean isValid(CubeDesc cube, long cuboidID) { - return cube.getAllCuboids().contains(cuboidID); - } - public static long getBaseCuboidId(CubeDesc cube) { return cube.getRowkey().getFullMask(); } public static Cuboid getBaseCuboid(CubeDesc cube) { - return findById(cube, getBaseCuboidId(cube)); - } - - static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) { - long baseCuboidId = getBaseCuboidId(cubeDesc); - if (cubeDesc.getAllCuboids().contains(cuboidID)) { - return cuboidID; - } - List<Long> onTreeCandidates = Lists.newArrayList(); - for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { - Long candidate = translateToOnTreeCuboid(agg, cuboidID); - if (candidate != null) { - onTreeCandidates.add(candidate); - } - } - - if (onTreeCandidates.size() == 0) { - return baseCuboidId; - } - - long onTreeCandi = Collections.min(onTreeCandidates, cuboidSelectComparator); - if (isValid(cubeDesc, onTreeCandi)) { - return onTreeCandi; - } - - return cubeDesc.getCuboidScheduler().findBestMatchCuboid(onTreeCandi); - } - - private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) { - if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) { - //the partial cube might not contain all required dims - return null; - } - - // add mandantory - cuboidID = cuboidID | agg.getMandatoryColumnMask(); - - // add hierarchy - for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { - long fullMask = hierarchyMask.fullMask; - long intersect = cuboidID & fullMask; - if (intersect != 0 && intersect != fullMask) { - - boolean startToFill = false; - for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) { - if (startToFill) { - cuboidID |= hierarchyMask.dims[i]; - } else { - if ((cuboidID & hierarchyMask.dims[i]) != 0) { - startToFill = true; - cuboidID |= hierarchyMask.dims[i]; - } - } - } - } - } - - // add joint dims - for (Long joint : agg.getJoints()) { - if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) { - cuboidID = cuboidID | joint; - } - } - - if (!agg.isOnTree(cuboidID)) { - // no column, add one column - long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()), agg.getJoints()); - if (nonJointDims != 0) { - long nonJointNonHierarchy = removeBits(nonJointDims, Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() { - @Override - public Long apply(HierarchyMask input) { - return input.fullMask; - } - })); - if (nonJointNonHierarchy != 0) { - //there exists dim that does not belong to any joint or any hierarchy, that's perfect - return cuboidID | Long.lowestOneBit(nonJointNonHierarchy); - } else { - //choose from a hierarchy that does not intersect with any joint dim, only check level 1 - long allJointDims = agg.getJointDimsMask(); - for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { - long dim = hierarchyMask.allMasks[0]; - if ((dim & allJointDims) == 0) { - return cuboidID | dim; - } - } - } - } - - cuboidID = cuboidID | Collections.min(agg.getJoints(), cuboidSelectComparator); - Preconditions.checkState(agg.isOnTree(cuboidID)); - } - return cuboidID; - } - - private static long removeBits(long original, Collection<Long> toRemove) { - long ret = original; - for (Long joint : toRemove) { - ret = ret & ~joint; - } - return ret; + return findForMandatory(cube, getBaseCuboidId(cube)); } // ============================================================================ @@ -310,8 +230,12 @@ public class Cuboid implements Comparable<Cuboid>, Serializable { CUBOID_CACHE.clear(); } - public static void reloadCache(String cubeDescName) { - CUBOID_CACHE.remove(cubeDescName); + public static void reloadCache(CubeInstance cubeInstance) { + reloadCache(cubeInstance.getCuboidScheduler()); + } + + private static void reloadCache(CuboidScheduler cuboidScheduler) { + CUBOID_CACHE.remove(cuboidScheduler.getResponsibleKey()); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java index d657a43..4300a6c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java @@ -44,7 +44,7 @@ public class CuboidCLI { } public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) { - CuboidScheduler scheduler = cubeDesc.getCuboidScheduler(); + CuboidScheduler scheduler = cubeDesc.getInitialCuboidScheduler(); long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc); Collection<Long> cuboidSet = new TreeSet<Long>(); cuboidSet.add(baseCuboid); @@ -88,28 +88,28 @@ public class CuboidCLI { //check all valid and invalid for (long i = 0; i < baseCuboid; ++i) { if (cuboidSet.contains(i)) { - if (!Cuboid.isValid(cubeDesc, i)) { + if (!cubeDesc.getInitialCuboidScheduler().isValid(i)) { throw new RuntimeException(); } - if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) { + if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i) != i) { throw new RuntimeException(); } } else { - if (Cuboid.isValid(cubeDesc, i)) { + if (cubeDesc.getInitialCuboidScheduler().isValid(i)) { throw new RuntimeException(); } - long corrected = Cuboid.translateToValidCuboid(cubeDesc, i); + long corrected = cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(i); if (corrected == i) { throw new RuntimeException(); } - if (!Cuboid.isValid(cubeDesc, corrected)) { + if (!cubeDesc.getInitialCuboidScheduler().isValid(corrected)) { throw new RuntimeException(); } - if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) { + if (cubeDesc.getInitialCuboidScheduler().findBestMatchCuboid(corrected) != corrected) { throw new RuntimeException(); } } @@ -125,7 +125,7 @@ public class CuboidCLI { long baseCuboid = Cuboid.getBaseCuboidId(cube); TreeSet<Long> expectedCuboids = new TreeSet<Long>(); for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) { - if (Cuboid.isValid(cube, cuboid)) { + if (cube.getInitialCuboidScheduler().isValid(cuboid)) { expectedCuboids.add(cuboid); } } @@ -133,10 +133,10 @@ public class CuboidCLI { } public static int[] calculateAllLevelCount(CubeDesc cube) { - int levels = cube.getBuildLevel(); + int levels = cube.getInitialBuildLevel(); int[] allLevelCounts = new int[levels + 1]; - CuboidScheduler scheduler = cube.getCuboidScheduler(); + CuboidScheduler scheduler = cube.getInitialCuboidScheduler(); LinkedList<Long> nextQueue = new LinkedList<Long>(); LinkedList<Long> currentQueue = new LinkedList<Long>(); long baseCuboid = Cuboid.getBaseCuboidId(cube); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java index e802230..cef0f77 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Set; import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.cube.model.AggregationGroup; import org.apache.kylin.cube.model.CubeDesc; import com.google.common.base.Preconditions; @@ -63,14 +62,25 @@ abstract public class CuboidScheduler { /** Returns a cuboid on the tree that best matches the request cuboid. */ abstract public long findBestMatchCuboid(long requestCuboid); - - /** (AggGroupScheduler) Calculate the cuboid set defined by an aggregation group. */ - abstract public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg); + + /** Returns whether requestCuboid is valid or not*/ + abstract public boolean isValid(long requestCuboid); + + /** Returns the key for what this cuboid scheduler responsible for*/ + abstract public String getResponsibleKey(); // ============================================================================ private transient List<List<Long>> cuboidsByLayer; + public long getBaseCuboidId() { + return Cuboid.getBaseCuboidId(cubeDesc); + } + + public CubeDesc getCubeDesc() { + return cubeDesc; + } + /** * Get cuboids by layer. It's built from pre-expanding tree. * @return layered cuboids @@ -105,4 +115,11 @@ abstract public class CuboidScheduler { return cuboidsByLayer; } + /** + * Get cuboid level count except base cuboid + * @return + */ + public int getBuildLevel() { + return getCuboidsByLayer().size() - 1; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java index f7d22da..cba417a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java @@ -35,10 +35,14 @@ import javax.annotation.Nullable; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.model.AggregationGroup; +import org.apache.kylin.cube.model.AggregationGroup.HierarchyMask; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.TooManyCuboidException; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -83,14 +87,48 @@ public class DefaultCuboidScheduler extends CuboidScheduler { return Sets.newHashSet(allCuboidIds); } + @Override + public boolean isValid(long requestCuboid) { + return allCuboidIds.contains(requestCuboid); + } + /** - * Get the parent cuboid really on the spanning tree. - * @param child an on-tree cuboid + * Get the parent cuboid rely on the spanning tree. + * @param cuboid an on-tree cuboid * @return */ @Override - public long findBestMatchCuboid(long child) { - long parent = getOnTreeParent(child); + public long findBestMatchCuboid(long cuboid) { + return findBestMatchCuboid1(cuboid); + } + + public long findBestMatchCuboid1(long cuboid) { + if (isValid(cuboid)) { + return cuboid; + } + + List<Long> onTreeCandidates = Lists.newArrayList(); + for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { + Long candidate = translateToOnTreeCuboid(agg, cuboid); + if (candidate != null) { + onTreeCandidates.add(candidate); + } + } + + if (onTreeCandidates.size() == 0) { + return getBaseCuboidId(); + } + + long onTreeCandi = Collections.min(onTreeCandidates, Cuboid.cuboidSelectComparator); + if (isValid(onTreeCandi)) { + return onTreeCandi; + } + + return doFindBestMatchCuboid1(onTreeCandi); + } + + public long doFindBestMatchCuboid1(long cuboid) { + long parent = getOnTreeParent(cuboid); while (parent > 0) { if (cubeDesc.getAllCuboids().contains(parent)) { break; @@ -99,11 +137,88 @@ public class DefaultCuboidScheduler extends CuboidScheduler { } if (parent <= 0) { - throw new IllegalStateException("Can't find valid parent for Cuboid " + child); + throw new IllegalStateException("Can't find valid parent for Cuboid " + cuboid); } return parent; } + private static Long translateToOnTreeCuboid(AggregationGroup agg, long cuboidID) { + if ((cuboidID & ~agg.getPartialCubeFullMask()) > 0) { + //the partial cube might not contain all required dims + return null; + } + + // add mandantory + cuboidID = cuboidID | agg.getMandatoryColumnMask(); + + // add hierarchy + for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { + long fullMask = hierarchyMask.fullMask; + long intersect = cuboidID & fullMask; + if (intersect != 0 && intersect != fullMask) { + + boolean startToFill = false; + for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) { + if (startToFill) { + cuboidID |= hierarchyMask.dims[i]; + } else { + if ((cuboidID & hierarchyMask.dims[i]) != 0) { + startToFill = true; + cuboidID |= hierarchyMask.dims[i]; + } + } + } + } + } + + // add joint dims + for (Long joint : agg.getJoints()) { + if (((cuboidID | joint) != cuboidID) && ((cuboidID & ~joint) != cuboidID)) { + cuboidID = cuboidID | joint; + } + } + + if (!agg.isOnTree(cuboidID)) { + // no column, add one column + long nonJointDims = removeBits((agg.getPartialCubeFullMask() ^ agg.getMandatoryColumnMask()), + agg.getJoints()); + if (nonJointDims != 0) { + long nonJointNonHierarchy = removeBits(nonJointDims, + Collections2.transform(agg.getHierarchyMasks(), new Function<HierarchyMask, Long>() { + @Override + public Long apply(HierarchyMask input) { + return input.fullMask; + } + })); + if (nonJointNonHierarchy != 0) { + //there exists dim that does not belong to any joint or any hierarchy, that's perfect + return cuboidID | Long.lowestOneBit(nonJointNonHierarchy); + } else { + //choose from a hierarchy that does not intersect with any joint dim, only check level 1 + long allJointDims = agg.getJointDimsMask(); + for (HierarchyMask hierarchyMask : agg.getHierarchyMasks()) { + long dim = hierarchyMask.allMasks[0]; + if ((dim & allJointDims) == 0) { + return cuboidID | dim; + } + } + } + } + + cuboidID = cuboidID | Collections.min(agg.getJoints(), Cuboid.cuboidSelectComparator); + Preconditions.checkState(agg.isOnTree(cuboidID)); + } + return cuboidID; + } + + private static long removeBits(long original, Collection<Long> toRemove) { + long ret = original; + for (Long joint : toRemove) { + ret = ret & ~joint; + } + return ret; + } + private long getOnTreeParent(long child) { Collection<Long> candidates = getOnTreeParents(child); if (candidates == null || candidates.isEmpty()) { @@ -259,7 +374,6 @@ public class DefaultCuboidScheduler extends CuboidScheduler { * @param agg agg group * @return cuboidId list */ - @Override public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg) { Set<Long> cuboidHolder = new HashSet<>(); @@ -352,4 +466,40 @@ public class DefaultCuboidScheduler extends CuboidScheduler { return Long.bitCount(cuboidID) <= dimCap; } + public long findBestMatchCuboid2(long cuboid) { + long bestParent = doFindBestMatchCuboid2(cuboid, Cuboid.getBaseCuboidId(cubeDesc)); + if (bestParent < -1) { + throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboid); + } + return bestParent; + } + + private long doFindBestMatchCuboid2(long cuboid, long parent) { + if (!canDerive(cuboid, parent)) { + return -1; + } + List<Long> children = parent2child.get(parent); + List<Long> candidates = Lists.newArrayList(); + if (children != null) { + for (long child : children) { + long candidate = doFindBestMatchCuboid2(cuboid, child); + if (candidate > 0) { + candidates.add(candidate); + } + } + } + if (candidates.isEmpty()) { + candidates.add(parent); + } + + return Collections.min(candidates, Cuboid.cuboidSelectComparator); + } + + private boolean canDerive(long cuboidId, long parentCuboid) { + return (cuboidId & ~parentCuboid) == 0; + } + + public String getResponsibleKey() { + return CubeDesc.class.getName() + "-" + cubeDesc.getName(); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java new file mode 100644 index 0000000..414fc4a --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.cuboid; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.model.CubeDesc; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +public class TreeCuboidScheduler extends CuboidScheduler { + + final private CuboidTree cuboidTree; + + public TreeCuboidScheduler(CubeDesc cubeDesc, List<Long> allCuboidIds, Comparator<Long> cuboidComparator) { + super(cubeDesc); + cuboidTree = CuboidTree.createFromCuboids(allCuboidIds, cuboidComparator); + } + + @Override + public Set<Long> getAllCuboidIds() { + return cuboidTree.getAllCuboidIds(); + } + + @Override + public int getCuboidCount() { + return cuboidTree.getCuboidCount(Cuboid.getBaseCuboidId(cubeDesc)); + } + + @Override + public List<Long> getSpanningCuboid(long cuboidId) { + return cuboidTree.getSpanningCuboid(cuboidId); + } + + @Override + public long findBestMatchCuboid(long cuboidId) { + return cuboidTree.findBestMatchCuboid(cuboidId); + } + + @Override + public boolean isValid(long requestCuboid) { + return cuboidTree.isValid(requestCuboid); + } + + public static class CuboidTree { + private int treeLevels; + + private TreeNode root; + + private Comparator<Long> cuboidComparator; + + private Map<Long, TreeNode> index = new HashMap<>(); + + @VisibleForTesting + static CuboidTree createFromCuboids(List<Long> allCuboidIds) { + return createFromCuboids(allCuboidIds, Cuboid.cuboidSelectComparator); + } + + @VisibleForTesting + static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) { + // sort the cuboid ids in descending order, so that don't need to adjust + // the cuboid tree when adding cuboid id to the tree. + Collections.sort(allCuboidIds, new Comparator<Long>() { + @Override + public int compare(Long o1, Long o2) { + return Long.compare(o2, o1); + } + }); + long basicCuboidId = allCuboidIds.get(0); + CuboidTree cuboidTree = new CuboidTree(cuboidComparator); + cuboidTree.setRoot(basicCuboidId); + + for (long cuboidId : allCuboidIds) { + cuboidTree.addCuboid(cuboidId); + } + cuboidTree.buildIndex(); + return cuboidTree; + } + + private CuboidTree(Comparator<Long> cuboidComparator) { + this.cuboidComparator = cuboidComparator; + } + + public Set<Long> getAllCuboidIds() { + return index.keySet(); + } + + public List<Long> getSpanningCuboid(long cuboidId) { + TreeNode node = index.get(cuboidId); + if (node == null) { + throw new IllegalArgumentException("the cuboid:" + cuboidId + " is not exist in the tree"); + } + + return Lists.transform(node.children, new Function<TreeNode, Long>() { + @Nullable + @Override + public Long apply(@Nullable TreeNode input) { + return input.cuboidId; + } + }); + } + + public long findBestMatchCuboid(long cuboidId) { + // exactly match + if (isValid(cuboidId)) { + return cuboidId; + } + + return findBestParent(cuboidId).cuboidId; + } + + public boolean isValid(long cuboidId) { + return index.containsKey(cuboidId); + } + + private int getCuboidCount(long cuboidId) { + int r = 1; + for (Long child : getSpanningCuboid(cuboidId)) { + r += getCuboidCount(child); + } + return r; + } + + public void print(PrintWriter out) { + int dimensionCnt = Long.bitCount(root.cuboidId); + doPrint(root, dimensionCnt, 0, out); + } + + private void doPrint(TreeNode node, int dimensionCount, int depth, PrintWriter out) { + printCuboid(node.cuboidId, dimensionCount, depth, out); + + for (TreeNode child : node.children) { + doPrint(child, dimensionCount, depth + 1, out); + } + } + + private void printCuboid(long cuboidID, int dimensionCount, int depth, PrintWriter out) { + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < depth; i++) { + sb.append(" "); + } + String cuboidName = Cuboid.getDisplayName(cuboidID, dimensionCount); + sb.append("|---- Cuboid ").append(cuboidName).append("(" + cuboidID + ")"); + out.println(sb.toString()); + } + + private void setRoot(long basicCuboidId) { + this.root = new TreeNode(basicCuboidId, 0); + this.treeLevels = 0; + } + + private void buildIndex() { + LinkedList<TreeNode> queue = new LinkedList<>(); + queue.add(root); + while (!queue.isEmpty()) { + TreeNode node = queue.removeFirst(); + index.put(node.cuboidId, node); + for (TreeNode child : node.children) { + queue.add(child); + } + } + } + + private void addCuboid(long cuboidId) { + TreeNode parent = findBestParent(cuboidId); + if (parent != null && parent.cuboidId != cuboidId) { + parent.addChild(cuboidId, parent.level); + this.treeLevels = Math.max(this.treeLevels, parent.level + 1); + } + } + + private TreeNode findBestParent(long cuboidId) { + TreeNode bestParent = doFindBestParent(cuboidId, root); + if (bestParent == null) { + throw new IllegalStateException("Cannot find the parent of the cuboid:" + cuboidId); + } + return bestParent; + } + + private TreeNode doFindBestParent(long cuboidId, TreeNode parentCuboid) { + if (!canDerive(cuboidId, parentCuboid.cuboidId)) { + return null; + } + + List<TreeNode> candidates = Lists.newArrayList(); + for (TreeNode childCuboid : parentCuboid.children) { + TreeNode candidate = doFindBestParent(cuboidId, childCuboid); + if (candidate != null) { + candidates.add(candidate); + } + } + if (candidates.isEmpty()) { + candidates.add(parentCuboid); + } + + return Collections.min(candidates, new Comparator<TreeNode>() { + @Override + public int compare(TreeNode o1, TreeNode o2) { + return cuboidComparator.compare(o1.cuboidId, o2.cuboidId); + } + }); + } + + private boolean canDerive(long cuboidId, long parentCuboid) { + return (cuboidId & ~parentCuboid) == 0; + } + } + + public static class TreeNode { + @JsonProperty("cuboid_id") + long cuboidId; + @JsonIgnore + int level; + @JsonProperty("children") + List<TreeNode> children = new ArrayList<>(); + + public long getCuboidId() { + return cuboidId; + } + + public int getLevel() { + return level; + } + + public List<TreeNode> getChildren() { + return children; + } + + TreeNode(long cuboidId, int level) { + this.cuboidId = cuboidId; + this.level = level; + } + + void addChild(long childId, int parentlevel) { + this.children.add(new TreeNode(childId, parentlevel + 1)); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (cuboidId ^ (cuboidId >>> 32)); + result = prime * result + level; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + TreeNode other = (TreeNode) obj; + if (cuboidId != other.cuboidId) + return false; + if (level != other.level) + return false; + return true; + } + } + + /** + * Compare cuboid according to the cuboid data row count + */ + public static class CuboidCostComparator implements Comparator<Long> { + private Map<Long, Long> cuboidStatistics; + + public CuboidCostComparator(Map<Long, Long> cuboidStatistics) { + Preconditions.checkArgument(cuboidStatistics != null, + "the input " + cuboidStatistics + " should not be null!!!"); + this.cuboidStatistics = cuboidStatistics; + } + + @Override + public int compare(Long cuboid1, Long cuboid2) { + Long rowCnt1 = cuboidStatistics.get(cuboid1); + Long rowCnt2 = cuboidStatistics.get(cuboid2); + if (rowCnt2 == null || rowCnt1 == null) { + return Cuboid.cuboidSelectComparator.compare(cuboid1, cuboid2); + } + return Long.compare(rowCnt1, rowCnt2); + } + } + + public String getResponsibleKey() { + return CubeInstance.class.getName() + "-" + cubeDesc.getName(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java index c7a4a05..ae38261 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.gridtable.GTScanRequest; @@ -42,6 +43,7 @@ abstract public class AbstractInMemCubeBuilder { private static Logger logger = LoggerFactory.getLogger(AbstractInMemCubeBuilder.class); + final protected CuboidScheduler cuboidScheduler; final protected IJoinedFlatTableDesc flatDesc; final protected CubeDesc cubeDesc; final protected Map<TblColRef, Dictionary<String>> dictionaryMap; @@ -49,16 +51,23 @@ abstract public class AbstractInMemCubeBuilder { protected int taskThreadCount = 1; protected int reserveMemoryMB = 100; - public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { - if (flatDesc == null) + // @Deprecated + // public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { + // this(cubeDesc.getInitialCuboidScheduler(), cubeDesc, flatDesc, dictionaryMap); + // } + + protected AbstractInMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, + Map<TblColRef, Dictionary<String>> dictionaryMap) { + if (cuboidScheduler == null) throw new NullPointerException(); - if (cubeDesc == null) + if (flatDesc == null) throw new NullPointerException(); if (dictionaryMap == null) throw new IllegalArgumentException("dictionary cannot be null"); + this.cuboidScheduler = cuboidScheduler; this.flatDesc = flatDesc; - this.cubeDesc = cubeDesc; + this.cubeDesc = cuboidScheduler.getCubeDesc(); this.dictionaryMap = dictionaryMap; } http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java index a9211da..dd92a2b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java @@ -35,7 +35,7 @@ import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.MemoryBudgetController; -import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.gridtable.GTScanRequestBuilder; import org.apache.kylin.gridtable.IGTScanner; @@ -57,8 +57,9 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { private int splitRowThreshold = Integer.MAX_VALUE; private int unitRows = 1000; - public DoggedCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { - super(cubeDesc, flatDesc, dictionaryMap); + public DoggedCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, + Map<TblColRef, Dictionary<String>> dictionaryMap) { + super(cuboidScheduler, flatDesc, dictionaryMap); // check memory more often if a single row is big if (cubeDesc.hasMemoryHungryMeasures()) @@ -276,7 +277,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { RuntimeException exception; public SplitThread() { - this.builder = new InMemCubeBuilder(cubeDesc, flatDesc, dictionaryMap); + this.builder = new InMemCubeBuilder(cuboidScheduler, flatDesc, dictionaryMap); this.builder.setConcurrentThreads(taskThreadCount); this.builder.setReserveMemoryMB(reserveMemoryMB); } http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index 93736c9..684c26b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -39,7 +39,6 @@ import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.cuboid.CuboidScheduler; import org.apache.kylin.cube.gridtable.CubeGridTable; import org.apache.kylin.cube.kv.CubeDimEncMap; -import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.gridtable.GTAggregateScanner; import org.apache.kylin.gridtable.GTBuilder; import org.apache.kylin.gridtable.GTInfo; @@ -71,7 +70,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { private static final double DERIVE_AGGR_CACHE_CONSTANT_FACTOR = 0.1; private static final double DERIVE_AGGR_CACHE_VARIABLE_FACTOR = 0.9; - private final CuboidScheduler cuboidScheduler; private final long baseCuboidId; private final int totalCuboidCount; private final String[] metricsAggrFuncs; @@ -90,9 +88,9 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { private Object[] totalSumForSanityCheck; private ICuboidCollector resultCollector; - public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { - super(cubeDesc, flatDesc, dictionaryMap); - this.cuboidScheduler = cubeDesc.getCuboidScheduler(); + public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, + Map<TblColRef, Dictionary<String>> dictionaryMap) { + super(cuboidScheduler, flatDesc, dictionaryMap); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); this.totalCuboidCount = cuboidScheduler.getCuboidCount(); @@ -109,8 +107,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { } private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { - GTInfo info = CubeGridTable.newGTInfo( - Cuboid.findById(cubeDesc, cuboidID), + GTInfo info = CubeGridTable.newGTInfo(Cuboid.findById(cuboidScheduler, cuboidID), new CubeDimEncMap(cubeDesc, dictionaryMap) ); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java index d5948d4..5a1f668 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java @@ -36,6 +36,7 @@ import org.apache.kylin.metadata.model.TblColRef; */ public class RowKeyDecoder { + private final CubeSegment cubeSegment; private final CubeDesc cubeDesc; private final RowKeyColumnIO colIO; private final RowKeySplitter rowKeySplitter; @@ -44,6 +45,7 @@ public class RowKeyDecoder { private List<String> values; public RowKeyDecoder(CubeSegment cubeSegment) { + this.cubeSegment = cubeSegment; this.cubeDesc = cubeSegment.getCubeDesc(); this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255); this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap()); @@ -73,7 +75,7 @@ public class RowKeyDecoder { if (this.cuboid != null && this.cuboid.getId() == cuboidID) { return; } - this.cuboid = Cuboid.findById(cubeDesc, cuboidID); + this.cuboid = Cuboid.findById(cubeSegment, cuboidID); } private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java index a669fb1..40cda76 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java @@ -18,7 +18,11 @@ package org.apache.kylin.cube.kv; -import com.google.common.base.Preconditions; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.common.util.ImmutableBitSet; @@ -28,10 +32,7 @@ import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.metadata.model.TblColRef; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; +import com.google.common.base.Preconditions; public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable { http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index af026af..852451d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.TreeSet; import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler; import org.apache.kylin.metadata.model.TblColRef; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -295,7 +296,8 @@ public class AggregationGroup implements Serializable { long combination = 1; if (this.getDimCap() > 0) { - combination = cubeDesc.getCuboidScheduler().calculateCuboidsForAggGroup(this).size(); + DefaultCuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler(); + combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size(); } else { Set<String> includeDims = new TreeSet<>(Arrays.asList(includes)); Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims)); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 3cbba50..dff3485 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -51,7 +51,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.Pair; -import org.apache.kylin.cube.cuboid.CuboidScheduler; +import org.apache.kylin.cube.cuboid.DefaultCuboidScheduler; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType; import org.apache.kylin.metadata.MetadataConstants; @@ -189,8 +189,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @JsonInclude(JsonInclude.Include.NON_NULL) private int parentForward = 3; - transient private CuboidScheduler cuboidScheduler = null; - public boolean isEnableSharding() { //in the future may extend to other storage that is shard-able return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID; @@ -459,14 +457,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { return true; } - /** - * Get cuboid level count except base cuboid - * @return - */ - public int getBuildLevel() { - return getCuboidScheduler().getCuboidsByLayer().size() - 1; - } - @Override public int hashCode() { int result = 0; @@ -560,7 +550,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { hostToDerivedMap = Maps.newHashMap(); extendedColumnToHosts = Maps.newHashMap(); cuboidBlackSet = Sets.newHashSet(); - cuboidScheduler = null; } public void init(KylinConfig config) { @@ -618,16 +607,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { amendAllColumns(); } - public CuboidScheduler getCuboidScheduler() { - if (cuboidScheduler != null) - return cuboidScheduler; + public DefaultCuboidScheduler getInitialCuboidScheduler() { + return new DefaultCuboidScheduler(this); + } - synchronized (this) { - if (cuboidScheduler == null) { - cuboidScheduler = CuboidScheduler.getInstance(this); - } - return cuboidScheduler; - } + /** + * Get cuboid level count except base cuboid + * @return + */ + public int getInitialBuildLevel() { + return getInitialCuboidScheduler().getCuboidsByLayer().size() - 1; } public boolean isBlackedCuboid(long cuboidID) { @@ -1169,7 +1158,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } public Set<Long> getAllCuboids() { - return getCuboidScheduler().getAllCuboidIds(); + return getInitialCuboidScheduler().getAllCuboidIds(); } public int getParentForward() { http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java index 223df7c..73da802 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java @@ -53,8 +53,7 @@ public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc, Serializ // check what columns from hive tables are required, and index them private void parseCubeDesc() { - long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); - Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); + Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); // build index for rowkey columns List<TblColRef> cuboidColumns = baseCuboid.getColumns(); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java index 1c3c395..ec4c04a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java @@ -57,7 +57,7 @@ public class CubingUtils { public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) { final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc); final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length; - final Set<Long> allCuboidIds = cubeDesc.getCuboidScheduler().getAllCuboidIds(); + final Set<Long> allCuboidIds = cubeDesc.getInitialCuboidScheduler().getAllCuboidIds(); final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMap(); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java index 58c0edb..3f3a7c4 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java @@ -115,7 +115,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings)); for (long child : spannings) { - assertTrue(Cuboid.isValid(cube, child)); + assertTrue(scheduler.isValid(child)); } } @@ -128,7 +128,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testGetSpanningCuboid2() { CubeDesc cube = getTestKylinCubeWithSeller(); - CuboidScheduler scheduler = cube.getCuboidScheduler(); + CuboidScheduler scheduler = cube.getInitialCuboidScheduler(); // generate 8d System.out.println("Spanning for 8D Cuboids"); @@ -156,7 +156,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testGetSpanningCuboid1() { CubeDesc cube = getTestKylinCubeWithoutSeller(); - CuboidScheduler scheduler = cube.getCuboidScheduler(); + CuboidScheduler scheduler = cube.getInitialCuboidScheduler(); // generate 7d System.out.println("Spanning for 7D Cuboids"); @@ -228,7 +228,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testCuboidCounts1() { CubeDesc cube = getTestKylinCubeWithoutSeller(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); int[] counts = CuboidCLI.calculateAllLevelCount(cube); printCount(counts); int sum = 0; @@ -241,7 +241,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testCuboidCounts2() { CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); int[] counts = CuboidCLI.calculateAllLevelCount(cube); printCount(counts); int sum = 0; @@ -254,7 +254,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testCuboidCounts3() { CubeDesc cube = getTestKylinCubeWithSeller(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); int[] counts = CuboidCLI.calculateAllLevelCount(cube); printCount(counts); int sum = 0; @@ -267,7 +267,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testCuboidCounts4() { CubeDesc cube = getTestKylinCubeWithSellerLeft(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); int[] counts = CuboidCLI.calculateAllLevelCount(cube); printCount(counts); int sum = 0; @@ -280,7 +280,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testCuboidCounts5() { CubeDesc cube = getStreamingCubeDesc(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); int[] counts = CuboidCLI.calculateAllLevelCount(cube); printCount(counts); int sum = 0; @@ -293,7 +293,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testCuboidCounts6() { CubeDesc cube = getCIInnerJoinCube(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); int[] counts = CuboidCLI.calculateAllLevelCount(cube); printCount(counts); int sum = 0; @@ -306,7 +306,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { @Test public void testLargeCube() { CubeDesc cube = getFiftyDimCubeDesc(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); long start = System.currentTimeMillis(); System.out.println(cuboidScheduler.getCuboidCount()); System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms"); @@ -317,7 +317,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim"); twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4))); CubeDesc cube = getTwentyDimCubeDesc(); - CuboidScheduler cuboidScheduler = cube.getCuboidScheduler(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); cuboidScheduler.getCuboidCount(); twentyFile.renameTo(new File(twentyFile.getPath() + ".bad")); } @@ -332,10 +332,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase { } CubeDescManager.clearCache(); CubeDesc cube = getCubeDescManager().getCubeDesc("ut_large_dimension_number"); - CuboidScheduler scheduler = cube.getCuboidScheduler(); + CuboidScheduler scheduler = cube.getInitialCuboidScheduler(); Cuboid baseCuboid = Cuboid.getBaseCuboid(cube); - assertTrue(Cuboid.isValid(cube, baseCuboid.getId())); + assertTrue(scheduler.isValid(baseCuboid.getId())); List<Long> spanningChild = scheduler.getSpanningCuboid(baseCuboid.getId()); assertTrue(spanningChild.size() > 0); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java index 2e64791..8dc944b 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java @@ -73,25 +73,26 @@ public class CuboidTest extends LocalFileMetadataTestCase { public void testIsValid() { CubeDesc cube = getTestKylinCubeWithSeller(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); // base - assertEquals(false, Cuboid.isValid(cube, 0)); - assertEquals(true, Cuboid.isValid(cube, toLong("111111111"))); + assertEquals(false, cuboidScheduler.isValid(0)); + assertEquals(true, cuboidScheduler.isValid(toLong("111111111"))); // mandatory column - assertEquals(false, Cuboid.isValid(cube, toLong("011111110"))); - assertEquals(false, Cuboid.isValid(cube, toLong("100000000"))); + assertEquals(false, cuboidScheduler.isValid(toLong("011111110"))); + assertEquals(false, cuboidScheduler.isValid(toLong("100000000"))); // zero tail - assertEquals(true, Cuboid.isValid(cube, toLong("111111000"))); + assertEquals(true, cuboidScheduler.isValid(toLong("111111000"))); // aggregation group & zero tail - assertEquals(true, Cuboid.isValid(cube, toLong("110000111"))); - assertEquals(true, Cuboid.isValid(cube, toLong("110111000"))); - assertEquals(true, Cuboid.isValid(cube, toLong("111110111"))); - assertEquals(false, Cuboid.isValid(cube, toLong("111110001"))); - assertEquals(false, Cuboid.isValid(cube, toLong("111110100"))); - assertEquals(false, Cuboid.isValid(cube, toLong("110000100"))); + assertEquals(true, cuboidScheduler.isValid(toLong("110000111"))); + assertEquals(true, cuboidScheduler.isValid(toLong("110111000"))); + assertEquals(true, cuboidScheduler.isValid(toLong("111110111"))); + assertEquals(false, cuboidScheduler.isValid(toLong("111110001"))); + assertEquals(false, cuboidScheduler.isValid(toLong("111110100"))); + assertEquals(false, cuboidScheduler.isValid(toLong("110000100"))); } @Test @@ -124,41 +125,45 @@ public class CuboidTest extends LocalFileMetadataTestCase { @Test public void testIsValid2() { CubeDesc cube = getTestKylinCubeWithoutSeller(); - assertEquals(false, Cuboid.isValid(cube, toLong("111111111"))); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); + + assertEquals(false, cuboidScheduler.isValid(toLong("111111111"))); // base - assertEquals(false, Cuboid.isValid(cube, 0)); - assertEquals(true, Cuboid.isValid(cube, toLong("11111111"))); + assertEquals(false, cuboidScheduler.isValid(0)); + assertEquals(true, cuboidScheduler.isValid(toLong("11111111"))); // aggregation group & zero tail - assertEquals(true, Cuboid.isValid(cube, toLong("10000111"))); - assertEquals(false, Cuboid.isValid(cube, toLong("10001111"))); - assertEquals(false, Cuboid.isValid(cube, toLong("11001111"))); - assertEquals(true, Cuboid.isValid(cube, toLong("10000001"))); - assertEquals(true, Cuboid.isValid(cube, toLong("10000101"))); + assertEquals(true, cuboidScheduler.isValid(toLong("10000111"))); + assertEquals(false, cuboidScheduler.isValid(toLong("10001111"))); + assertEquals(false, cuboidScheduler.isValid(toLong("11001111"))); + assertEquals(true, cuboidScheduler.isValid(toLong("10000001"))); + assertEquals(true, cuboidScheduler.isValid(toLong("10000101"))); // hierarchy - assertEquals(true, Cuboid.isValid(cube, toLong("10100000"))); - assertEquals(true, Cuboid.isValid(cube, toLong("10110000"))); - assertEquals(true, Cuboid.isValid(cube, toLong("10111000"))); - assertEquals(false, Cuboid.isValid(cube, toLong("10001000"))); - assertEquals(false, Cuboid.isValid(cube, toLong("10011000"))); + assertEquals(true, cuboidScheduler.isValid(toLong("10100000"))); + assertEquals(true, cuboidScheduler.isValid(toLong("10110000"))); + assertEquals(true, cuboidScheduler.isValid(toLong("10111000"))); + assertEquals(false, cuboidScheduler.isValid(toLong("10001000"))); + assertEquals(false, cuboidScheduler.isValid(toLong("10011000"))); } @Test public void testIsValid3() { CubeDesc cube = getSSBCubeDesc(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); - assertEquals(false, Cuboid.isValid(cube, toLong("10000000000"))); + assertEquals(false, cuboidScheduler.isValid(toLong("10000000000"))); // the 4th is mandatory and isMandatoryOnlyValid is true - assertEquals(true, Cuboid.isValid(cube, toLong("10000001000"))); - assertEquals(true, Cuboid.isValid(cube, toLong("00000001000"))); + assertEquals(true, cuboidScheduler.isValid(toLong("10000001000"))); + assertEquals(true, cuboidScheduler.isValid(toLong("00000001000"))); } @Test public void testFindCuboidByIdWithSingleAggrGroup2() { CubeDesc cube = getTestKylinCubeWithSeller(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); Cuboid cuboid; cuboid = Cuboid.findById(cube, 0); @@ -186,6 +191,7 @@ public class CuboidTest extends LocalFileMetadataTestCase { @Test public void testFindCuboidByIdWithMultiAggrGroup() { CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin(); + CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); Cuboid cuboid; cuboid = Cuboid.findById(cube, toLong("111111110")); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java index 1d1d147..ec1f221 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java @@ -91,7 +91,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase { data[7] = "15"; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); - Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); + Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java index 75e2458..5af8d8a 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java @@ -66,7 +66,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { data[7] = "15"; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); - Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); + Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); @@ -97,7 +97,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { data[8] = "15"; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); - Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); + Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); @@ -133,7 +133,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase { data[8] = null; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); - Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); + Cuboid baseCuboid = Cuboid.findById(cube, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index a3af511..a1a39be 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -131,7 +131,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { Set<TblColRef> dimensionsD = new LinkedHashSet<TblColRef>(); dimensionsD.addAll(groupsD); dimensionsD.addAll(otherDimsD); - Cuboid cuboid = findCuboid(cubeDesc, dimensionsD, metrics); + Cuboid cuboid = findCuboid(cubeInstance, dimensionsD, metrics); context.setCuboid(cuboid); // set whether to aggr at storage @@ -171,8 +171,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { protected abstract String getGTStorage(); - protected Cuboid findCuboid(CubeDesc cubeDesc, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) { - return Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics); + protected Cuboid findCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) { + return Cuboid.identifyCuboid(cubeInstance, dimensionsD, metrics); } protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java index 5db3611..85678ac 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java @@ -84,7 +84,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> { public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, CubeSegment cubeSeg, CubeDesc cubeDesc) { this.cubeSeg = cubeSeg; long cuboidId = this.calculateCuboidID(cubeDesc, dimensionColumns); - this.cuboid = Cuboid.findById(cubeDesc, cuboidId); + this.cuboid = Cuboid.findById(cubeSeg, cuboidId); this.flatOrAndFilter = Lists.newLinkedList(); this.flatOrAndFilter.add(andDimensionRanges); init(andDimensionRanges); http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java index 1ec23b6..f64365a 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java @@ -64,7 +64,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport { // Phase 3: Build Cube RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey(); - final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel(); + final int groupRowkeyColumnsCount = seg.getCuboidScheduler().getBuildLevel(); // base cuboid step result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId)); // n dim cuboid steps http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java index 106077c..51c9056 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java @@ -76,7 +76,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport { } protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) { - final int maxLevel = seg.getCubeDesc().getBuildLevel(); + final int maxLevel = seg.getCuboidScheduler().getBuildLevel(); // base cuboid step result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId)); // n dim cuboid steps http://git-wip-us.apache.org/repos/asf/kylin/blob/465c5070/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java index 07b636b..40f1ac5 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java @@ -18,7 +18,11 @@ package org.apache.kylin.engine.mr.common; -import com.google.common.collect.Sets; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.CubeSegment; @@ -35,10 +39,7 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.Set; +import com.google.common.collect.Sets; /** */ @@ -88,8 +89,7 @@ public class BaseCuboidBuilder implements java.io.Serializable { } private void init() { - long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); - baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); + baseCuboid = Cuboid.getBaseCuboid(cubeDesc); initNullBytes(); }
