Merge commit '658353ad57ebb30e6e6a740769a85472354212d9'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/452f4685 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/452f4685 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/452f4685 Branch: refs/heads/master Commit: 452f468533de169f6fe5aa24717367281350376d Parents: b15164a 658353a Author: Li Yang <liy...@apache.org> Authored: Sat Sep 23 21:45:52 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sat Sep 23 21:45:52 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 32 + .../org/apache/kylin/common/util/JsonUtil.java | 3 +- .../common/util/MemoryBudgetController.java | 2 +- .../main/resources/kylin-defaults.properties | 9 + .../org/apache/kylin/cube/CubeInstance.java | 9 +- .../java/org/apache/kylin/cube/CubeManager.java | 34 +- .../java/org/apache/kylin/cube/CubeUpdate.java | 11 + .../kylin/cube/cuboid/TreeCuboidScheduler.java | 3 +- .../algorithm/AbstractRecommendAlgorithm.java | 82 + .../cube/cuboid/algorithm/BPUSCalculator.java | 165 + .../cube/cuboid/algorithm/BenefitPolicy.java | 41 + .../cuboid/algorithm/CuboidBenefitModel.java | 210 + .../algorithm/CuboidRecommendAlgorithm.java | 49 + .../cuboid/algorithm/CuboidRecommender.java | 163 + .../cube/cuboid/algorithm/CuboidStats.java | 291 ++ .../cube/cuboid/algorithm/CuboidStatsUtil.java | 206 + .../cube/cuboid/algorithm/PBPUSCalculator.java | 53 + .../cube/cuboid/algorithm/SPBPUSCalculator.java | 41 + .../algorithm/generic/BitsChromosome.java | 133 + .../generic/CombinedStoppingCondition.java | 43 + .../cuboid/algorithm/generic/CuboidEncoder.java | 44 + .../algorithm/generic/GeneticAlgorithm.java | 283 ++ .../generic/RouletteWheelSelection.java | 61 + .../algorithm/generic/lib/BitsMutation.java | 59 + .../algorithm/generic/lib/Chromosome.java | 104 + .../lib/ChromosomeMismatchException.java | 48 + .../algorithm/generic/lib/ChromosomePair.java | 67 + .../algorithm/generic/lib/CrossoverPolicy.java | 37 + .../generic/lib/ElitisticListPopulation.java | 117 + .../cuboid/algorithm/generic/lib/Fitness.java | 33 + .../generic/lib/FixedGenerationCount.java | 71 + .../algorithm/generic/lib/ListPopulation.java | 223 + .../algorithm/generic/lib/MutationPolicy.java | 34 + .../generic/lib/OnePointCrossover.java | 126 + .../algorithm/generic/lib/Population.java | 59 + .../algorithm/generic/lib/SelectionPolicy.java | 33 + .../generic/lib/StoppingCondition.java | 34 + .../generic/lib/TournamentSelection.java | 114 + .../algorithm/greedy/GreedyAlgorithm.java | 170 + .../cube/inmemcubing/ConcurrentDiskStore.java | 2 +- .../kylin/cube/inmemcubing/MemDiskStore.java | 2 +- .../kylin/cube/model/AggregationGroup.java | 70 +- .../org/apache/kylin/cube/model/CubeDesc.java | 4 +- .../cube/model/TooManyCuboidException.java | 1 + .../kylin/cube/AggregationGroupRuleTest.java | 10 +- .../org/apache/kylin/cube/CubeDescTest.java | 4 +- .../org/apache/kylin/cube/CubeInstanceTest.java | 57 + .../org/apache/kylin/cube/CubeManagerTest.java | 3 +- .../cuboid/algorithm/AlgorithmTestBase.java | 310 ++ .../cuboid/algorithm/CuboidStatsUtilTest.java | 170 + .../cuboid/algorithm/GeneticAlgorithmTest.java | 57 + .../cuboid/algorithm/GreedyAlgorithmTest.java | 58 + .../kylin/cube/project/ProjectManagerTest.java | 5 +- .../src/test/resources/learn_kylin_cube2.json | 84 + core-cube/src/test/resources/statistics.txt | 4092 ++++++++++++++++++ .../apache/kylin/dict/AppendTrieDictionary.java | 37 +- .../apache/kylin/dict/DictionaryManager.java | 10 + .../kylin/dict/GlobalDictionaryBuilder.java | 8 +- .../global/AppendTrieDictionaryBuilder.java | 15 +- .../kylin/dict/global/GlobalDictHDFSStore.java | 80 +- .../kylin/dict/global/GlobalDictMetadata.java | 1 - .../kylin/dict/global/GlobalDictStore.java | 5 +- .../global/SegmentAppendTrieDictBuilder.java | 73 + .../apache/kylin/dict/NumberDictionaryTest.java | 8 +- .../dict/global/AppendTrieDictionaryTest.java | 3 +- .../kylin/measure/bitmap/BitmapMeasureType.java | 21 +- .../kylin/source/datagen/DataGenTest.java | 4 +- .../kylin/engine/mr/common/CubeStatsReader.java | 4 + .../engine/mr/common/CuboidRecommenderUtil.java | 106 + .../engine/mr/common/CuboidStatsReaderUtil.java | 135 + .../mr/common/StatisticsDecisionUtil.java | 114 + .../engine/mr/steps/SaveStatisticsStep.java | 55 +- .../mr/steps/NumberDictionaryForestTest.java | 17 +- .../localmeta/cube_desc/ci_inner_join_cube.json | 2 +- pom.xml | 5 +- .../apache/kylin/rest/service/QueryService.java | 9 +- .../storage/hbase/steps/CreateHTableJob.java | 6 +- webapp/app/js/model/cubeConfig.js | 3 +- .../cubeDesigner/advanced_settings.html | 13 +- webapp/app/partials/query/query.html | 2 +- 80 files changed, 8717 insertions(+), 245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-common/src/main/resources/kylin-defaults.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --cc core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 425e32f,0ba7f21..33dc312 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@@ -314,9 -314,9 +314,6 @@@ public class CubeManager implements IRe // delete cube from project ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName); -- if (listener != null) -- listener.afterCubeDelete(cube); -- return cube; } @@@ -332,9 -332,9 +329,6 @@@ updateCubeWithRetry(new CubeUpdate(cube), 0); ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner); -- if (listener != null) -- listener.afterCubeCreate(cube); -- return cube; } @@@ -348,18 -348,18 +342,11 @@@ ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cube.getName(), projectName, owner); -- if (listener != null) -- listener.afterCubeCreate(cube); -- return cube; } public CubeInstance updateCube(CubeUpdate update) throws IOException { CubeInstance cube = updateCubeWithRetry(update, 0); -- -- if (listener != null) -- listener.afterCubeUpdate(cube); -- return cube; } @@@ -843,23 -847,23 +834,6 @@@ return getCube(name); } -- // ============================================================================ -- -- public interface CubeChangeListener { -- void afterCubeCreate(CubeInstance cube); -- -- void afterCubeUpdate(CubeInstance cube); -- -- void afterCubeDelete(CubeInstance cube); -- } -- -- private CubeChangeListener listener; -- -- public void setCubeChangeListener(CubeChangeListener listener) { -- this.listener = listener; -- } -- -- /** * Calculate the holes (gaps) in segments. * @param cubeName http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --cc core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index d473858,74ac715..d976c79 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@@ -300,42 -301,48 +301,51 @@@ public class AggregationGroup implement public long calculateCuboidCombination() { long combination = 1; - if (this.getDimCap() > 0) { - CuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler(); - combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size(); - } else { - Set<String> includeDims = new TreeSet<>(Arrays.asList(includes)); - Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims)); - - Set<String> hierarchyDims = new TreeSet<>(); - for (String[] ss : selectRule.hierarchyDims) { - hierarchyDims.addAll(Arrays.asList(ss)); - combination = combination * (ss.length + 1); - } - - Set<String> jointDims = new TreeSet<>(); - for (String[] ss : selectRule.jointDims) { - jointDims.addAll(Arrays.asList(ss)); - combination = combination * 2; - } - - Set<String> normalDims = new TreeSet<>(); - normalDims.addAll(includeDims); - normalDims.removeAll(mandatoryDims); - normalDims.removeAll(hierarchyDims); - normalDims.removeAll(jointDims); - - combination = combination * (1L << normalDims.size()); - - if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) { - combination += 1; + try { + if (this.getDimCap() > 0) { + CuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler(); + combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size(); + } else { + Set<String> includeDims = new TreeSet<>(Arrays.asList(includes)); + Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims)); + + Set<String> hierarchyDims = new TreeSet<>(); + for (String[] ss : selectRule.hierarchyDims) { + hierarchyDims.addAll(Arrays.asList(ss)); + combination = LongMath.checkedMultiply(combination, (ss.length + 1)); + } + + Set<String> jointDims = new TreeSet<>(); + for (String[] ss : selectRule.jointDims) { + jointDims.addAll(Arrays.asList(ss)); + } + combination = LongMath.checkedMultiply(combination, (1L << selectRule.jointDims.length)); + + Set<String> normalDims = new TreeSet<>(); + normalDims.addAll(includeDims); + normalDims.removeAll(mandatoryDims); + normalDims.removeAll(hierarchyDims); + normalDims.removeAll(jointDims); + + combination = LongMath.checkedMultiply(combination, (1L << normalDims.size())); + + if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) { + combination += 1; + } + combination -= 1; // not include cuboid 0 } - combination -= 1; // not include cuboid 0 + + if (combination < 0) + throw new ArithmeticException(); + + } catch (ArithmeticException ae) { + // long overflow, give max value + combination = Long.MAX_VALUE; } + if (combination < 0) { // overflow + combination = Long.MAX_VALUE - 1; + } return combination; } http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ----------------------------------------------------------------------
