KYLIN-2201 fix combination check overflow error Signed-off-by: Hongbin Ma <mahong...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1bc8353 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1bc8353 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1bc8353 Branch: refs/heads/v1.6.0-rc2 Commit: a1bc83537b0a62af1f451db96c0c253c143458bd Parents: 791f570 Author: Roger Shi <rogershijich...@hotmail.com> Authored: Thu Nov 17 11:07:17 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Sat Nov 19 15:56:53 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/model/CubeDesc.java | 6 +- .../validation/rule/AggregationGroupRule.java | 4 +- .../kylin/cube/AggregationGroupRuleTest.java | 11 + .../org/apache/kylin/cube/CubeDescTest.java | 7 + .../ut_cube_desc_combination_int_overflow.json | 398 +++++++++++++++++++ 5 files changed, 421 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 47e9be8..6cbed80 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -568,7 +568,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { throw new IllegalStateException("Aggregation group " + index + " select rule field not set"); } - int combination = 1; + long combination = 1; Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(includeDims, agg.getIncludes()); @@ -586,7 +586,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims); if (jointDimsList.size() > 0) { - combination = combination * (1 << jointDimsList.size()); + combination = combination * (1L << jointDimsList.size()); } if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { @@ -607,7 +607,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { normalDims.removeAll(hierarchyDims); normalDims.removeAll(jointDims); - combination = combination * (1 << normalDims.size()); + combination = combination * (1L << normalDims.size()); if (combination > config.getCubeAggrGroupMaxCombination()) { String msg = "Aggregation group " + index + " has too many combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max.combination' to a bigger value."; http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java index c1a062a..a5cf407 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java @@ -62,7 +62,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { continue; } - int combination = 1; + long combination = 1; Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); if (agg.getIncludes() != null) { for (String include : agg.getIncludes()) { @@ -115,7 +115,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { normalDims.removeAll(hierarchyDims); normalDims.removeAll(jointDims); - combination = combination * (1 << normalDims.size()); + combination = combination * (1L << normalDims.size()); if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { Set<String> intersection = new HashSet<>(mandatoryDims); http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java index 0ebb249..563e139 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; @@ -115,6 +116,16 @@ public class AggregationGroupRuleTest { assertEquals("Aggregation group 0 joint dimensions has overlap with more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", (vContext.getResults()[0].getMessage())); } + @Test + public void testCombinationIntOverflow() throws IOException { + ValidateContext vContext = new ValidateContext(); + CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class); + + IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); + rule.validate(desc, vContext); + assertEquals(1, vContext.getResults().length); + } + public AggregationGroupRule getAggregationGroupRule() { AggregationGroupRule rule = new AggregationGroupRule() { @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 417ad46..ca7ae50 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -193,6 +193,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase { } @Test + public void testCombinationIntOverflow() throws Exception { + thrown.expect(IllegalStateException.class); + CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_cube_desc_combination_int_overflow"); + cubeDesc.init(getTestConfig()); + } + + @Test public void testSerialize() throws Exception { CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); String str = JsonUtil.writeValueAsIndentString(desc); http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json new file mode 100644 index 0000000..ec6f8f4 --- /dev/null +++ b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json @@ -0,0 +1,398 @@ +{ + "uuid" : "9e89e128-f13e-4209-82d2-973985114793", + "last_modified" : 1479207711845, + "name" : "ut_cube_desc_combination_int_overflow", + "model_name" : "ut_large_dimension_number", + "description" : "", + "null_string" : null, + "dimensions" : [ { + "name" : "DEFAULT.WIDE_TABLE.A", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "A", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.B", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "B", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.C", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "C", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.D", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "D", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.E", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "E", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.F", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "F", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.G", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "G", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.H", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "H", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.I", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "I", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.J", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "J", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.K", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "K", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.L", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "L", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.M", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "M", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.N", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "N", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.O", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "O", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.P", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "P", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.Q", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "Q", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.R", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "R", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.S", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "S", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.T", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "T", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.U", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "U", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.V", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "V", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.W", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "W", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.X", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "X", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.Y", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "Y", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.Z", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "Z", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AA", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AA", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AB", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AB", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AC", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AC", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AD", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AD", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AE", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AE", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AF", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AF", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AG", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AG", + "derived" : null + },{ + "name" : "DEFAULT.WIDE_TABLE.AH", + "table" : "DEFAULT.WIDE_TABLE", + "column" : "AH", + "derived" : null + } ], + "measures" : [ { + "name" : "_COUNT_", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1", + "next_parameter" : null + }, + "returntype" : "bigint" + }, + "dependent_measure_ref" : null + } ], + "dictionaries" : [ ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "A", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "B", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "C", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "D", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "E", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "F", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "G", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "H", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "I", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "J", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "K", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "L", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "M", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "N", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "O", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "P", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "Q", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "R", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "S", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "T", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "U", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "V", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "W", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "X", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "Y", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "Z", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AA", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AB", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AC", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AD", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AE", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AF", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AG", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }, { + "column" : "AH", + "encoding" : "dict", + "isShardBy" : false, + "index" : "eq" + }] + }, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "_COUNT_" ] + } ] + } ] + }, + "aggregation_groups" : [ { + "includes" : [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "AA", "AB", "AC", "AD", "AE", "AF", "AG", "AH"], + "select_rule" : { + "hierarchy_dims" : [ ], + "mandatory_dims" : [ ], + "joint_dims" : [ ] + } + }], + "signature" : "4NqJVcVIYRF5PkJ0jS+9Rg==", + "notify_list" : [ ], + "status_need_notify" : [ "ERROR", "DISCARDED", "SUCCEED" ], + "partition_date_start" : 1469923200000, + "partition_date_end" : 3153600000000, + "auto_merge_time_ranges" : [ 604800000, 2419200000 ], + "retention_range" : 0, + "engine_type" : 100, + "storage_type" : 100, + "override_kylin_properties" : { + "kylin.cube.algorithm" : "auto", + "kylin.cube.aggrgroup.max.combination" : "4096", + "kylin.job.cubing.inmem.sampling.percent" : "100", + "kylin.hbase.default.compression.codec" : "none" + } +} \ No newline at end of file