This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 47cb8de4a7baf7982c9a1cc26ba9c4ce68e527ff
Author: Liu Shaohui <liushao...@xiaomi.com>
AuthorDate: Fri May 31 17:00:34 2019 +0800
KYLIN-4005: Saving Cube of a aggregation Groups(40 Dimensions, Max
Dimension Combination:5) may cause kylin server OOM
---
.../kylin/cube/cuboid/DefaultCuboidScheduler.java | 4 +-
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 25 +
.../localmeta/cube_desc/fifty_dim_five_cap.json | 576 +++++++++++++++++++++
3 files changed, 603 insertions(+), 2 deletions(-)
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
index 727f61b..acccdf0 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/DefaultCuboidScheduler.java
@@ -175,11 +175,11 @@ public class DefaultCuboidScheduler extends
CuboidScheduler {
long maxCombination = config.getCubeAggrGroupMaxCombination() * 10;
maxCombination = maxCombination < 0 ? Long.MAX_VALUE : maxCombination;
while (!children.isEmpty()) {
+ cuboidHolder.addAll(children);
if (cuboidHolder.size() > maxCombination) {
throw new IllegalStateException("Too many cuboids for the
cube. Cuboid combination reached "
- + cuboidHolder.size() + " and limit is " +
maxCombination + ". Abort calculation.");
+ + cuboidHolder.size() + " and limit is " + maxCombination
+ ". Abort calculation.");
}
- cuboidHolder.addAll(children);
children = getOnTreeParentsByLayer(children);
}
cuboidHolder.add(Cuboid.getBaseCuboidId(cubeDesc));
diff --git
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index c40e766..aac1497 100644
---
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.cube.cuboid;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -31,6 +32,8 @@ import java.util.Set;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -92,6 +95,9 @@ public class CuboidSchedulerTest extends
LocalFileMetadataTestCase {
private CubeDesc getFiftyDimCubeDesc() {
return getCubeDescManager().getCubeDesc("fifty_dim");
}
+ private CubeDesc getFiftyDimFiveCapCubeDesc() {
+ return getCubeDescManager().getCubeDesc("fifty_dim_five_cap");
+ }
private CubeDesc getTwentyDimCubeDesc() {
return getCubeDescManager().getCubeDesc("twenty_dim");
@@ -311,6 +317,25 @@ public class CuboidSchedulerTest extends
LocalFileMetadataTestCase {
System.out.println("build tree takes: " + (System.currentTimeMillis()
- start) + "ms");
}
+ @Test
+ public void testTooLargeCube() {
+ CubeDesc cube = getFiftyDimFiveCapCubeDesc();
+ AggregationGroupRule rule = new AggregationGroupRule();
+ ValidateContext context = new ValidateContext();
+ rule.validate(cube, context);
+ assertFalse(context.ifPass());
+ assertEquals(1, context.getResults().length);
+
+ try {
+ CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
+ long start = System.currentTimeMillis();
+ System.out.println(cuboidScheduler.getCuboidCount());
+ System.out.println("build tree takes: " +
(System.currentTimeMillis() - start) + "ms");
+ assertTrue("Never be here", false);
+ } catch (RuntimeException e) {
+ }
+ }
+
@Test(expected=RuntimeException.class)
public void testTooManyCombination() {
File twentyFile = new File(new
File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim");
diff --git
a/examples/test_case_data/localmeta/cube_desc/fifty_dim_five_cap.json
b/examples/test_case_data/localmeta/cube_desc/fifty_dim_five_cap.json
new file mode 100644
index 0000000..2e10289
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/fifty_dim_five_cap.json
@@ -0,0 +1,576 @@
+{
+ "uuid" :
"examples/test_case_data/localmeta/cube_desc/fifty_dim_five_cap.json",
+ "name" : "fifty_dim_five_cap",
+ "description" : "",
+ "dimensions" : [
+ {
+ "name": "A",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "A",
+ "derived": null
+ },
+ {
+ "name": "B",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "B",
+ "derived": null
+ },
+ {
+ "name": "C",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "C",
+ "derived": null
+ },
+ {
+ "name": "D",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "D",
+ "derived": null
+ },
+ {
+ "name": "E",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "E",
+ "derived": null
+ },
+ {
+ "name": "F",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "F",
+ "derived": null
+ },
+ {
+ "name": "G",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "G",
+ "derived": null
+ },
+ {
+ "name": "H",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "H",
+ "derived": null
+ },
+ {
+ "name": "I",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "I",
+ "derived": null
+ },
+ {
+ "name": "J",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "J",
+ "derived": null
+ },
+ {
+ "name": "K",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "K",
+ "derived": null
+ },
+ {
+ "name": "L",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "L",
+ "derived": null
+ },
+ {
+ "name": "M",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "M",
+ "derived": null
+ },
+ {
+ "name": "N",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "N",
+ "derived": null
+ },
+ {
+ "name": "O",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "O",
+ "derived": null
+ },
+ {
+ "name": "P",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "P",
+ "derived": null
+ },
+ {
+ "name": "Q",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "Q",
+ "derived": null
+ },
+ {
+ "name": "R",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "R",
+ "derived": null
+ },
+ {
+ "name": "S",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "S",
+ "derived": null
+ },
+ {
+ "name": "T",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "T",
+ "derived": null
+ },
+ {
+ "name": "U",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "U",
+ "derived": null
+ },
+ {
+ "name": "V",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "V",
+ "derived": null
+ },
+ {
+ "name": "W",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "W",
+ "derived": null
+ },
+ {
+ "name": "X",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "X",
+ "derived": null
+ },
+ {
+ "name": "Y",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "Y",
+ "derived": null
+ },
+ {
+ "name": "Z",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "Z",
+ "derived": null
+ },
+ {
+ "name": "AA",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "AA",
+ "derived": null
+ },
+ {
+ "name": "BB",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "BB",
+ "derived": null
+ },
+ {
+ "name": "CC",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "CC",
+ "derived": null
+ },
+ {
+ "name": "DD",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "DD",
+ "derived": null
+ },
+ {
+ "name": "EE",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "EE",
+ "derived": null
+ },
+ {
+ "name": "FF",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "FF",
+ "derived": null
+ },
+ {
+ "name": "GG",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "GG",
+ "derived": null
+ },
+ {
+ "name": "HH",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "HH",
+ "derived": null
+ },
+ {
+ "name": "II",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "II",
+ "derived": null
+ },
+ {
+ "name": "JJ",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "JJ",
+ "derived": null
+ },
+ {
+ "name": "KK",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "KK",
+ "derived": null
+ },
+ {
+ "name": "LL",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "LL",
+ "derived": null
+ },
+ {
+ "name": "MM",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "MM",
+ "derived": null
+ },
+ {
+ "name": "NN",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "NN",
+ "derived": null
+ },
+ {
+ "name": "OO",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "OO",
+ "derived": null
+ },
+ {
+ "name": "PP",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "PP",
+ "derived": null
+ },
+ {
+ "name": "QQ",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "QQ",
+ "derived": null
+ },
+ {
+ "name": "RR",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "RR",
+ "derived": null
+ },
+ {
+ "name": "SS",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "SS",
+ "derived": null
+ },
+ {
+ "name": "TT",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "TT",
+ "derived": null
+ },
+ {
+ "name": "UU",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "UU",
+ "derived": null
+ },
+ {
+ "name": "VV",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "VV",
+ "derived": null
+ },
+ {
+ "name": "WW",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "WW",
+ "derived": null
+ },
+ {
+ "name": "XX",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "XX",
+ "derived": null
+ },
+ {
+ "name": "YY",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "YY",
+ "derived": null
+ },
+ {
+ "name": "ZZ",
+ "table": "DEFAULT.FIFTY_DIM",
+ "column": "ZZ",
+ "derived": null
+ }
+ ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [
+ {
+ "column": "A",
+ "encoding": "dict"
+ },
+ {
+ "column": "B",
+ "encoding": "dict"
+ },
+ {
+ "column": "C",
+ "encoding": "dict"
+ },
+ {
+ "column": "D",
+ "encoding": "dict"
+ },
+ {
+ "column": "E",
+ "encoding": "dict"
+ },
+ {
+ "column": "F",
+ "encoding": "dict"
+ },
+ {
+ "column": "G",
+ "encoding": "dict"
+ },
+ {
+ "column": "H",
+ "encoding": "dict"
+ },
+ {
+ "column": "I",
+ "encoding": "dict"
+ },
+ {
+ "column": "J",
+ "encoding": "dict"
+ },
+ {
+ "column": "K",
+ "encoding": "dict"
+ },
+ {
+ "column": "L",
+ "encoding": "dict"
+ },
+ {
+ "column": "M",
+ "encoding": "dict"
+ },
+ {
+ "column": "N",
+ "encoding": "dict"
+ },
+ {
+ "column": "O",
+ "encoding": "dict"
+ },
+ {
+ "column": "P",
+ "encoding": "dict"
+ },
+ {
+ "column": "Q",
+ "encoding": "dict"
+ },
+ {
+ "column": "R",
+ "encoding": "dict"
+ },
+ {
+ "column": "S",
+ "encoding": "dict"
+ },
+ {
+ "column": "T",
+ "encoding": "dict"
+ },
+ {
+ "column": "U",
+ "encoding": "dict"
+ },
+ {
+ "column": "V",
+ "encoding": "dict"
+ },
+ {
+ "column": "W",
+ "encoding": "dict"
+ },
+ {
+ "column": "X",
+ "encoding": "dict"
+ },
+ {
+ "column": "Y",
+ "encoding": "dict"
+ },
+ {
+ "column": "Z",
+ "encoding": "dict"
+ },
+ {
+ "column": "AA",
+ "encoding": "dict"
+ },
+ {
+ "column": "BB",
+ "encoding": "dict"
+ },
+ {
+ "column": "CC",
+ "encoding": "dict"
+ },
+ {
+ "column": "DD",
+ "encoding": "dict"
+ },
+ {
+ "column": "EE",
+ "encoding": "dict"
+ },
+ {
+ "column": "FF",
+ "encoding": "dict"
+ },
+ {
+ "column": "GG",
+ "encoding": "dict"
+ },
+ {
+ "column": "HH",
+ "encoding": "dict"
+ },
+ {
+ "column": "II",
+ "encoding": "dict"
+ },
+ {
+ "column": "JJ",
+ "encoding": "dict"
+ },
+ {
+ "column": "KK",
+ "encoding": "dict"
+ },
+ {
+ "column": "LL",
+ "encoding": "dict"
+ },
+ {
+ "column": "MM",
+ "encoding": "dict"
+ },
+ {
+ "column": "NN",
+ "encoding": "dict"
+ },
+ {
+ "column": "OO",
+ "encoding": "dict"
+ },
+ {
+ "column": "PP",
+ "encoding": "dict"
+ },
+ {
+ "column": "QQ",
+ "encoding": "dict"
+ },
+ {
+ "column": "RR",
+ "encoding": "dict"
+ },
+ {
+ "column": "SS",
+ "encoding": "dict"
+ },
+ {
+ "column": "TT",
+ "encoding": "dict"
+ },
+ {
+ "column": "UU",
+ "encoding": "dict"
+ },
+ {
+ "column": "VV",
+ "encoding": "dict"
+ },
+ {
+ "column": "WW",
+ "encoding": "dict"
+ },
+ {
+ "column": "XX",
+ "encoding": "dict"
+ },
+ {
+ "column": "YY",
+ "encoding": "dict"
+ },
+ {
+ "column": "ZZ",
+ "encoding": "dict"
+ }
+ ]
+ },
+ "signature" : null,
+ "last_modified" : 1457503036686,
+ "model_name" : "fifty_dim",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L",
"M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "AA",
"BB", "CC", "DD", "EE", "FF", "GG", "HH", "II", "JJ", "KK", "LL", "MM", "NN",
"OO", "PP", "QQ", "RR", "SS", "TT", "UU", "VV", "WW", "XX", "YY", "ZZ"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ],
+ "dim_cap" : 5
+ }
+ } ],
+ "parent_forward" : 0,
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "partition_date_start" : 694224000000,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {}
+}
