Repository: incubator-kylin Updated Branches: refs/heads/KYLIN-242-new 894ac9b8a -> bb192414f
KYLIN-242 refine AggregationGroupRuleTest Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/15d489c1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/15d489c1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/15d489c1 Branch: refs/heads/KYLIN-242-new Commit: 15d489c1ae9c625315619ead4966fb611645b91a Parents: 894ac9b Author: honma <ho...@ebay.com> Authored: Tue Dec 1 18:39:40 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Tue Dec 1 18:39:40 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/model/RowKeyDesc.java | 4 - .../cube/model/validation/ValidateContext.java | 2 +- .../rule/AggregationGroupOverlapRule.java | 85 --------- .../validation/rule/AggregationGroupRule.java | 53 ++++-- .../model/validation/rule/RowKeyAttrRule.java | 27 +-- .../kylin/cube/AggregationGroupRuleTest.java | 127 +++++++------ .../apache/kylin/cube/RowKeyAttrRuleTest.java | 45 ++--- .../src/test/resources/data/TEST1_desc.json | 183 ------------------- .../src/test/resources/data/TEST2_desc.json | 168 ----------------- .../src/test/resources/data/TEST3_desc.json | 182 ------------------ 10 files changed, 141 insertions(+), 735 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java index 87b182d..d085067 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java @@ -90,10 +90,6 @@ public class RowKeyDesc { buildRowKey(colNameAbbr); } - public RowKeyColDesc[] getRowkeyColumns() { - return rowkeyColumns; - } - public void setRowkeyColumns(RowKeyColDesc[] rowkeyColumns) { this.rowkeyColumns = rowkeyColumns; } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java index 7f1f617..632cbff 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java @@ -81,7 +81,7 @@ public class ValidateContext { */ public void print(PrintStream out) { if (results.isEmpty()) { - out.print("The element is perfect."); + out.println("The element is perfect."); } Iterator<Result> it = results.iterator(); while (it.hasNext()) { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java deleted file mode 100644 index fb78927..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupOverlapRule.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.model.validation.rule; - -import org.apache.kylin.cube.model.AggregationGroup; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.validation.IValidatorRule; -import org.apache.kylin.cube.model.validation.ResultLevel; -import org.apache.kylin.cube.model.validation.ValidateContext; - -/** - * find forbid overlaps in each AggregationGroup - * the include dims in AggregationGroup must contain all mandatory, hierarchy and joint - */ -public class AggregationGroupOverlapRule implements IValidatorRule<CubeDesc> { - - /* - * (non-Javadoc) - * - * @see - * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object - * , org.apache.kylin.metadata.validation.ValidateContext) - */ - @Override - public void validate(CubeDesc cube, ValidateContext context) { - - int index = 0; - for (AggregationGroup agg : cube.getAggregationGroups()) { - - if ((agg.getMandatoryColumnMask() & agg.getHierarchyDimsMask()) != 0) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with hierarchy dims"); - } - if ((agg.getMandatoryColumnMask() & agg.getJointDimsMask()) != 0) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with joint dims"); - } - - int jointDimNum = 0; - for (Long joint : agg.getJoints()) { - jointDimNum += Long.bitCount(joint); - if (jointDimNum < 2) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dims in a joint"); - } - - int overlapHierarchies = 0; - for (AggregationGroup.HierarchyMask mask : agg.getHierarchyMasks()) { - long share = (joint & mask.fullMask); - if (share != 0) { - overlapHierarchies++; - } - if (Long.bitCount(share) > 1) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 dim in same hierarchy"); - } - } - - if (overlapHierarchies > 1) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 hierarchies"); - } - } - - if (jointDimNum != Long.bitCount(agg.getJointDimsMask())) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than 1 joint"); - } - - index++; - } - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java index 4e7b99e..17f5b2d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java @@ -36,33 +36,39 @@ import org.apache.kylin.cube.model.validation.ValidateContext; */ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { - /* - * (non-Javadoc) - * - * @see - * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object - * , org.apache.kylin.metadata.validation.ValidateContext) - */ @Override public void validate(CubeDesc cube, ValidateContext context) { - innerValidateMaxSize(cube, context); + inner(cube, context); } - /** - * @param cube - * @param context - */ - private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) { + private int count(String[][] input) { + if (input == null) { + return 0; + } else { + int count = 0; + for (String[] x : input) { + count += count(x); + } + return count; + } + } + + private int count(String[] input) { + return input == null ? 0 : input.length; + } + + private void inner(CubeDesc cube, ValidateContext context) { int maxSize = getMaxAgrGroupSize(); int index = 0; for (AggregationGroup agg : cube.getAggregationGroups()) { if (agg.getIncludes() == null) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " include dims not set"); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " includes field not set"); continue; } if (agg.getSelectRule() == null) { + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " select rule field not set"); continue; } @@ -75,8 +81,8 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); if (agg.getSelectRule().mandatory_dims != null) { - for (String include : agg.getSelectRule().mandatory_dims) { - mandatoryDims.add(include); + for (String m : agg.getSelectRule().mandatory_dims) { + mandatoryDims.add(m); } } @@ -98,6 +104,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " Include dims not containing all the used dims"); + continue; } Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); @@ -107,17 +114,21 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { normalDims.removeAll(jointDims); int normalDimSize = normalDims.size(); - int hierarchySize = (agg.getSelectRule().hierarchy_dims == null ? 0 : agg.getSelectRule().hierarchy_dims.length); - int jointSize = agg.getSelectRule().joint_dims == null ? 0 : agg.getSelectRule().joint_dims.length; + int hierarchySize = count(agg.getSelectRule().hierarchy_dims); + int jointSize = count(agg.getSelectRule().joint_dims); + if (normalDimSize + hierarchySize + jointSize > maxSize) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " has too many dimensions"); + continue; } if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with hierarchy dims"); + continue; } if (CollectionUtils.containsAny(mandatoryDims, jointDims)) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " mandatory dims overlap with joint dims"); + continue; } int jointDimNum = 0; @@ -131,6 +142,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { if (oneJoint.size() < 2) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " require at least 2 dims in a joint"); + continue; } jointDimNum += oneJoint.size(); @@ -145,17 +157,20 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> { } if (share.size() > 1) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 dim in same hierarchy"); + continue; } } if (overlapHierarchies > 1) { context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " joint columns overlap with more than 1 hierarchies"); + continue; } } } if (jointDimNum != jointDims.size()) { - context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than 1 joint"); + context.addResult(ResultLevel.ERROR, "Aggregation group " + index + " a dim exist in more than one joint"); + continue; } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java index 7f4b6e7..932754c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java @@ -30,13 +30,6 @@ import org.apache.kylin.cube.model.validation.ValidateContext; */ public class RowKeyAttrRule implements IValidatorRule<CubeDesc> { - /* - * (non-Javadoc) - * - * @see - * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object - * , org.apache.kylin.metadata.validation.ValidateContext) - */ @Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); @@ -46,25 +39,21 @@ public class RowKeyAttrRule implements IValidatorRule<CubeDesc> { } RowKeyColDesc[] rcd = row.getRowKeyColumns(); - if (rcd == null) { + if (rcd == null || rcd.length == 0) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } - if (rcd.length == 0) { - context.addResult(ResultLevel.ERROR, "Rowkey columns is empty"); - return; - } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; - // if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) { - // context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute"); - // } - // if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) { - // context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty"); - // } + if (rd.getColumn() == null || rd.getColumn().length() == 0) { + context.addResult(ResultLevel.ERROR, "Rowkey column empty"); + continue; + } + if (!rd.getEncoding().equalsIgnoreCase("dict") && !rd.getEncoding().toLowerCase().startsWith("fixed_length")) { + context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " encoding not dict nor fixed_length"); + } } } - } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java index a2395e1..2b75d48 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java @@ -21,84 +21,107 @@ package org.apache.kylin.cube; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Arrays; + import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.validation.IValidatorRule; import org.apache.kylin.cube.model.validation.ValidateContext; import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule; -import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -/** - * @author jianliu - * - */ - -@Ignore public class AggregationGroupRuleTest { - private CubeDesc cube; - private ValidateContext vContext = new ValidateContext(); - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class); - this.cube = desc2; + @Test + public void testGoodDesc() throws IOException { + AggregationGroupRule rule = getAggregationGroupRule(); + for (File f : new File("../examples/test_case_data/localmeta/cube_desc/").listFiles()) { + CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); + ValidateContext vContext = new ValidateContext(); + rule.validate(desc, vContext); + vContext.print(System.out); + assertTrue(vContext.getResults().length == 0); + } } @Test - public void testOneMandatoryColumn() { - IValidatorRule<CubeDesc> rule = new AggregationGroupRule() { - /* - * (non-Javadoc) - * - * @see - * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule - * #getMaxAgrGroupSize() - */ + public void testGoodBecomeBadDesc() throws IOException { + AggregationGroupRule rule = new AggregationGroupRule() { @Override protected int getMaxAgrGroupSize() { - return 3; + return 4; } }; - rule.validate(cube, vContext); + + for (File f : new File("../examples/test_case_data/localmeta/cube_desc/").listFiles()) { + System.out.println(f.getName()); + CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); + ValidateContext vContext = new ValidateContext(); + rule.validate(desc, vContext); + vContext.print(System.out); + assertEquals(1, vContext.getResults().length); + assertEquals("Aggregation group 0 has too many dimensions", (vContext.getResults()[0].getMessage())); + } + } + + @Test + public void testGoodDesc2() throws IOException { + + ValidateContext vContext = new ValidateContext(); + CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); + desc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {// + new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "CATEG_LVL2_NAME" } }; + + IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); + rule.validate(desc, vContext); vContext.print(System.out); - assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2); - assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number")); - assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number")); - // assertTrue("Failed to validate aggragation group error", - // vContext.getResults()[2].getMessage() - // .startsWith("Hierachy column")); + assertEquals(0, vContext.getResults().length); } @Test - public void testAggColumnSize() { + public void testBadDesc1() throws IOException { + + ValidateContext vContext = new ValidateContext(); + CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); + String[] temp = Arrays.asList(desc.getAggregationGroups().get(0).getIncludes()).subList(0, 3).toArray(new String[3]); + + desc.getAggregationGroups().get(0).setIncludes(temp); + IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); + rule.validate(desc, vContext); + vContext.print(System.out); + // System.out.println(vContext.getResults().length); + // System.out.println(vContext.getResults()[0].getMessage()); + assertEquals(1, vContext.getResults().length); + assertEquals("Aggregation group 0 Include dims not containing all the used dims", (vContext.getResults()[0].getMessage())); + } + + @Test + public void testBadDesc2() throws IOException { + + ValidateContext vContext = new ValidateContext(); + CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); + desc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {// + new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "META_CATEG_NAME", "CATEG_LVL2_NAME" } }; + + IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); + rule.validate(desc, vContext); + vContext.print(System.out); + assertEquals(1, vContext.getResults().length); + assertEquals("Aggregation group 0 joint columns overlap with more than 1 dim in same hierarchy", (vContext.getResults()[0].getMessage())); + } + + public AggregationGroupRule getAggregationGroupRule() { AggregationGroupRule rule = new AggregationGroupRule() { - /* - * (non-Javadoc) - * - * @see - * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule - * #getMaxAgrGroupSize() - */ @Override protected int getMaxAgrGroupSize() { return 20; } }; - rule.validate(cube, vContext); - vContext.print(System.out); - assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0); - // assertTrue("Failed to validate aggragation group error", - // vContext.getResults()[0].getMessage() - // .startsWith("Aggregation group")); - // assertTrue("Failed to validate aggragation group error", - // vContext.getResults()[0].getMessage() - // .startsWith("Hierachy column")); + + return rule; } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java index 6b53df8..be7b624 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java @@ -20,41 +20,42 @@ package org.apache.kylin.cube; import static org.junit.Assert.assertTrue; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.validation.IValidatorRule; import org.apache.kylin.cube.model.validation.ValidateContext; import org.apache.kylin.cube.model.validation.rule.RowKeyAttrRule; -import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -/** - * @author jianliu - * - */ -@Ignore public class RowKeyAttrRuleTest { - private CubeDesc cube; - private ValidateContext vContext = new ValidateContext(); - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST3_desc.json"), CubeDesc.class); - this.cube = desc2; - + @Test + public void testGoodDesc() throws IOException { + for (File f : new File("../examples/test_case_data/localmeta/cube_desc/").listFiles()) { + CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); + ValidateContext vContext = new ValidateContext(); + IValidatorRule<CubeDesc> rule = new RowKeyAttrRule(); + rule.validate(desc, vContext); + vContext.print(System.out); + assertTrue(vContext.getResults().length == 0); + } } @Test - public void testOneMandatoryColumn() { + public void testBadDesc() throws IOException { + ValidateContext vContext = new ValidateContext(); + CubeDesc desc = JsonUtil.readValue(new FileInputStream("../examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); + desc.getRowkey().getRowKeyColumns()[1].setEncoding("non-supported-encoding"); + desc.getRowkey().getRowKeyColumns()[2].setColumn(""); IValidatorRule<CubeDesc> rule = new RowKeyAttrRule(); - rule.validate(cube, vContext); + rule.validate(desc, vContext); vContext.print(System.out); - assertTrue("Failed to validate rowkey", vContext.getResults().length == 1); - assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("Rowkey column")); + assertTrue(vContext.getResults().length == 2); + assertTrue("Rowkey column cal_dt encoding not dict nor fixed_length".equals(vContext.getResults()[0].getMessage())); + assertTrue("Rowkey column empty".equalsIgnoreCase(vContext.getResults()[1].getMessage())); } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/test/resources/data/TEST1_desc.json ---------------------------------------------------------------------- diff --git a/core-cube/src/test/resources/data/TEST1_desc.json b/core-cube/src/test/resources/data/TEST1_desc.json deleted file mode 100644 index 082a3cf..0000000 --- a/core-cube/src/test/resources/data/TEST1_desc.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", - "last_modified" : 1401429176099, - "name" : "TEST1_desc", - "model_name": "TEST1_model_desc", - "fact_table" : "TEST_KYLIN_FACT", - "dimensions" : [ { - "id" : 1, - "name" : "CAL_DT", - "join" : { - "type" : "inner", - "primary_key" : [ "CAL_DT" ], - "foreign_key" : [ "CAL_DT" ] - }, - "hierarchy" : null, - "table" : "TEST_CAL_DT", - "column" : ["TEST_CAL_DT.CAL_DT"], - "derived" : [ "WEEK_BEG_DT" ] - }, { - "id" : 2, - "name" : "CATEGORY", - "join" : { - "type" : "inner", - "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], - "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] - }, - "hierarchy" : true, - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"], - "datatype" : null, - "derived" : null - }, { - "id" : 3, - "name" : "LSTG_FORMAT_NAME", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"], - "datatype" : "string", - "derived" : null - }, { - "id" : 4, - "name" : "SITE_ID", - "join" : { - "type" : "inner", - "primary_key" : [ "SITE_ID" ], - "foreign_key" : [ "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "table" : "TEST_SITES", - "column" : ["TEST_SITES.SITE_ID"], - "datatype" : "string", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "id" : 5, - "name" : "SELLER_TYPE_CD", - "join" : { - "type" : "inner", - "primary_key" : [ "SELLER_TYPE_CD" ], - "foreign_key" : [ "SLR_SEGMENT_CD" ] - }, - "hierarchy" : null, - "table" : "TEST_SELLER_TYPE_DIM", - "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"], - "datatype" : "string", - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "id" : 1, - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 2, - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 3, - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 4, - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "long" - } - }, { - "id" : 5, - "name" : "SELLER_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "SELLER_ID" - }, - "returntype" : "hllc10" - } - }, { - "id" : 6, - "name" : "SELLER_FORMAT_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "LSTG_FORMAT_NAME,SELLER_ID" - }, - "returntype" : "hllc10" - } - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "CAL_DT", - "length" : 0, - "dictionary" : "date(yyyy-mm-dd)", - "mandatory" : true - }, { - "column" : "META_CATEG_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL2_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL3_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "LSTG_FORMAT_NAME", - "length" : 12, - "dictionary" : null, - "mandatory" : false - }, { - "column" : "SITE_ID", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "SELLER_TYPE_CD", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - } ], - "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ] - }, - "hbase_mapping" : { - "column_family" : [ { - "name" : "F1", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] - } ] - } ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/test/resources/data/TEST2_desc.json ---------------------------------------------------------------------- diff --git a/core-cube/src/test/resources/data/TEST2_desc.json b/core-cube/src/test/resources/data/TEST2_desc.json deleted file mode 100644 index 90bd806..0000000 --- a/core-cube/src/test/resources/data/TEST2_desc.json +++ /dev/null @@ -1,168 +0,0 @@ -{ - "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", - "last_modified" : 1401429176099, - "model_name": "TEST2_model_desc", - "name" : "TEST2_desc", - "dimensions" : [ { - "name" : "CAL_DT", - "hierarchy" : null, - "column" : ["TEST_CAL_DT.CAL_DT"], - "derived" : [ "WEEK_BEG_DT" ] - }, { - "id" : 2, - "name" : "CATEGORY", - "join" : { - "type" : "inner", - "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], - "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] - }, - "hierarchy" : true, - "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"], - "derived" : null - }, { - "id" : 3, - "name" : "LSTG_FORMAT_NAME", - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"], - "derived" : null - }, { - "id" : 4, - "name" : "SITE_ID", - "join" : { - "type" : "inner", - "primary_key" : [ "SITE_ID" ], - "foreign_key" : [ "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "column" : ["TEST_SITES.SITE_ID"], - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "id" : 5, - "name" : "SELLER_TYPE_CD", - "join" : { - "type" : "inner", - "primary_key" : [ "SELLER_TYPE_CD" ], - "foreign_key" : [ "SLR_SEGMENT_CD" ] - }, - "hierarchy" : null, - "table" : "TEST_SELLER_TYPE_DIM", - "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"], - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "id" : 1, - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 2, - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 3, - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 4, - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "long" - } - }, { - "id" : 5, - "name" : "SELLER_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "SELLER_ID" - }, - "returntype" : "hllc10" - } - }, { - "id" : 6, - "name" : "SELLER_FORMAT_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "LSTG_FORMAT_NAME,SELLER_ID" - }, - "returntype" : "hllc10" - } - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "CAL_DT", - "length" : 0, - "dictionary" : "date(yyyy-mm-dd)", - "mandatory" : true - }, { - "column" : "META_CATEG_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL2_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL3_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "LSTG_FORMAT_NAME", - "length" : 12, - "dictionary" : null, - "mandatory" : false - }, { - "column" : "SITE_ID", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "SELLER_TYPE_CD", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - } ], - "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD", "CATEG_LVL2_NAME" ] ] - }, - "hbase_mapping" : { - "column_family" : [ { - "name" : "F1", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] - } ] - } ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/15d489c1/core-cube/src/test/resources/data/TEST3_desc.json ---------------------------------------------------------------------- diff --git a/core-cube/src/test/resources/data/TEST3_desc.json b/core-cube/src/test/resources/data/TEST3_desc.json deleted file mode 100644 index 4b0836b..0000000 --- a/core-cube/src/test/resources/data/TEST3_desc.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", - "last_modified" : 1401429176099, - "name" : "TEST1_desc", - "model_name": "TEST1_model_desc", - "dimensions" : [ { - "id" : 1, - "name" : "CAL_DT", - "join" : { - "type" : "inner", - "primary_key" : [ "CAL_DT" ], - "foreign_key" : [ "CAL_DT" ] - }, - "hierarchy" : null, - "table" : "TEST_CAL_DT", - "column" : ["TEST_CAL_DT.CAL_DT"], - "datatype" : "date", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "id" : 2, - "name" : "CATEGORY", - "join" : { - "type" : "inner", - "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], - "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] - }, - "hierarchy" : true, - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"], - "derived" : null - }, { - "id" : 3, - "name" : "LSTG_FORMAT_NAME", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"], - "datatype" : "string", - "derived" : null - }, { - "id" : 4, - "name" : "SITE_ID", - "join" : { - "type" : "inner", - "primary_key" : [ "SITE_ID" ], - "foreign_key" : [ "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "table" : "TEST_SITES", - "column" : ["TEST_SITES.SITE_ID"], - "datatype" : "string", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "id" : 5, - "name" : "SELLER_TYPE_CD", - "join" : { - "type" : "inner", - "primary_key" : [ "SELLER_TYPE_CD" ], - "foreign_key" : [ "SLR_SEGMENT_CD" ] - }, - "hierarchy" : null, - "table" : "TEST_SELLER_TYPE_DIM", - "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"], - "datatype" : "string", - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "id" : 1, - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 2, - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 3, - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 4, - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "long" - } - }, { - "id" : 5, - "name" : "SELLER_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "SELLER_ID" - }, - "returntype" : "hllc10" - } - }, { - "id" : 6, - "name" : "SELLER_FORMAT_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "LSTG_FORMAT_NAME,SELLER_ID" - }, - "returntype" : "hllc10" - } - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "CAL_DT", - "length" : 10, - "dictionary" : "date(yyyy-mm-dd)", - "mandatory" : false - }, { - "column" : "META_CATEG_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL2_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL3_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "LSTG_FORMAT_NAME", - "length" : 12, - "dictionary" : null, - "mandatory" : false - }, { - "column" : "SITE_ID", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "SELLER_TYPE_CD", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - } ], - "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ] - }, - "hbase_mapping" : { - "column_family" : [ { - "name" : "F1", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] - } ] - } ] - } -} \ No newline at end of file