Repository: kylin Updated Branches: refs/heads/KYLIN-2294 614b6ecb9 -> 8787bb56a
model ready, pending build and test query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8787bb56 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8787bb56 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8787bb56 Branch: refs/heads/KYLIN-2294 Commit: 8787bb56a7b6f0366a82d2eac1cf3753aa949cbd Parents: 614b6ec Author: Li Yang <liy...@apache.org> Authored: Tue Dec 20 18:27:11 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Tue Dec 20 18:27:11 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 5 - .../kylin/common/persistence/ResourceStore.java | 2 +- .../org/apache/kylin/cube/CubeDescTest.java | 8 + .../source/datagen/ModelDataGenerator.java | 31 ++- .../kylin/metadata/MetadataManagerTest.java | 7 + .../localmeta/cube_desc/ci_inner_join_cube.json | 206 +++++++++---------- .../localmeta/cube_desc/ci_left_join_cube.json | 206 +++++++++---------- .../model_desc/ci_left_join_model.json | 4 +- .../table/DEFAULT.TEST_KYLIN_FACT.json | 10 + 9 files changed, 255 insertions(+), 224 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 5153562..1efad16 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -492,11 +492,6 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.source.hive.beeline-params", ""); } - @Deprecated - public String getCreateFlatHiveTableMethod() { - return getOptional("kylin.source.hive.create-flat-table-method", "1"); - } - // ============================================================================ // STORAGE.HBASE // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index 466d9d2..0580576 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -109,7 +109,7 @@ abstract public class ResourceStore { final protected KylinConfig kylinConfig; - public ResourceStore(KylinConfig kylinConfig) { + protected ResourceStore(KylinConfig kylinConfig) { this.kylinConfig = kylinConfig; } http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index 1daa748..fb003bc 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -41,6 +41,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.Maps; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** * @author yangli9 @@ -91,6 +92,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase { } @Test + public void testCiCube() { + CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig()); + assertNotNull(mgr.getCubeDesc("ci_left_join_cube")); + assertNotNull(mgr.getCubeDesc("ci_inner_join_cube")); + } + + @Test public void testGoodInit() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.init(getTestConfig()); http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java index 60f2736..f1e6d9f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java @@ -21,6 +21,7 @@ package org.apache.kylin.source.datagen; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; @@ -34,8 +35,10 @@ import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; @@ -56,10 +59,14 @@ public class ModelDataGenerator { boolean outprint = false; // for debug public ModelDataGenerator(DataModelDesc model, int nRows) { - this(model, nRows, ResourceStore.getStore(model.getConfig()), "/data"); + this(model, nRows, ResourceStore.getStore(model.getConfig())); } - public ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore, String outputPath) { + private ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore) { + this(model, nRows, outputStore, "/data"); + } + + private ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore, String outputPath) { this.model = model; this.targetRows = nRows; this.outputStore = outputStore; @@ -268,8 +275,8 @@ public class ModelDataGenerator { } private void saveResource(byte[] content, String path) throws IOException { + System.out.println("Generated " + outputStore.getReadableResourcePath(path)); if (outprint) { - System.out.println("Generated " + path); System.out.println(Bytes.toString(content)); } outputStore.putResource(path, new ByteArrayInputStream(content), System.currentTimeMillis()); @@ -287,4 +294,22 @@ public class ModelDataGenerator { return model; } + public static void main(String[] args) throws IOException { + String modelName = args[0]; + int nRows = Integer.parseInt(args[1]); + String outputDir = args.length > 2 ? args[2] : null; + + KylinConfig conf = KylinConfig.getInstanceFromEnv(); + DataModelDesc model = MetadataManager.getInstance(conf).getDataModelDesc(modelName); + ResourceStore store = outputDir == null ? ResourceStore.getStore(conf) : ResourceStore.getStore(mockup(outputDir)); + + ModelDataGenerator gen = new ModelDataGenerator(model, nRows, store); + gen.generate(); + } + + private static KylinConfig mockup(String outputDir) { + KylinConfig mockup = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv()); + mockup.setMetadataUrl(new File(outputDir).getAbsolutePath()); + return mockup; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java index 204b4e0..a27f4cf 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java @@ -46,6 +46,13 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { public void after() throws Exception { this.cleanupTestMetadata(); } + + @Test + public void testCiModel() { + MetadataManager mgr = getInstance(getTestConfig()); + Assert.assertNotNull(mgr.getDataModelDesc("ci_left_join_model")); + Assert.assertNotNull(mgr.getDataModelDesc("ci_inner_join_model")); + } @Test public void testListAllTables() throws Exception { http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json index e9a38f3..3df5993 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json @@ -4,6 +4,10 @@ "model_name" : "ci_inner_join_model", "description" : null, "dimensions" : [ { + "name" : "ORDER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "ORDER_ID" + }, { "name" : "CAL_DT", "table" : "EDW.TEST_CAL_DT", "column" : "{FK}", @@ -43,93 +47,74 @@ "table" : "EDW.TEST_SELLER_TYPE_DIM", "column" : "{FK}", "derived" : [ "SELLER_TYPE_DESC" ] + }, { + "name" : "SELLER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "SELLER_ID", + "derived" : null } ], "measures" : [ { - "name" : "GMV_SUM", + "name" : "TRANS_CNT", "function" : { - "expression" : "SUM", + "expression" : "COUNT", "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "type" : "constant", + "value" : "1" }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null + "returntype" : "bigint" + } }, { - "name" : "GMV_MIN", + "name" : "ITEM_COUNT_SUM", "function" : { - "expression" : "MIN", + "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "value" : "ITEM_COUNT" }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null + "returntype" : "bigint" + } }, { - "name" : "GMV_MAX", + "name" : "GMV_SUM", "function" : { - "expression" : "MAX", + "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "value" : "PRICE" }, "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1", - "next_parameter" : null - }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null + } }, { - "name" : "ITEM_COUNT_SUM", + "name" : "GMV_MIN", "function" : { - "expression" : "SUM", + "expression" : "MIN", "parameter" : { "type" : "column", - "value" : "ITEM_COUNT", - "next_parameter" : null + "value" : "PRICE" }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null + "returntype" : "decimal(19,4)" + } }, { - "name" : "SELLER_CNT_BITMAP", + "name" : "GMV_MAX", "function" : { - "expression" : "COUNT_DISTINCT", + "expression" : "MAX", "parameter" : { "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null + "value" : "PRICE" }, - "returntype" : "bitmap" - }, - "dependent_measure_ref" : null + "returntype" : "decimal(19,4)" + } }, { - "name" : "USER_COUNT_BITMAP", + "name" : "SELLER_HLL", "function" : { "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "ORDER_ID", - "next_parameter" : null + "value" : "SELLER_ID" }, - "returntype" : "bitmap" - }, - "dependent_measure_ref" : null + "returntype" : "hllc(10)" + } }, { - "name" : "SELLER_FORMAT_CNT", + "name" : "SELLER_FORMAT_HLL", "function" : { "expression" : "COUNT_DISTINCT", "parameter" : { @@ -137,13 +122,11 @@ "value" : "LSTG_FORMAT_NAME", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null + "value" : "SELLER_ID" } }, "returntype" : "hllc(10)" - }, - "dependent_measure_ref" : null + } }, { "name" : "TOP_SELLER", "function" : { @@ -153,91 +136,101 @@ "value" : "PRICE", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null + "value" : "SELLER_ID" } }, "returntype" : "topn(100)", "configuration": {"topn.encoding.SELLER_ID" : "int:4"} - }, - "dependent_measure_ref" : null + } }, { - "name" : "CAL_DT_RAW", + "name" : "TEST_COUNT_DISTINCT_BITMAP", "function" : { - "expression" : "RAW", + "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "CAL_DT", - "next_parameter" : null + "value" : "TEST_COUNT_DISTINCT_BITMAP" }, - "returntype" : "raw" - }, - "dependent_measure_ref" : null + "returntype" : "bitmap" + } + }, { + "name" : "TEST_EXTENDED_COLUMN", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } }, { - "name" : "LSTG_FORMAT_NAME_RAW", + "name" : "TRANS_ID_RAW", "function" : { "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "LSTG_FORMAT_NAME", - "next_parameter" : null + "value" : "TRANS_ID" }, "returntype" : "raw" - }, - "dependent_measure_ref" : null + } }, { - "name" : "LEAF_CATEG_ID_RAW", + "name" : "PRICE_RAW", "function" : { "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "LEAF_CATEG_ID", - "next_parameter" : null + "value" : "PRICE" }, "returntype" : "raw" - }, - "dependent_measure_ref" : null + } }, { - "name" : "PRICE_RAW", + "name" : "ITEM_COUNT_RAW", "function" : { "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "value" : "ITEM_COUNT" }, "returntype" : "raw" - }, - "dependent_measure_ref" : null + } } ], "dictionaries": [ { - "column": "ORDER_ID", + "column": "TEST_COUNT_DISTINCT_BITMAP", "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" } ], "rowkey" : { "rowkey_columns" : [ { - "column" : "cal_dt", - "encoding" : "dict" + "column" : "SELLER_ID", + "encoding" : "int:4" }, { - "column" : "leaf_categ_id", + "column" : "ORDER_ID", + "encoding" : "int:4" + }, { + "column" : "CAL_DT", "encoding" : "dict" }, { - "column" : "meta_categ_name", + "column" : "LEAF_CATEG_ID", "encoding" : "dict" }, { - "column" : "categ_lvl2_name", + "column" : "META_CATEG_NAME", "encoding" : "dict" }, { - "column" : "categ_lvl3_name", + "column" : "CATEG_LVL2_NAME", "encoding" : "dict" }, { - "column" : "lstg_format_name", + "column" : "CATEG_LVL3_NAME", "encoding" : "dict" }, { - "column" : "lstg_site_id", + "column" : "LSTG_FORMAT_NAME", + "encoding" : "fixed_length:12" + }, { + "column" : "LSTG_SITE_ID", "encoding" : "dict" }, { - "column" : "slr_segment_cd", + "column" : "SLR_SEGMENT_CD", "encoding" : "dict" } ] }, @@ -249,35 +242,35 @@ "name" : "f1", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum", "CAL_DT_RAW", "LSTG_FORMAT_NAME_RAW", "LEAF_CATEG_ID_RAW", "PRICE_RAW" ] + "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ] } ] }, { "name" : "f2", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "seller_cnt_bitmap", "user_count_bitmap", "seller_format_cnt"] + "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ] } ] }, { "name" : "f3", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "top_seller" ] + "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "ITEM_COUNT_RAW" ] } ] } ] }, "aggregation_groups" : [ { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"], + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ], "select_rule" : { - "hierarchy_dims" : [ ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ] + "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], + "mandatory_dims" : [ ], + "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] } }, { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ], + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "ORDER_ID" ], "select_rule" : { - "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ ] + "hierarchy_dims" : [ ], + "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID" ], + "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] } } ], "notify_list" : null, @@ -287,8 +280,7 @@ "engine_type" : 2, "storage_type" : 2, "override_kylin_properties": { - "kylin.job.cubing.inmem.sampling.hll.precision": "16", - "kylin.hive.create.flat.table.method": "2" + "kylin.cube.algorithm": "LAYER" }, "partition_date_start": 0 } http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json index d85c6d8..b9647ea 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json @@ -4,6 +4,10 @@ "model_name" : "ci_left_join_model", "description" : null, "dimensions" : [ { + "name" : "ORDER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "ORDER_ID" + }, { "name" : "CAL_DT", "table" : "EDW.TEST_CAL_DT", "column" : "{FK}", @@ -43,93 +47,74 @@ "table" : "EDW.TEST_SELLER_TYPE_DIM", "column" : "{FK}", "derived" : [ "SELLER_TYPE_DESC" ] + }, { + "name" : "SELLER_ID", + "table" : "DEFAULT.TEST_KYLIN_FACT", + "column" : "SELLER_ID", + "derived" : null } ], "measures" : [ { - "name" : "GMV_SUM", + "name" : "TRANS_CNT", "function" : { - "expression" : "SUM", + "expression" : "COUNT", "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "type" : "constant", + "value" : "1" }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null + "returntype" : "bigint" + } }, { - "name" : "GMV_MIN", + "name" : "ITEM_COUNT_SUM", "function" : { - "expression" : "MIN", + "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "value" : "ITEM_COUNT" }, - "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null + "returntype" : "bigint" + } }, { - "name" : "GMV_MAX", + "name" : "GMV_SUM", "function" : { - "expression" : "MAX", + "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "value" : "PRICE" }, "returntype" : "decimal(19,4)" - }, - "dependent_measure_ref" : null - }, { - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1", - "next_parameter" : null - }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null + } }, { - "name" : "ITEM_COUNT_SUM", + "name" : "GMV_MIN", "function" : { - "expression" : "SUM", + "expression" : "MIN", "parameter" : { "type" : "column", - "value" : "ITEM_COUNT", - "next_parameter" : null + "value" : "PRICE" }, - "returntype" : "bigint" - }, - "dependent_measure_ref" : null + "returntype" : "decimal(19,4)" + } }, { - "name" : "SELLER_CNT_BITMAP", + "name" : "GMV_MAX", "function" : { - "expression" : "COUNT_DISTINCT", + "expression" : "MAX", "parameter" : { "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null + "value" : "PRICE" }, - "returntype" : "bitmap" - }, - "dependent_measure_ref" : null + "returntype" : "decimal(19,4)" + } }, { - "name" : "USER_COUNT_BITMAP", + "name" : "SELLER_HLL", "function" : { "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "ORDER_ID", - "next_parameter" : null + "value" : "SELLER_ID" }, - "returntype" : "bitmap" - }, - "dependent_measure_ref" : null + "returntype" : "hllc(10)" + } }, { - "name" : "SELLER_FORMAT_CNT", + "name" : "SELLER_FORMAT_HLL", "function" : { "expression" : "COUNT_DISTINCT", "parameter" : { @@ -137,13 +122,11 @@ "value" : "LSTG_FORMAT_NAME", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null + "value" : "SELLER_ID" } }, "returntype" : "hllc(10)" - }, - "dependent_measure_ref" : null + } }, { "name" : "TOP_SELLER", "function" : { @@ -153,91 +136,101 @@ "value" : "PRICE", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null + "value" : "SELLER_ID" } }, "returntype" : "topn(100)", "configuration": {"topn.encoding.SELLER_ID" : "int:4"} - }, - "dependent_measure_ref" : null + } }, { - "name" : "CAL_DT_RAW", + "name" : "TEST_COUNT_DISTINCT_BITMAP", "function" : { - "expression" : "RAW", + "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "CAL_DT", - "next_parameter" : null + "value" : "TEST_COUNT_DISTINCT_BITMAP" }, - "returntype" : "raw" - }, - "dependent_measure_ref" : null + "returntype" : "bitmap" + } + }, { + "name" : "TEST_EXTENDED_COLUMN", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } }, { - "name" : "LSTG_FORMAT_NAME_RAW", + "name" : "TRANS_ID_RAW", "function" : { "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "LSTG_FORMAT_NAME", - "next_parameter" : null + "value" : "TRANS_ID" }, "returntype" : "raw" - }, - "dependent_measure_ref" : null + } }, { - "name" : "LEAF_CATEG_ID_RAW", + "name" : "PRICE_RAW", "function" : { "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "LEAF_CATEG_ID", - "next_parameter" : null + "value" : "PRICE" }, "returntype" : "raw" - }, - "dependent_measure_ref" : null + } }, { - "name" : "PRICE_RAW", + "name" : "ITEM_COUNT_RAW", "function" : { "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "PRICE", - "next_parameter" : null + "value" : "ITEM_COUNT" }, "returntype" : "raw" - }, - "dependent_measure_ref" : null + } } ], "dictionaries": [ { - "column": "ORDER_ID", + "column": "TEST_COUNT_DISTINCT_BITMAP", "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" } ], "rowkey" : { "rowkey_columns" : [ { - "column" : "cal_dt", - "encoding" : "dict" + "column" : "SELLER_ID", + "encoding" : "int:4" }, { - "column" : "leaf_categ_id", + "column" : "ORDER_ID", + "encoding" : "int:4" + }, { + "column" : "CAL_DT", "encoding" : "dict" }, { - "column" : "meta_categ_name", + "column" : "LEAF_CATEG_ID", "encoding" : "dict" }, { - "column" : "categ_lvl2_name", + "column" : "META_CATEG_NAME", "encoding" : "dict" }, { - "column" : "categ_lvl3_name", + "column" : "CATEG_LVL2_NAME", "encoding" : "dict" }, { - "column" : "lstg_format_name", + "column" : "CATEG_LVL3_NAME", "encoding" : "dict" }, { - "column" : "lstg_site_id", + "column" : "LSTG_FORMAT_NAME", + "encoding" : "fixed_length:12" + }, { + "column" : "LSTG_SITE_ID", "encoding" : "dict" }, { - "column" : "slr_segment_cd", + "column" : "SLR_SEGMENT_CD", "encoding" : "dict" } ] }, @@ -249,35 +242,35 @@ "name" : "f1", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum", "CAL_DT_RAW", "LSTG_FORMAT_NAME_RAW", "LEAF_CATEG_ID_RAW", "PRICE_RAW" ] + "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ] } ] }, { "name" : "f2", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "seller_cnt_bitmap", "user_count_bitmap", "seller_format_cnt"] + "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ] } ] }, { "name" : "f3", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "top_seller" ] + "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "ITEM_COUNT_RAW" ] } ] } ] }, "aggregation_groups" : [ { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"], + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ], "select_rule" : { - "hierarchy_dims" : [ ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ] + "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], + "mandatory_dims" : [ ], + "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] } }, { - "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ], + "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "ORDER_ID" ], "select_rule" : { - "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ], - "mandatory_dims" : [ "cal_dt" ], - "joint_dims" : [ ] + "hierarchy_dims" : [ ], + "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID" ], + "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] } } ], "notify_list" : null, @@ -287,8 +280,7 @@ "engine_type" : 2, "storage_type" : 2, "override_kylin_properties": { - "kylin.job.cubing.inmem.sampling.hll.precision": "16", - "kylin.hive.create.flat.table.method": "2" + "kylin.cube.algorithm": "INMEM" }, "partition_date_start": 0 } http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json index 2913eed..a3088ec 100644 --- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json +++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json @@ -78,7 +78,9 @@ "LSTG_SITE_ID", "LEAF_CATEG_ID", "SLR_SEGMENT_CD", - "SELLER_ID" + "SELLER_ID", + "TEST_EXTENDED_COLUMN", + "TEST_COUNT_DISTINCT_BITMAP" ] }, { http://git-wip-us.apache.org/repos/asf/kylin/blob/8787bb56/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json index 45ebf83..1aafaca 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json @@ -50,6 +50,16 @@ "name" : "ITEM_COUNT", "datatype" : "int", "data_gen" : "RAND" + }, { + "id" : "11", + "name" : "TEST_EXTENDED_COLUMN", + "datatype" : "string", + "data_gen" : "RAND" + }, { + "id" : "12", + "name" : "TEST_COUNT_DISTINCT_BITMAP", + "datatype" : "string", + "data_gen" : "RAND" } ], "database" : "DEFAULT", "last_modified" : 0
