KYLIN-2314 Use col identity instead of col name in dictionary refs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6c67197 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6c67197 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6c67197 Branch: refs/heads/master Commit: c6c6719730e05c66c14407cfd7926a0a54a6009a Parents: 30d3763 Author: Li Yang <liy...@apache.org> Authored: Fri Dec 23 18:06:58 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sat Dec 24 14:46:39 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/DeployUtil.java | 26 +- .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../java/org/apache/kylin/cube/CubeSegment.java | 30 +- .../org/apache/kylin/cube/CubeDescTest.java | 40 +- .../kylin/metadata/MetadataManagerTest.java | 67 ++- .../kylin/metadata/model/JoinsTreeTest.java | 11 +- .../kylin/engine/mr/JobBuilderSupport.java | 2 - .../engine/mr/steps/CreateDictionaryJob.java | 5 +- .../engine/mr/steps/CubingExecutableUtil.java | 8 - .../engine/mr/steps/FactDistinctColumnsJob.java | 3 + .../mr/steps/FactDistinctColumnsReducer.java | 39 +- .../mr/steps/FactDistinctHiveColumnsMapper.java | 81 ++-- .../mr/steps/UpdateCubeInfoAfterBuildStep.java | 4 +- .../mr/steps/UpdateCubeInfoAfterMergeStep.java | 1 - .../localmeta/cube_desc/ci_inner_join_cube.json | 248 ++++++++--- .../localmeta/cube_desc/ci_left_join_cube.json | 248 ++++++++--- .../test_kylin_snowflake_sales_cube.json | 268 ------------ .../localmeta/data/DEFAULT.TEST_COUNTRY.csv | 244 +++++++++++ .../model_desc/ci_inner_join_model.json | 100 ++++- .../model_desc/ci_left_join_model.json | 100 ++++- .../test_kylin_snowflake_sales_model.json | 87 ---- .../localmeta/table/DEFAULT.TEST_ACCOUNT.json | 34 ++ .../localmeta/table/DEFAULT.TEST_COUNTRY.json | 24 ++ .../table/DEFAULT.TEST_KYLIN_COUNTRY.json | 24 -- .../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json | 28 -- .../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 408 ------------------- .../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 152 ------- .../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 24 -- .../localmeta/table/SNOWTEST.KYLIN_SALES.json | 56 --- .../java/org/apache/kylin/query/H2Database.java | 11 +- 30 files changed, 1080 insertions(+), 1295 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index 564a4e2..089c3ed 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -125,11 +125,15 @@ public class DeployUtil { static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings"; static final String TABLE_KYLIN_FACT = "default.test_kylin_fact"; static final String TABLE_ORDER = "default.test_order"; + static final String TABLE_ACCOUNT = "default.test_account"; + static final String TABLE_COUNTRY = "default.test_country"; static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim"; static final String TABLE_SELLER_TYPE_DIM_TABLE = "edw.test_seller_type_dim_table"; static final String TABLE_SITES = "edw.test_sites"; - static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES }; + static final String[] TABLE_NAMES = new String[] { // + TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, // + TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES, TABLE_ACCOUNT, TABLE_COUNTRY }; public static void prepareTestDataForNormalCubes(String modelName) throws Exception { @@ -228,21 +232,15 @@ public class DeployUtil { IHiveClient hiveClient = HiveClientFactory.getHiveClient(); // create hive tables hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW"); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase()))); - hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase()))); + for (String tablename : TABLE_NAMES) { + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(tablename.toUpperCase()))); + } // load data to hive tables // LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename - hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_ORDER, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM_TABLE, tableFileDir)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir)); + for (String tablename : TABLE_NAMES) { + hiveClient.executeHQL(generateLoadDataHql(tablename.toUpperCase(), tableFileDir)); + } final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatements(generateCreateViewHql(VIEW_SELLER_TYPE_DIM, TABLE_SELLER_TYPE_DIM_TABLE)); @@ -251,7 +249,7 @@ public class DeployUtil { } private static String generateLoadDataHql(String tableName, String tableFileDir) { - return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName.toUpperCase() + ".csv' OVERWRITE INTO TABLE " + tableName.toUpperCase(); + return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName + ".csv' OVERWRITE INTO TABLE " + tableName; } private static String[] generateCreateTableHql(TableDesc tableDesc) { http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index fe2030a..9670b89 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -240,7 +240,7 @@ public class CubeManager implements IRealizationProvider { if (dictInfo != null) { Dictionary<?> dict = dictInfo.getDictionaryObject(); cubeSeg.putDictResPath(col, dictInfo.getResourcePath()); - cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), dict.getSize(), dict.getSizeOfId() }); + cubeSeg.getRowkeyStats().add(new Object[] { col.getIdentity(), dict.getSize(), dict.getSizeOfId() }); CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance()); update.setToUpdateSegs(cubeSeg); http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index e155f86..36a6044 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -98,9 +98,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen @JsonProperty("snapshots") private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path - @JsonProperty("index_path") - private String indexPath; - @JsonProperty("rowkey_stats") private List<Object[]> rowkeyStats = Lists.newArrayList(); @@ -296,15 +293,22 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen } public String getDictResPath(TblColRef col) { - return getDictionaries().get(dictKey(col)); + String r; + String dictKey = col.getIdentity(); + r = getDictionaries().get(dictKey); + + // try Kylin v1.x dict key as well + if (r == null) { + String v1DictKey = col.getTable() + "/" + col.getName(); + r = getDictionaries().get(v1DictKey); + } + + return r; } public void putDictResPath(TblColRef col, String dictResPath) { - getDictionaries().put(dictKey(col), dictResPath); - } - - private String dictKey(TblColRef col) { - return col.getTable() + "/" + col.getName(); + String dictKey = col.getIdentity(); + getDictionaries().put(dictKey, dictResPath); } public void setStorageLocationIdentifier(String storageLocationIdentifier) { @@ -523,14 +527,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen return cubeInstance; } - public String getIndexPath() { - return indexPath; - } - - public void setIndexPath(String indexPath) { - this.indexPath = indexPath; - } - public Map<String, String> getAdditionalInfo() { return additionalInfo; } http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java index fb003bc..39b6433 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java @@ -30,7 +30,9 @@ import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.model.AggregationGroup; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.DimensionDesc; import org.apache.kylin.cube.model.SelectRule; +import org.apache.kylin.metadata.model.MeasureDesc; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -40,8 +42,7 @@ import org.junit.rules.ExpectedException; import com.google.common.collect.Maps; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; /** * @author yangli9 @@ -94,8 +95,39 @@ public class CubeDescTest extends LocalFileMetadataTestCase { @Test public void testCiCube() { CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig()); - assertNotNull(mgr.getCubeDesc("ci_left_join_cube")); - assertNotNull(mgr.getCubeDesc("ci_inner_join_cube")); + CubeDesc lc = mgr.getCubeDesc("ci_left_join_cube"); + CubeDesc ic = mgr.getCubeDesc("ci_inner_join_cube"); + assertNotNull(lc); + assertNotNull(ic); + + // assert the two CI cubes are identical apart from the left/inner difference + assertEquals(lc.getDimensions().size(), ic.getDimensions().size()); + for (int i = 0, n = lc.getDimensions().size(); i < n; i++) { + DimensionDesc ld = lc.getDimensions().get(i); + DimensionDesc id = ic.getDimensions().get(i); + assertEquals(ld.getTable(), id.getTable()); + assertEquals(ld.getColumn(), id.getColumn()); + assertArrayEquals(ld.getDerived(), id.getDerived()); + } + + assertEquals(lc.getMeasures().size(), ic.getMeasures().size()); + for (int i = 0, n = lc.getMeasures().size(); i < n; i++) { + MeasureDesc lm = lc.getMeasures().get(i); + MeasureDesc im = ic.getMeasures().get(i); + assertEquals(lm.getName(), im.getName()); + assertEquals(lm.getFunction().getFullExpression(), im.getFunction().getFullExpression()); + assertEquals(lm.getFunction().getReturnType(), im.getFunction().getReturnType()); + } + + assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size()); + for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) { + AggregationGroup lag = lc.getAggregationGroups().get(i); + AggregationGroup iag = ic.getAggregationGroups().get(i); + assertArrayEquals(lag.getIncludes(), iag.getIncludes()); + assertArrayEquals(lag.getSelectRule().mandatory_dims, iag.getSelectRule().mandatory_dims); + assertArrayEquals(lag.getSelectRule().hierarchy_dims, iag.getSelectRule().hierarchy_dims); + assertArrayEquals(lag.getSelectRule().joint_dims, iag.getSelectRule().joint_dims); + } } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java index a27f4cf..b372be2 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java @@ -19,6 +19,7 @@ package org.apache.kylin.metadata; import static org.apache.kylin.metadata.MetadataManager.getInstance; +import static org.junit.Assert.*; import java.io.IOException; import java.util.ArrayList; @@ -26,6 +27,8 @@ import java.util.List; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.JoinTableDesc; +import org.apache.kylin.metadata.model.ModelDimensionDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TableExtDesc; import org.junit.After; @@ -50,8 +53,50 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { @Test public void testCiModel() { MetadataManager mgr = getInstance(getTestConfig()); - Assert.assertNotNull(mgr.getDataModelDesc("ci_left_join_model")); - Assert.assertNotNull(mgr.getDataModelDesc("ci_inner_join_model")); + DataModelDesc lm = mgr.getDataModelDesc("ci_left_join_model"); + DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model"); + assertSnowflakeQuality(lm); + assertSnowflakeQuality(im); + + // check inner/left models are identical apart from the left/inner difference + assertEquals(lm.getJoinTables().length, im.getJoinTables().length); + for (int i = 0, n = im.getJoinTables().length; i < n; i++) { + JoinTableDesc lt = lm.getJoinTables()[i]; + JoinTableDesc it = im.getJoinTables()[i]; + assertEquals(lt.getAlias(), it.getAlias()); + assertEquals(lt.getKind(), it.getKind()); + assertEquals(lt.getTable(), it.getTable()); + assertArrayEquals(lt.getJoin().getForeignKey(), it.getJoin().getForeignKey()); + assertArrayEquals(lt.getJoin().getPrimaryKey(), it.getJoin().getPrimaryKey()); + assertTrue(lt.getJoin().isLeftJoin()); + assertTrue(it.getJoin().isInnerJoin()); + } + + assertEquals(lm.getDimensions().size(), im.getDimensions().size()); + for (int i = 0, n = im.getDimensions().size(); i < n; i++) { + ModelDimensionDesc ld = lm.getDimensions().get(i); + ModelDimensionDesc id = im.getDimensions().get(i); + assertEquals(ld.getTable(), id.getTable()); + assertArrayEquals(ld.getColumns(), id.getColumns()); + } + + assertArrayEquals(lm.getMetrics(), im.getMetrics()); + } + + private void assertSnowflakeQuality(DataModelDesc model) { + Assert.assertNotNull(model); + try { + model.findTable("TEST_COUNTRY"); + Assert.fail(); + } catch (IllegalArgumentException ex) { + // excepted + } + + Assert.assertNotNull(model.findTable("BUYER_COUNTRY")); + Assert.assertNotNull(model.findTable("SELLER_COUNTRY")); + Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME")); + Assert.assertNotNull(model.findColumn("BUYER_ID")); + } @Test @@ -82,24 +127,6 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase { } @Test - public void testSnowflakeDataModel() throws Exception { - DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_sales_model"); - Assert.assertTrue(model.getDimensions().size() > 0); - - try { - model.findTable("KYLIN_COUNTRY"); - Assert.fail(); - } catch (IllegalArgumentException ex) { - // excepted - } - - Assert.assertNotNull(model.findTable("BUYER_COUNTRY")); - Assert.assertNotNull(model.findTable("SELLER_COUNTRY")); - Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME")); - Assert.assertNotNull(model.findColumn("BUYER_ID")); - } - - @Test public void testTableSample() throws IOException { TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("TEST.TEST_TABLE"); Assert.assertNotNull(tableExtDesc); http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java index f52ef45..d5e9de7 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java @@ -48,21 +48,22 @@ public class JoinsTreeTest extends LocalFileMetadataTestCase { @Test public void testBasics() { MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model"); + DataModelDesc model = mgr.getDataModelDesc("ci_left_join_model"); JoinsTree joinsTree = model.getJoinsTree(); Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY"); assertTrue(chain.table == model.findTable("BUYER_COUNTRY")); assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT")); - assertTrue(chain.fkSide.fkSide.table == model.findTable("KYLIN_SALES")); - assertTrue(chain.fkSide.fkSide.join == null); - assertTrue(chain.fkSide.fkSide.fkSide == null); + assertTrue(chain.fkSide.fkSide.table == model.findTable("TEST_ORDER")); + assertTrue(chain.fkSide.fkSide.fkSide.table == model.findTable("TEST_KYLIN_FACT")); + assertTrue(chain.fkSide.fkSide.fkSide.join == null); + assertTrue(chain.fkSide.fkSide.fkSide.fkSide == null); } @Test public void testMatch() { MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model"); + DataModelDesc model = mgr.getDataModelDesc("ci_inner_join_model"); JoinsTree joinsTree = model.getJoinsTree(); Map<String, String> matchUp = joinsTree.matches(joinsTree); http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java index 47eb9c3..5f5814b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java @@ -100,7 +100,6 @@ public class JobBuilderSupport { CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams()); CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams()); CubingExecutableUtil.setCubingJobId(jobId, result.getParams()); - CubingExecutableUtil.setIndexPath(this.getSecondaryIndexPath(jobId), result.getParams()); return result; @@ -125,7 +124,6 @@ public class JobBuilderSupport { CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams()); CubingExecutableUtil.setCubingJobId(jobId, result.getParams()); CubingExecutableUtil.setMergingSegmentIds(mergingSegmentIds, result.getParams()); - CubingExecutableUtil.setIndexPath(this.getSecondaryIndexPath(jobId), result.getParams()); return result; } http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java index 4985503..8b9697e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java @@ -57,14 +57,13 @@ public class CreateDictionaryJob extends AbstractHadoopJob { DictionaryGeneratorCLI.processSegment(config, cubeName, segmentID, new DistinctColumnValuesProvider() { @Override public ReadableTable getDistinctValuesFor(TblColRef col) { - return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getName(), col.getType()); + return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getIdentity(), col.getType()); } }, new DictionaryProvider() { @Override public Dictionary<String> getDictionary(TblColRef col) throws IOException { - Path colDir = new Path(factColumnsInputPath, col.getName()); - Path dictFile = new Path(colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX); + Path dictFile = new Path(factColumnsInputPath, col.getIdentity() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX); FileSystem fs = HadoopUtil.getFileSystem(dictFile.toString()); if (fs.exists(dictFile) == false) return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java index b0d5a89..65c5869 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java @@ -135,12 +135,4 @@ public class CubingExecutableUtil { return params.get(MERGED_STATISTICS_PATH); } - public static void setIndexPath(String indexPath, Map<String, String> params) { - params.put(INDEX_PATH, indexPath); - } - - public static String getIndexPath(Map<String, String> params) { - return params.get(INDEX_PATH); - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java index 2eb694e..9fc8922 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java @@ -145,6 +145,9 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob { job.setPartitionerClass(FactDistinctColumnPartitioner.class); job.setNumReduceTasks(numberOfReducers); + // important, reducer writes HDFS directly at the moment + job.setReduceSpeculativeExecution(false); + FileOutputFormat.setOutputPath(job, output); job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString()); http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java index 0223914..5d42797 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java @@ -59,7 +59,7 @@ import com.google.common.collect.Maps; */ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableKey, Text, NullWritable, Text> { - protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class); + private static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class); private List<TblColRef> columnList; private String statisticsOutput = null; @@ -76,6 +76,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK private int uhcReducerCount; private Map<Integer, Integer> reducerIdToColumnIndex = new HashMap<>(); private int taskId; + private boolean isPartitionCol = false; + private int rowCount = 0; //local build dict private boolean isReducerLocalBuildDict; @@ -84,7 +86,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK private long timeMinValue = Long.MAX_VALUE; public static final String DICT_FILE_POSTFIX = ".rldict"; public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".pci"; - private boolean isPartitionCol = false; @Override protected void setup(Context context) throws IOException { @@ -111,6 +112,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK baseCuboidRowCountInMappers = Lists.newArrayList(); cuboidHLLMap = Maps.newHashMap(); samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT)); + logger.info("Reducer " + taskId + " handling stats"); } else if (collectStatistics && (taskId == numberOfTasks - 2)) { // partition col isStatistics = false; @@ -120,6 +122,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK logger.info("Do not have partition col. This reducer will keep empty"); } colValues = Lists.newLinkedList(); + logger.info("Reducer " + taskId + " handling partition column " + col); } else { // normal col isStatistics = false; @@ -135,6 +138,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0); } + logger.info("Reducer " + taskId + " handling column " + col + ", isReducerLocalBuildDict=" + isReducerLocalBuildDict); } } @@ -178,6 +182,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK } else if (isPartitionCol) { // partition col String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1); + logAFewRows(value); long time = DateFormat.stringToMillis(value); timeMinValue = Math.min(timeMinValue, time); timeMaxValue = Math.max(timeMaxValue, time); @@ -185,6 +190,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK // normal col if (isReducerLocalBuildDict) { String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1); + logAFewRows(value); builder.addValue(value); } else { colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1))); @@ -195,14 +201,22 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK } } } + + rowCount++; + } + + private void logAFewRows(String value) { + if (rowCount < 10) { + logger.info("Received value: " + value); + } } private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException { final Configuration conf = context.getConfiguration(); final FileSystem fs = FileSystem.get(conf); final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH); - final Path colDir = new Path(outputPath, col.getName()); - final String fileName = col.getName() + "-" + taskId % uhcReducerCount; + final Path colDir = new Path(outputPath, col.getIdentity()); + final String fileName = col.getIdentity() + "-" + taskId % uhcReducerCount; final Path outputFile = new Path(colDir, fileName); FSDataOutputStream out = null; @@ -229,7 +243,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK } private void outputDict(TblColRef col, Dictionary<String> dict, Context context) throws IOException { - final String fileName = col.getName() + DICT_FILE_POSTFIX; + final String fileName = col.getIdentity() + DICT_FILE_POSTFIX; FSDataOutputStream out = getOutputStream(context, fileName); try { String dictClassName = dict.getClass().getName(); @@ -242,7 +256,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK } private void outputPartitionInfo(Context context) throws IOException { - final String fileName = col.getName() + PARTITION_COL_INFO_FILE_POSTFIX; + final String fileName = col.getIdentity() + PARTITION_COL_INFO_FILE_POSTFIX; FSDataOutputStream out = getOutputStream(context, fileName); try { out.writeLong(timeMinValue); @@ -256,15 +270,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException { final Configuration conf = context.getConfiguration(); final FileSystem fs = FileSystem.get(conf); - final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH); - final Path colDir = new Path(outputPath, col.getName()); - final Path outputFile = new Path(colDir, outputFileName); - FSDataOutputStream out = null; - if (!fs.exists(colDir)) { - fs.mkdirs(colDir); + final Path outputPath = new Path(conf.get(BatchConstants.CFG_OUTPUT_PATH)); + final Path outputFile = new Path(outputPath, outputFileName); + if (!fs.exists(outputPath)) { + fs.mkdirs(outputPath); } - fs.deleteOnExit(outputFile); - out = fs.create(outputFile); + FSDataOutputStream out = fs.create(outputFile); return out; } http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java index 5692c76..ed65343 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java @@ -31,6 +31,8 @@ import org.apache.kylin.engine.mr.common.BatchConstants; import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.measure.hllc.HLLCounter; import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.common.hash.HashFunction; @@ -41,6 +43,8 @@ import com.google.common.hash.Hashing; */ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMapperBase<KEYIN, Object> { + private static final Logger logger = LoggerFactory.getLogger(FactDistinctHiveColumnsMapper.class); + protected boolean collectStatistics = false; protected CuboidScheduler cuboidScheduler = null; protected int nRowKey; @@ -51,7 +55,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap private int rowCount = 0; private int samplingPercentage; private ByteArray[] row_hashcodes = null; - private ByteBuffer keyBuffer; + private ByteBuffer tmpbuf; private static final Text EMPTY_TEXT = new Text(); public static final byte MARK_FOR_PARTITION_COL = (byte) 0xFE; public static final byte MARK_FOR_HLL = (byte) 0xFF; @@ -62,7 +66,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap @Override protected void setup(Context context) throws IOException { super.setup(context); - keyBuffer = ByteBuffer.allocate(4096); + tmpbuf = ByteBuffer.allocate(4096); collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED)); if (collectStatistics) { samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT)); @@ -127,55 +131,54 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap public void doMap(KEYIN key, Object record, Context context) throws IOException, InterruptedException { String[] row = flatTableInputFormat.parseMapperInput(record); - keyBuffer.clear(); - try { - for (int i = 0; i < factDictCols.size(); i++) { - String fieldValue = row[dictionaryColumnIndex[i]]; - if (fieldValue == null) - continue; - int offset = keyBuffer.position(); - - int reducerIndex; - if (uhcIndex[i] == 0) { - //for the normal dictionary column - reducerIndex = columnIndexToReducerBeginId.get(i); - } else { - //for the uhc - reducerIndex = columnIndexToReducerBeginId.get(i) + (fieldValue.hashCode() & 0x7fffffff) % uhcReducerCount; - } - keyBuffer.put(Bytes.toBytes(reducerIndex)[3]); - keyBuffer.put(Bytes.toBytes(fieldValue)); - outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); - sortableKey.setText(outputKey); - //judge type - sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType()); - context.write(sortableKey, EMPTY_TEXT); + for (int i = 0; i < factDictCols.size(); i++) { + String fieldValue = row[dictionaryColumnIndex[i]]; + if (fieldValue == null) + continue; + + int reducerIndex; + if (uhcIndex[i] == 0) { + //for the normal dictionary column + reducerIndex = columnIndexToReducerBeginId.get(i); + } else { + //for the uhc + reducerIndex = columnIndexToReducerBeginId.get(i) + (fieldValue.hashCode() & 0x7fffffff) % uhcReducerCount; + } + + tmpbuf.clear(); + tmpbuf.put(Bytes.toBytes(reducerIndex)[3]); + tmpbuf.put(Bytes.toBytes(fieldValue)); + outputKey.set(tmpbuf.array(), 0, tmpbuf.position()); + sortableKey.setText(outputKey); + //judge type + sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType()); + context.write(sortableKey, EMPTY_TEXT); + + // log a few rows for troubleshooting + if (rowCount < 10) { + logger.info("Sample output: " + factDictCols.get(i) + " '" + fieldValue + "' => reducer " + reducerIndex); } - } catch (Exception ex) { - handleErrorRecord(row, ex); } if (collectStatistics) { - if (rowCount < samplingPercentage) { + if (rowCount % 100 < samplingPercentage) { putRowKeyToHLL(row); } if (needFetchPartitionCol == true) { String fieldValue = row[partitionColumnIndex]; if (fieldValue != null) { - int offset = keyBuffer.position(); - keyBuffer.put(MARK_FOR_PARTITION_COL); - keyBuffer.put(Bytes.toBytes(fieldValue)); - outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); + tmpbuf.clear(); + tmpbuf.put(MARK_FOR_PARTITION_COL); + tmpbuf.put(Bytes.toBytes(fieldValue)); + outputKey.set(tmpbuf.array(), 0, tmpbuf.position()); sortableKey.setText(outputKey); sortableKey.setTypeId((byte) 0); context.write(sortableKey, EMPTY_TEXT); } } } - - if (rowCount++ == 100) - rowCount = 0; + rowCount++; } private void putRowKeyToHLL(String[] row) { @@ -211,10 +214,10 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap for (int i = 0; i < cuboidIds.length; i++) { hll = allCuboidsHLL[i]; - keyBuffer.clear(); - keyBuffer.put(MARK_FOR_HLL); // one byte - keyBuffer.putLong(cuboidIds[i]); - outputKey.set(keyBuffer.array(), 0, keyBuffer.position()); + tmpbuf.clear(); + tmpbuf.put(MARK_FOR_HLL); // one byte + tmpbuf.putLong(cuboidIds[i]); + outputKey.set(tmpbuf.array(), 0, tmpbuf.position()); hllBuf.clear(); hll.writeRegisters(hllBuf); outputValue.set(hllBuf.array(), 0, hllBuf.position()); http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java index d3becfe..dcc9190 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java @@ -59,7 +59,6 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable { long cubeSizeBytes = cubingJob.findCubeSizeBytes(); segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams())); - segment.setIndexPath(CubingExecutableUtil.getIndexPath(this.getParams())); segment.setLastBuildTime(System.currentTimeMillis()); segment.setSizeKB(cubeSizeBytes / 1024); segment.setInputRecords(sourceCount); @@ -81,8 +80,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable { private void updateTimeRange(CubeSegment segment) throws IOException { final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef(); final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH); - Path colDir = new Path(factColumnsInputPath, partitionCol.getName()); - Path outputFile = new Path(colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX); + Path outputFile = new Path(factColumnsInputPath, partitionCol.getIdentity() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX); FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString()); FSDataInputStream is = null; long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE; http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java index d2fa73e..add5c42 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java @@ -75,7 +75,6 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable { mergedSegment.setInputRecords(sourceCount); mergedSegment.setInputRecordsSize(sourceSize); mergedSegment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams())); - mergedSegment.setIndexPath(CubingExecutableUtil.getIndexPath(this.getParams())); mergedSegment.setLastBuildTime(System.currentTimeMillis()); try { http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json index 287ead6..1ebd7f3 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json @@ -5,61 +5,96 @@ "description" : null, "dimensions" : [ { "name" : "CAL_DT", - "table" : "EDW.TEST_CAL_DT", + "table" : "TEST_CAL_DT", "column" : "{FK}", "derived" : [ "WEEK_BEG_DT" ] }, { "name" : "ORDER_ID", - "table" : "DEFAULT.TEST_KYLIN_FACT", + "table" : "TEST_KYLIN_FACT", "column" : "ORDER_ID" }, { "name" : "TEST_DATE_ENC", - "table" : "DEFAULT.TEST_ORDER", + "table" : "TEST_ORDER", "column" : "TEST_DATE_ENC" }, { "name" : "TEST_TIME_ENC", - "table" : "DEFAULT.TEST_ORDER", + "table" : "TEST_ORDER", "column" : "TEST_TIME_ENC" }, { "name" : "CATEGORY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "{FK}", "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] }, { "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "META_CATEG_NAME", "derived" : null }, { "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "CATEG_LVL2_NAME", "derived" : null }, { "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "CATEG_LVL3_NAME", "derived" : null }, { "name" : "LSTG_FORMAT_NAME", - "table" : "DEFAULT.TEST_KYLIN_FACT", + "table" : "TEST_KYLIN_FACT", "column" : "LSTG_FORMAT_NAME", "derived" : null }, { "name" : "SITE_ID", - "table" : "EDW.TEST_SITES", + "table" : "TEST_SITES", "column" : "{FK}", "derived" : [ "SITE_NAME", "CRE_USER" ] }, { "name" : "SELLER_TYPE_CD", - "table" : "EDW.TEST_SELLER_TYPE_DIM", + "table" : "TEST_SELLER_TYPE_DIM", "column" : "{FK}", "derived" : [ "SELLER_TYPE_DESC" ] }, { "name" : "SELLER_ID", - "table" : "DEFAULT.TEST_KYLIN_FACT", - "column" : "SELLER_ID", - "derived" : null + "table" : "TEST_KYLIN_FACT", + "column" : "SELLER_ID" + }, { + "name" : "SELLER_BUYER_LEVEL", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_BUYER_LEVEL" + }, { + "name" : "SELLER_SELLER_LEVEL", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_SELLER_LEVEL" + }, { + "name" : "SELLER_COUNTRY", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_COUNTRY" + }, { + "name" : "SELLER_COUNTRY_NAME", + "table" : "SELLER_COUNTRY", + "column" : "NAME" + }, { + "name" : "BUYER_ID", + "table" : "TEST_ORDER", + "column" : "BUYER_ID" + }, { + "name" : "BUYER_BUYER_LEVEL", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_BUYER_LEVEL" + }, { + "name" : "BUYER_SELLER_LEVEL", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_SELLER_LEVEL" + }, { + "name" : "BUYER_COUNTRY", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_COUNTRY" + }, { + "name" : "BUYER_COUNTRY_NAME", + "table" : "BUYER_COUNTRY", + "column" : "NAME" } ], "measures" : [ { "name" : "TRANS_CNT", @@ -77,7 +112,7 @@ "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "ITEM_COUNT" + "value" : "TEST_KYLIN_FACT.ITEM_COUNT" }, "returntype" : "bigint" } @@ -87,7 +122,7 @@ "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "decimal(19,4)" } @@ -97,7 +132,7 @@ "expression" : "MIN", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "decimal(19,4)" } @@ -107,7 +142,7 @@ "expression" : "MAX", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "decimal(19,4)" } @@ -117,7 +152,7 @@ "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "SELLER_ID" + "value" : "TEST_KYLIN_FACT.SELLER_ID" }, "returntype" : "hllc(10)" } @@ -127,10 +162,10 @@ "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "LSTG_FORMAT_NAME", + "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID" + "value" : "TEST_KYLIN_FACT.SELLER_ID" } }, "returntype" : "hllc(10)" @@ -141,10 +176,10 @@ "expression" : "TOP_N", "parameter" : { "type" : "column", - "value" : "PRICE", + "value" : "TEST_KYLIN_FACT.PRICE", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID" + "value" : "TEST_KYLIN_FACT.SELLER_ID" } }, "returntype" : "topn(100)", @@ -156,7 +191,7 @@ "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "TEST_COUNT_DISTINCT_BITMAP" + "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP" }, "returntype" : "bitmap" } @@ -166,10 +201,38 @@ "expression" : "EXTENDED_COLUMN", "parameter": { "type": "column", - "value": "ORDER_ID", + "value": "TEST_KYLIN_FACT.ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_ORDER.TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } + }, { + "name" : "BUYER_CONTACT", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_ORDER.BUYER_ID", + "next_parameter": { + "type": "column", + "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, { + "name" : "SELLER_CONTACT", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID", "next_parameter": { "type": "column", - "value": "TEST_EXTENDED_COLUMN" + "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT" } }, "returntype": "extendedcolumn(100)" @@ -180,7 +243,7 @@ "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "TRANS_ID" + "value" : "TEST_KYLIN_FACT.TRANS_ID" }, "returntype" : "raw" } @@ -190,7 +253,7 @@ "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "raw" } @@ -200,52 +263,79 @@ "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "CAL_DT" + "value" : "TEST_KYLIN_FACT.CAL_DT" }, "returntype" : "raw" } } ], "dictionaries": [ { - "column": "TEST_COUNT_DISTINCT_BITMAP", + "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" } ], "rowkey" : { "rowkey_columns" : [ { - "column" : "SELLER_ID", + "column" : "TEST_KYLIN_FACT.SELLER_ID", "encoding" : "int:4" }, { - "column" : "ORDER_ID", + "column" : "TEST_KYLIN_FACT.ORDER_ID", "encoding" : "int:4" }, { - "column" : "CAL_DT", + "column" : "TEST_KYLIN_FACT.CAL_DT", "encoding" : "dict" }, { - "column" : "LEAF_CATEG_ID", + "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID", "encoding" : "dict" }, { - "column" : "META_CATEG_NAME", + "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "encoding" : "dict" }, { - "column" : "CATEG_LVL2_NAME", + "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "encoding" : "dict" }, { - "column" : "CATEG_LVL3_NAME", + "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", "encoding" : "dict" }, { - "column" : "LSTG_FORMAT_NAME", + "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", "encoding" : "fixed_length:12" }, { - "column" : "LSTG_SITE_ID", + "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID", "encoding" : "dict" }, { - "column" : "SLR_SEGMENT_CD", + "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD", "encoding" : "dict" }, { - "column" : "TEST_TIME_ENC", + "column" : "TEST_ORDER.TEST_TIME_ENC", "encoding" : "time" }, { - "column" : "TEST_DATE_ENC", + "column" : "TEST_ORDER.TEST_DATE_ENC", "encoding" : "date" + }, { + "column" : "TEST_ORDER.BUYER_ID", + "encoding" : "int:4" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding" : "dict" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding" : "dict" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding" : "dict" + }, { + "column" : "BUYER_COUNTRY.NAME", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding" : "dict" + }, { + "column" : "SELLER_COUNTRY.NAME", + "encoding" : "dict" } ] }, "signature" : null, @@ -268,26 +358,80 @@ "name" : "f3", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ] + "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ] } ] } ] }, "aggregation_groups" : [ { - "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", - "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ], + "includes" : [ "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ], "select_rule" : { - "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], + "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ], "mandatory_dims" : [ ], - "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ] } }, { - "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", - "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", - "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ], + "includes" : [ "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME", + + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" ], "select_rule" : { "hierarchy_dims" : [ ], - "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ], - "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ], + + "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" ], + + [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ], + + [ "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME" ], + + [ "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" ] ] } } ], "notify_list" : null, http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json index be1ea99..4048b6e 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json @@ -5,61 +5,96 @@ "description" : null, "dimensions" : [ { "name" : "CAL_DT", - "table" : "EDW.TEST_CAL_DT", + "table" : "TEST_CAL_DT", "column" : "{FK}", "derived" : [ "WEEK_BEG_DT" ] }, { "name" : "ORDER_ID", - "table" : "DEFAULT.TEST_KYLIN_FACT", + "table" : "TEST_KYLIN_FACT", "column" : "ORDER_ID" }, { "name" : "TEST_DATE_ENC", - "table" : "DEFAULT.TEST_ORDER", + "table" : "TEST_ORDER", "column" : "TEST_DATE_ENC" }, { "name" : "TEST_TIME_ENC", - "table" : "DEFAULT.TEST_ORDER", + "table" : "TEST_ORDER", "column" : "TEST_TIME_ENC" }, { "name" : "CATEGORY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "{FK}", "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] }, { "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "META_CATEG_NAME", "derived" : null }, { "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "CATEG_LVL2_NAME", "derived" : null }, { "name" : "CATEGORY_HIERARCHY", - "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", + "table" : "TEST_CATEGORY_GROUPINGS", "column" : "CATEG_LVL3_NAME", "derived" : null }, { "name" : "LSTG_FORMAT_NAME", - "table" : "DEFAULT.TEST_KYLIN_FACT", + "table" : "TEST_KYLIN_FACT", "column" : "LSTG_FORMAT_NAME", "derived" : null }, { "name" : "SITE_ID", - "table" : "EDW.TEST_SITES", + "table" : "TEST_SITES", "column" : "{FK}", "derived" : [ "SITE_NAME", "CRE_USER" ] }, { "name" : "SELLER_TYPE_CD", - "table" : "EDW.TEST_SELLER_TYPE_DIM", + "table" : "TEST_SELLER_TYPE_DIM", "column" : "{FK}", "derived" : [ "SELLER_TYPE_DESC" ] }, { "name" : "SELLER_ID", - "table" : "DEFAULT.TEST_KYLIN_FACT", - "column" : "SELLER_ID", - "derived" : null + "table" : "TEST_KYLIN_FACT", + "column" : "SELLER_ID" + }, { + "name" : "SELLER_BUYER_LEVEL", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_BUYER_LEVEL" + }, { + "name" : "SELLER_SELLER_LEVEL", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_SELLER_LEVEL" + }, { + "name" : "SELLER_COUNTRY", + "table" : "SELLER_ACCOUNT", + "column" : "ACCOUNT_COUNTRY" + }, { + "name" : "SELLER_COUNTRY_NAME", + "table" : "SELLER_COUNTRY", + "column" : "NAME" + }, { + "name" : "BUYER_ID", + "table" : "TEST_ORDER", + "column" : "BUYER_ID" + }, { + "name" : "BUYER_BUYER_LEVEL", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_BUYER_LEVEL" + }, { + "name" : "BUYER_SELLER_LEVEL", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_SELLER_LEVEL" + }, { + "name" : "BUYER_COUNTRY", + "table" : "BUYER_ACCOUNT", + "column" : "ACCOUNT_COUNTRY" + }, { + "name" : "BUYER_COUNTRY_NAME", + "table" : "BUYER_COUNTRY", + "column" : "NAME" } ], "measures" : [ { "name" : "TRANS_CNT", @@ -77,7 +112,7 @@ "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "ITEM_COUNT" + "value" : "TEST_KYLIN_FACT.ITEM_COUNT" }, "returntype" : "bigint" } @@ -87,7 +122,7 @@ "expression" : "SUM", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "decimal(19,4)" } @@ -97,7 +132,7 @@ "expression" : "MIN", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "decimal(19,4)" } @@ -107,7 +142,7 @@ "expression" : "MAX", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "decimal(19,4)" } @@ -117,7 +152,7 @@ "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "SELLER_ID" + "value" : "TEST_KYLIN_FACT.SELLER_ID" }, "returntype" : "hllc(10)" } @@ -127,10 +162,10 @@ "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "LSTG_FORMAT_NAME", + "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID" + "value" : "TEST_KYLIN_FACT.SELLER_ID" } }, "returntype" : "hllc(10)" @@ -141,10 +176,10 @@ "expression" : "TOP_N", "parameter" : { "type" : "column", - "value" : "PRICE", + "value" : "TEST_KYLIN_FACT.PRICE", "next_parameter" : { "type" : "column", - "value" : "SELLER_ID" + "value" : "TEST_KYLIN_FACT.SELLER_ID" } }, "returntype" : "topn(100)", @@ -156,7 +191,7 @@ "expression" : "COUNT_DISTINCT", "parameter" : { "type" : "column", - "value" : "TEST_COUNT_DISTINCT_BITMAP" + "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP" }, "returntype" : "bitmap" } @@ -166,10 +201,38 @@ "expression" : "EXTENDED_COLUMN", "parameter": { "type": "column", - "value": "ORDER_ID", + "value": "TEST_KYLIN_FACT.ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_ORDER.TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } + }, { + "name" : "BUYER_CONTACT", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_ORDER.BUYER_ID", + "next_parameter": { + "type": "column", + "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, { + "name" : "SELLER_CONTACT", + "function" : { + "expression" : "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID", "next_parameter": { "type": "column", - "value": "TEST_EXTENDED_COLUMN" + "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT" } }, "returntype": "extendedcolumn(100)" @@ -180,7 +243,7 @@ "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "TRANS_ID" + "value" : "TEST_KYLIN_FACT.TRANS_ID" }, "returntype" : "raw" } @@ -190,7 +253,7 @@ "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "PRICE" + "value" : "TEST_KYLIN_FACT.PRICE" }, "returntype" : "raw" } @@ -200,52 +263,79 @@ "expression" : "RAW", "parameter" : { "type" : "column", - "value" : "CAL_DT" + "value" : "TEST_KYLIN_FACT.CAL_DT" }, "returntype" : "raw" } } ], "dictionaries": [ { - "column": "TEST_COUNT_DISTINCT_BITMAP", + "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" } ], "rowkey" : { "rowkey_columns" : [ { - "column" : "SELLER_ID", + "column" : "TEST_KYLIN_FACT.SELLER_ID", "encoding" : "int:4" }, { - "column" : "ORDER_ID", + "column" : "TEST_KYLIN_FACT.ORDER_ID", "encoding" : "int:4" }, { - "column" : "CAL_DT", + "column" : "TEST_KYLIN_FACT.CAL_DT", "encoding" : "dict" }, { - "column" : "LEAF_CATEG_ID", + "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID", "encoding" : "dict" }, { - "column" : "META_CATEG_NAME", + "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "encoding" : "dict" }, { - "column" : "CATEG_LVL2_NAME", + "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "encoding" : "dict" }, { - "column" : "CATEG_LVL3_NAME", + "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", "encoding" : "dict" }, { - "column" : "LSTG_FORMAT_NAME", + "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", "encoding" : "fixed_length:12" }, { - "column" : "LSTG_SITE_ID", + "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID", "encoding" : "dict" }, { - "column" : "SLR_SEGMENT_CD", + "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD", "encoding" : "dict" }, { - "column" : "TEST_TIME_ENC", + "column" : "TEST_ORDER.TEST_TIME_ENC", "encoding" : "time" }, { - "column" : "TEST_DATE_ENC", + "column" : "TEST_ORDER.TEST_DATE_ENC", "encoding" : "date" + }, { + "column" : "TEST_ORDER.BUYER_ID", + "encoding" : "int:4" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding" : "dict" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding" : "dict" + }, { + "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding" : "dict" + }, { + "column" : "BUYER_COUNTRY.NAME", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding" : "dict" + }, { + "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding" : "dict" + }, { + "column" : "SELLER_COUNTRY.NAME", + "encoding" : "dict" } ] }, "signature" : null, @@ -268,26 +358,80 @@ "name" : "f3", "columns" : [ { "qualifier" : "m", - "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ] + "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ] } ] } ] }, "aggregation_groups" : [ { - "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", - "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ], + "includes" : [ "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ], "select_rule" : { - "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], + "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ], "mandatory_dims" : [ ], - "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ] } }, { - "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", - "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", - "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ], + "includes" : [ "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME", + + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" ], "select_rule" : { "hierarchy_dims" : [ ], - "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ], - "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ] + "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ], + + "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" ], + + [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ], + + [ "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME" ], + + [ "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" ] ] } } ], "notify_list" : null, http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json deleted file mode 100644 index ee0d68c..0000000 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf", - "last_modified" : 1451468470824, - "name" : "test_kylin_snowflake_sales_cube", - "model_name" : "test_kylin_snowflake_sales_model", - "description" : null, - "null_string" : null, - "dimensions" : [ { - "name" : "TRANS_ID", - "table" : "SNOWTEST.KYLIN_SALES", - "column" : "TRANS_ID", - "derived" : null - }, { - "name" : "CAL_DT", - "table" : "SNOWTEST.KYLIN_CAL_DT", - "column" : "{FK}", - "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ] - }, { - "name" : "CATEGORY", - "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", - "column" : "{FK}", - "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ] - }, { - "name" : "META_CATEG_NAME", - "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", - "column" : "META_CATEG_NAME", - "derived" : null - }, { - "name" : "CATEG_LVL2_NAME", - "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL2_NAME", - "derived" : null - }, { - "name" : "CATEG_LVL3_NAME", - "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL3_NAME", - "derived" : null - }, { - "name" : "LSTG_FORMAT_NAME", - "table" : "SNOWTEST.KYLIN_SALES", - "column" : "LSTG_FORMAT_NAME", - "derived" : null - }, { - "name" : "SELLER_ID", - "table" : "SNOWTEST.KYLIN_SALES", - "column" : "SELLER_ID", - "derived" : null - }, { - "name" : "BUYER_ID", - "table" : "SNOWTEST.KYLIN_SALES", - "column" : "BUYER_ID", - "derived" : null - }, { - "name" : "ACCOUNT_BUYER_LEVEL", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_BUYER_LEVEL", - "derived" : null - }, { - "name" : "ACCOUNT_SELLER_LEVEL", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_SELLER_LEVEL", - "derived" : null - }, { - "name" : "BUYER_COUNTRY", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_COUNTRY", - "derived" : null - }, { - "name" : "SELLER_COUNTRY", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_COUNTRY", - "derived" : null - }, { - "name" : "BUYER_COUNTRY_NAME", - "table" : "BUYER_COUNTRY", - "column" : "NAME", - "derived" : null - }, { - "name" : "SELLER_COUNTRY_NAME", - "table" : "SELLER_COUNTRY", - "column" : "NAME", - "derived" : null - }, { - "name" : "OPS_USER_ID", - "table" : "SNOWTEST.KYLIN_SALES", - "column" : "OPS_USER_ID", - "derived" : null - }, { - "name" : "OPS_REGION", - "table" : "SNOWTEST.KYLIN_SALES", - "column" : "OPS_REGION", - "derived" : null - } ], - "measures" : [ { - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - } - }, { - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - } - }, { - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : null - }, - "returntype" : "decimal(19,4)" - } - }, { - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1", - "next_parameter" : null - }, - "returntype" : "bigint" - } - }, { - "name" : "SELLER_CNT_HLL", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null - }, - "returntype" : "hllc(10)" - } - }, { - "name" : "TOP_SELLER", - "function" : { - "expression" : "TOP_N", - "parameter" : { - "type" : "column", - "value" : "PRICE", - "next_parameter" : { - "type" : "column", - "value" : "SELLER_ID", - "next_parameter" : null - } - }, - "returntype" : "topn(100)" - } - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "BUYER_ID", - "encoding" : "integer:4" - }, { - "column" : "SELLER_ID", - "encoding" : "integer:4" - }, { - "column" : "TRANS_ID", - "encoding" : "integer:4" - }, { - "column" : "PART_DT", - "encoding" : "date" - }, { - "column" : "LEAF_CATEG_ID", - "encoding" : "dict" - }, { - "column" : "META_CATEG_NAME", - "encoding" : "dict" - }, { - "column" : "CATEG_LVL2_NAME", - "encoding" : "dict" - }, { - "column" : "CATEG_LVL3_NAME", - "encoding" : "dict" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "encoding" : "dict" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY", - "encoding" : "dict" - }, { - "column" : "BUYER_COUNTRY.NAME", - "encoding" : "dict" - }, { - "column" : "SELLER_COUNTRY.NAME", - "encoding" : "dict" - }, { - "column" : "LSTG_FORMAT_NAME", - "encoding" : "dict" - }, { - "column" : "LSTG_SITE_ID", - "encoding" : "dict" - }, { - "column" : "OPS_USER_ID", - "encoding" : "dict" - }, { - "column" : "OPS_REGION", - "encoding" : "dict" - } ] - }, - "hbase_mapping" : { - "column_family" : [ { - "name" : "F1", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ] - } ] - }, { - "name" : "F2", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ] - } ] - } ] - }, - "aggregation_groups" : [ { - "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", - "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ], - "select_rule" : { - "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ], - "mandatory_dims" : [ "PART_DT" ], - "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ], - [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ] - } - }, { - "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID", - "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", - "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ], - "select_rule" : { - "hierarchy_dims" : [ ], - "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID", - "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION", - "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ], - "joint_dims" : [ ] - } - } ], - "notify_list" : null, - "status_need_notify" : [ ], - "partition_date_start" : 1325376000000, - "auto_merge_time_ranges" : null, - "retention_range" : 0, - "engine_type" : 2, - "storage_type" : 2, - "override_kylin_properties" : { - "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true" - } -}