KYLIN-2250 HiveMRInput accepts intermediate_view_table_name Signed-off-by: Li Yang <liy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5c05880 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5c05880 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5c05880 Branch: refs/heads/master-cdh5.7 Commit: b5c058801b916ddc6e00085d652ad6a0497ce7d9 Parents: 7699973 Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Tue Dec 6 15:48:21 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Tue Dec 6 16:46:40 2016 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/cube/CubeManager.java | 2 +- .../apache/kylin/cube/cli/DictionaryGeneratorCLI.java | 2 +- .../org/apache/kylin/metadata/model/TableDesc.java | 14 +++++++++----- .../org/apache/kylin/source/hive/HiveMRInput.java | 13 +++++++++---- 4 files changed, 20 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 0c80c07..bda1423 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -274,7 +274,7 @@ public class CubeManager implements IRealizationProvider { SnapshotManager snapshotMgr = getSnapshotManager(); TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable)); - if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) { + if (tableDesc.isView()) { String tableName = tableDesc.getMaterializedName(); tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable()); tableDesc.setName(tableName); http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java index 163c6ca..3e1ab0d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java @@ -98,7 +98,7 @@ public class DictionaryGeneratorCLI { } else { MetadataManager metadataManager = MetadataManager.getInstance(config); TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable)); - if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) { + if (tableDesc.isView()) { TableDesc materializedTbl = new TableDesc(); materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable()); materializedTbl.setName(tableDesc.getMaterializedName()); http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index 659de07..ab8c465 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -26,8 +26,8 @@ import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.StringSplitter; import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Table Metadata from Source. All name should be uppercase. @@ -36,7 +36,9 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class TableDesc extends RootPersistentEntity implements ISourceAware { - public static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW"; + private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW"; + private static final String materializedTableNamePrefix = "kylin_intermediate_"; + @JsonProperty("name") private String name; @JsonProperty("columns") @@ -46,8 +48,6 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { @JsonProperty("table_type") private String tableType; - private static final String materializedTableNamePrefix = "kylin_intermediate_"; - private DatabaseDesc database = new DatabaseDesc(); private String identity = null; @@ -96,6 +96,10 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { } return identity; } + + public boolean isView() { + return TABLE_TYPE_VIRTUAL_VIEW.equals(tableType); + } public static String concatResourcePath(String tableIdentity) { return ResourceStore.TABLE_RESOURCE_ROOT + "/" + tableIdentity + ".json"; @@ -211,7 +215,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { return false; if (!Arrays.equals(columns, tableDesc.columns)) return false; - + return getIdentity().equals(tableDesc.getIdentity()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index aa603d7..fc2b982 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -60,6 +60,11 @@ import com.google.common.collect.Sets; public class HiveMRInput implements IMRInput { + public static String getTableNameForHCat(TableDesc table) { + String tableName = table.isView() ? table.getMaterializedName() : table.getName(); + return String.format("%s.%s", table.getDatabase(), tableName).toUpperCase(); + } + @Override public IMRBatchCubingInputSide getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) { return new BatchCubingInputSide(flatDesc); @@ -67,9 +72,9 @@ public class HiveMRInput implements IMRInput { @Override public IMRTableInputFormat getTableInputFormat(TableDesc table) { - return new HiveTableInputFormat(table.getIdentity()); + return new HiveTableInputFormat(getTableNameForHCat(table)); } - + @Override public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) { return new IMRBatchMergeInputSide() { @@ -167,7 +172,7 @@ public class HiveMRInput implements IMRInput { for (JoinTableDesc lookupDesc : flatDesc.getDataModel().getJoinTables()) { TableDesc tableDesc = metadataManager.getTableDesc(lookupDesc.getTable()); - if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) { + if (tableDesc.isView()) { lookupViewsTables.add(tableDesc); } } @@ -180,7 +185,7 @@ public class HiveMRInput implements IMRInput { hiveCmdBuilder.addStatement(useDatabaseHql); hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf)); for (TableDesc lookUpTableDesc : lookupViewsTables) { - if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(lookUpTableDesc.getTableType())) { + if (lookUpTableDesc.isView()) { StringBuilder createIntermediateTableHql = new StringBuilder(); createIntermediateTableHql.append("DROP TABLE IF EXISTS " + lookUpTableDesc.getMaterializedName() + ";\n"); createIntermediateTableHql.append("CREATE TABLE IF NOT EXISTS " + lookUpTableDesc.getMaterializedName() + "\n");