kylin git commit: KYLIN-3052 Support Redshift as data source
Repository: kylin Updated Branches: refs/heads/KYLIN-3052 [created] fb2c9a5b6 KYLIN-3052 Support Redshift as data source Signed-off-by: shaofengshi Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb2c9a5b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb2c9a5b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb2c9a5b Branch: refs/heads/KYLIN-3052 Commit: fb2c9a5b6e033907e5109d2b0cb0027fdd5e5647 Parents: 6474e22 Author: ZhansShaoxiong Authored: Sat Dec 2 15:28:00 2017 +0800 Committer: shaofengshi Committed: Tue Dec 5 17:55:11 2017 +0800 -- .../org/apache/kylin/job/constant/ExecutableConstants.java | 1 + .../java/org/apache/kylin/rest/service/TableService.java| 8 .../java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java | 3 ++- .../kylin/source/jdbc/metadata/MySQLJdbcMetadata.java | 6 ++ .../kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java | 9 ++--- 5 files changed, 11 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index f3caf3b..cf61722 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -35,6 +35,7 @@ public final class ExecutableConstants { public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary"; public static final String STEP_NAME_BUILD_UHC_DICTIONARY = "Build UHC Dictionary"; public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table"; +public static final String STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE = "Sqoop To Flat Hive Table"; public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables"; public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns"; public static final String STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID = "Calculate Statistics from Base Cuboid"; http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index ffb71a6..901ac46 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -119,7 +119,7 @@ public class TableService extends BasicService { SetMultimap db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); -db2tables.put(parts[0].toUpperCase(), parts[1].toUpperCase()); +db2tables.put(parts[0], parts[1]); } // load all tables first @@ -128,9 +128,9 @@ public class TableService extends BasicService { for (Map.Entry entry : db2tables.entries()) { Pair pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); - Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey())); - Preconditions.checkState(tableDesc.getName().equals(entry.getValue())); - Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." + entry.getValue())); + Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase())); + Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase())); + Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase() + "." + entry.getValue().toUpperCase())); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRI
[1/2] kylin git commit: KYLIN-3052 Support Redshift as data source
Repository: kylin Updated Branches: refs/heads/master 577acc5a7 -> 14dd92e00 KYLIN-3052 Support Redshift as data source Signed-off-by: shaofengshi Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c435cd3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c435cd3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c435cd3 Branch: refs/heads/master Commit: 5c435cd3554e926f176143633a09a27554905788 Parents: 577acc5 Author: ZhansShaoxiong Authored: Sat Dec 2 15:28:00 2017 +0800 Committer: shaofengshi Committed: Wed Dec 6 09:08:13 2017 +0800 -- .../org/apache/kylin/job/constant/ExecutableConstants.java | 1 + .../java/org/apache/kylin/rest/service/TableService.java| 8 .../java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java | 3 ++- .../kylin/source/jdbc/metadata/MySQLJdbcMetadata.java | 6 ++ .../kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java | 9 ++--- 5 files changed, 11 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/5c435cd3/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index f3caf3b..cf61722 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -35,6 +35,7 @@ public final class ExecutableConstants { public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary"; public static final String STEP_NAME_BUILD_UHC_DICTIONARY = "Build UHC Dictionary"; public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table"; +public static final String STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE = "Sqoop To Flat Hive Table"; public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables"; public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns"; public static final String STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID = "Calculate Statistics from Base Cuboid"; http://git-wip-us.apache.org/repos/asf/kylin/blob/5c435cd3/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index ffb71a6..901ac46 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -119,7 +119,7 @@ public class TableService extends BasicService { SetMultimap db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); -db2tables.put(parts[0].toUpperCase(), parts[1].toUpperCase()); +db2tables.put(parts[0], parts[1]); } // load all tables first @@ -128,9 +128,9 @@ public class TableService extends BasicService { for (Map.Entry entry : db2tables.entries()) { Pair pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); - Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey())); - Preconditions.checkState(tableDesc.getName().equals(entry.getValue())); - Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." + entry.getValue())); + Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase())); + Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase())); + Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase() + "." + entry.getValue().toUpperCase())); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); http://git-wip-us.apache.org/repos/asf/kylin/blob/5c435cd3/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.