kylin git commit: KYLIN-3052 Support Redshift as data source

2017-12-05 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-3052 [created] fb2c9a5b6


KYLIN-3052 Support Redshift as data source

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb2c9a5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb2c9a5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb2c9a5b

Branch: refs/heads/KYLIN-3052
Commit: fb2c9a5b6e033907e5109d2b0cb0027fdd5e5647
Parents: 6474e22
Author: ZhansShaoxiong 
Authored: Sat Dec 2 15:28:00 2017 +0800
Committer: shaofengshi 
Committed: Tue Dec 5 17:55:11 2017 +0800

--
 .../org/apache/kylin/job/constant/ExecutableConstants.java  | 1 +
 .../java/org/apache/kylin/rest/service/TableService.java| 8 
 .../java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java  | 3 ++-
 .../kylin/source/jdbc/metadata/MySQLJdbcMetadata.java   | 6 ++
 .../kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java   | 9 ++---
 5 files changed, 11 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index f3caf3b..cf61722 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -35,6 +35,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension 
Dictionary";
 public static final String STEP_NAME_BUILD_UHC_DICTIONARY = "Build UHC 
Dictionary";
 public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create 
Intermediate Flat Hive Table";
+public static final String STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE = "Sqoop To 
Flat Hive Table";
 public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = 
"Materialize Hive View in Lookup Tables";
 public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact 
Table Distinct Columns";
 public static final String STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID = 
"Calculate Statistics from Base Cuboid";

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index ffb71a6..901ac46 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -119,7 +119,7 @@ public class TableService extends BasicService {
 SetMultimap db2tables = LinkedHashMultimap.create();
 for (String fullTableName : tables) {
 String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
-db2tables.put(parts[0].toUpperCase(), parts[1].toUpperCase());
+db2tables.put(parts[0], parts[1]);
 }
 
 // load all tables first
@@ -128,9 +128,9 @@ public class TableService extends BasicService {
 for (Map.Entry entry : db2tables.entries()) {
 Pair pair = 
explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
 TableDesc tableDesc = pair.getFirst();
-
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey()));
-
Preconditions.checkState(tableDesc.getName().equals(entry.getValue()));
-
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." + 
entry.getValue()));
+
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase()));
+
Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase()));
+
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase()
 + "." + entry.getValue().toUpperCase()));
 TableExtDesc extDesc = pair.getSecond();
 
Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
 allMeta.add(pair);

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRI

[1/2] kylin git commit: KYLIN-3052 Support Redshift as data source

2017-12-05 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master 577acc5a7 -> 14dd92e00


KYLIN-3052 Support Redshift as data source

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c435cd3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c435cd3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c435cd3

Branch: refs/heads/master
Commit: 5c435cd3554e926f176143633a09a27554905788
Parents: 577acc5
Author: ZhansShaoxiong 
Authored: Sat Dec 2 15:28:00 2017 +0800
Committer: shaofengshi 
Committed: Wed Dec 6 09:08:13 2017 +0800

--
 .../org/apache/kylin/job/constant/ExecutableConstants.java  | 1 +
 .../java/org/apache/kylin/rest/service/TableService.java| 8 
 .../java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java  | 3 ++-
 .../kylin/source/jdbc/metadata/MySQLJdbcMetadata.java   | 6 ++
 .../kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java   | 9 ++---
 5 files changed, 11 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c435cd3/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index f3caf3b..cf61722 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -35,6 +35,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension 
Dictionary";
 public static final String STEP_NAME_BUILD_UHC_DICTIONARY = "Build UHC 
Dictionary";
 public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create 
Intermediate Flat Hive Table";
+public static final String STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE = "Sqoop To 
Flat Hive Table";
 public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = 
"Materialize Hive View in Lookup Tables";
 public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact 
Table Distinct Columns";
 public static final String STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID = 
"Calculate Statistics from Base Cuboid";

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c435cd3/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index ffb71a6..901ac46 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -119,7 +119,7 @@ public class TableService extends BasicService {
 SetMultimap db2tables = LinkedHashMultimap.create();
 for (String fullTableName : tables) {
 String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
-db2tables.put(parts[0].toUpperCase(), parts[1].toUpperCase());
+db2tables.put(parts[0], parts[1]);
 }
 
 // load all tables first
@@ -128,9 +128,9 @@ public class TableService extends BasicService {
 for (Map.Entry entry : db2tables.entries()) {
 Pair pair = 
explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
 TableDesc tableDesc = pair.getFirst();
-
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey()));
-
Preconditions.checkState(tableDesc.getName().equals(entry.getValue()));
-
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." + 
entry.getValue()));
+
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase()));
+
Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase()));
+
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase()
 + "." + entry.getValue().toUpperCase()));
 TableExtDesc extDesc = pair.getSecond();
 
Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
 allMeta.add(pair);

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c435cd3/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.