[1/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
Repository: kylin Updated Branches: refs/heads/KYLIN-3044 [created] 695971a2a http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java -- diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java new file mode 100644 index 000..a5516ab --- /dev/null +++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.source.jdbc.metadata; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +import org.apache.kylin.source.hive.DBConnConf; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class SQLServerJdbcMetadataTest extends DefaultJdbcMetadataTest { + +@Before +public void setup() { +dbConnConf = new DBConnConf(); +dbConnConf.setUrl("jdbc:sqlserver://fakehost:1433;database=testdb"); +dbConnConf.setDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver"); +dbConnConf.setUser("user"); +dbConnConf.setPass("pass"); +jdbcMetadata = new SQLServerJdbcMetadata(dbConnConf); + +setupProperties(); +} + +@Test +public void testListDatabases() throws SQLException { +ResultSet rs = mock(ResultSet.class); +when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false); + when(rs.getString("TABLE_SCHEM")).thenReturn("schema1").thenReturn("schema2"); + when(rs.getString("TABLE_CATALOG")).thenReturn("catalog1").thenReturn("testdb"); + +when(connection.getCatalog()).thenReturn("testdb"); +when(connection.getMetaData()).thenReturn(dbmd); +when(dbmd.getSchemas("testdb", "%")).thenReturn(rs); + +List dbs = jdbcMetadata.listDatabases(); + +Assert.assertEquals(1, dbs.size()); +Assert.assertEquals("schema2", dbs.get(0)); +} + +@Test(expected = IllegalArgumentException.class) +public void testListDatabasesWithoutSpecificDB() throws SQLException { +when(connection.getCatalog()).thenReturn(""); +jdbcMetadata.listDatabases(); +} +}
[2/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
KYLIN-3044 support SQLServer as kylin data source Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cac1f8bb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cac1f8bb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cac1f8bb Branch: refs/heads/KYLIN-3044 Commit: cac1f8bb48239fe5a49ca720575915f3507a4010 Parents: d837e18 Author: etherge Authored: Wed Nov 22 00:26:49 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 13:03:47 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 31 +- .../main/resources/kylin-defaults.properties| 9 + .../org/apache/kylin/job/JoinedFlatTable.java | 45 +-- pom.xml | 2 +- source-hive/pom.xml | 25 ++ .../apache/kylin/source/jdbc/JdbcDialect.java | 26 ++ .../apache/kylin/source/jdbc/JdbcExplorer.java | 288 --- .../kylin/source/jdbc/JdbcHiveMRInput.java | 129 +++-- .../kylin/source/jdbc/JdbcTableReader.java | 32 ++- .../org/apache/kylin/source/jdbc/SqlUtil.java | 140 ++--- .../jdbc/metadata/DefaultJdbcMetadata.java | 76 + .../source/jdbc/metadata/IJdbcMetadata.java | 33 +++ .../jdbc/metadata/JdbcMetadataFactory.java | 35 +++ .../source/jdbc/metadata/MySQLJdbcMetadata.java | 69 + .../jdbc/metadata/SQLServerJdbcMetadata.java| 61 .../kylin/source/jdbc/JdbcExplorerTest.java | 156 ++ .../apache/kylin/source/jdbc/SqlUtilTest.java | 46 +++ .../jdbc/metadata/DefaultJdbcMetadataTest.java | 126 .../jdbc/metadata/JdbcMetadataFactoryTest.java | 35 +++ .../jdbc/metadata/MySQLJdbcMetadataTest.java| 104 +++ .../metadata/SQLServerJdbcMetadataTest.java | 68 + 21 files changed, 1258 insertions(+), 278 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 3d67ee3..b2368ce 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -287,7 +287,7 @@ abstract public class KylinConfigBase implements Serializable { r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html return r; } - + public String getDataModelImpl() { return getOptional("kylin.metadata.data-model-impl", null); } @@ -295,7 +295,7 @@ abstract public class KylinConfigBase implements Serializable { public String getDataModelManagerImpl() { return getOptional("kylin.metadata.data-model-manager-impl", null); } - + public String[] getRealizationProviders() { return getOptionalStringArray("kylin.metadata.realization-providers", // new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); @@ -314,7 +314,7 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } - + public String getHBaseMappingAdapter() { return getOptional("kylin.metadata.hbasemapping-adapter"); } @@ -431,11 +431,11 @@ abstract public class KylinConfigBase implements Serializable { // // Cube Planner // - + public boolean isCubePlannerEnabled() { return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled", "false")); } - + public boolean isCubePlannerEnabledForExistingCube() { return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube", "false")); } @@ -724,23 +724,23 @@ abstract public class KylinConfigBase implements Serializable { // SOURCE.JDBC // -public String getJdbcConnectionUrl() { +public String getJdbcSourceConnectionUrl() { return getOptional("kylin.source.jdbc.connection-url"); } -public String getJdbcDriver() { +public String getJdbcSourceDriver() { return getOptional("kylin.source.jdbc.driver"); }
[3/3] kylin git commit: KYLIN-3044 minor code review
KYLIN-3044 minor code review Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2 Branch: refs/heads/KYLIN-3044 Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d Parents: cac1f8b Author: Li Yang Authored: Sun Nov 26 13:04:27 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 13:04:27 2017 +0800 -- .../org/apache/kylin/common/KylinConfigBase.java | 2 +- .../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++- .../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +- .../apache/kylin/source/jdbc/JdbcHiveMRInput.java| 15 --- .../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +- .../org/apache/kylin/source/jdbc/SqlUtilTest.java| 6 +++--- 6 files changed, 16 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b2368ce..1a93dd4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4")); } -public String getFieldDelimiter() { +public String getSourceFieldDelimiter() { return getOptional("kylin.source.jdbc.field-delimiter", "|"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 0ab3d3d..d136ec6 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -85,7 +85,8 @@ public class JoinedFlatTable { public static String generateCreateTableStatement(IJoinedFlatTableDesc flatDesc, String storageDfsDir, String format) { -return generateCreateTableStatement(flatDesc, storageDfsDir, format, "|"); +String fieldDelimiter = flatDesc.getDataModel().getConfig().getSourceFieldDelimiter(); +return generateCreateTableStatement(flatDesc, storageDfsDir, format, fieldDelimiter); } public static String generateDropTableStatement(IJoinedFlatTableDesc flatDesc) { http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java index 1278128..2827b3d 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java @@ -112,7 +112,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(cname.toUpperCase()); -String kylinType = SqlUtil.jdbcTypetoKylinDataType(type); +String kylinType = SqlUtil.jdbcTypeToKylinDataType(type); int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1; int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1; http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java index e83518a..15259cc 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java @@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc; import java.util.List; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; impor
kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping
Repository: kylin Updated Branches: refs/heads/KYLIN-2966 [created] 0b6b0625d KYLIN-2966 add pushdown jdbc columntype mapping Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0b6b0625 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0b6b0625 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0b6b0625 Branch: refs/heads/KYLIN-2966 Commit: 0b6b0625d83b357ceae7aeab3d3e303d6fdd6f47 Parents: d837e18 Author: zhaiyuyong Authored: Wed Oct 25 17:06:04 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 14:03:11 2017 +0800 -- .../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++- 1 file changed, 44 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/0b6b0625/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java -- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java index 503e273..7283d66 100644 --- a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java +++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java @@ -23,6 +23,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import java.util.LinkedList; import java.util.List; @@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, -metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), +metaData.getPrecision(i), metaData.getScale(i), toSqlType(metaData.getColumnTypeName(i)), metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false)); } } finally { @@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { } } +// calcite does not understand all java SqlTypes, for example LONGNVARCHAR -16, thus need this mapping (KYLIN-2966) +public static int toSqlType(String type) throws SQLException { +if ("string".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("varchar".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("char".equalsIgnoreCase(type)) { +return Types.CHAR; +} else if ("float".equalsIgnoreCase(type)) { +return Types.FLOAT; +} else if ("real".equalsIgnoreCase(type)) { +return Types.REAL; +} else if ("double".equalsIgnoreCase(type)) { +return Types.DOUBLE; +} else if ("boolean".equalsIgnoreCase(type)) { +return Types.BOOLEAN; +} else if ("tinyint".equalsIgnoreCase(type)) { +return Types.TINYINT; +} else if ("smallint".equalsIgnoreCase(type)) { +return Types.SMALLINT; +} else if ("int".equalsIgnoreCase(type)) { +return Types.INTEGER; +} else if ("bigint".equalsIgnoreCase(type)) { +return Types.BIGINT; +} else if ("date".equalsIgnoreCase(type)) { +return Types.DATE; +} else if ("timestamp".equalsIgnoreCase(type)) { +return Types.TIMESTAMP; +} else if ("decimal".equalsIgnoreCase(type)) { +return Types.DECIMAL; +} else if ("binary".equalsIgnoreCase(type)) { +return Types.BINARY; +} else if ("map".equalsIgnoreCase(type)) { +return Types.JAVA_OBJECT; +} else if ("array".equalsIgnoreCase(type)) { +return Types.ARRAY; +} else if ("struct".equalsIgnoreCase(type)) { +return Types.STRUCT; +} +throw new SQLException("Unrecognized column type: " + type); +} + @Override public void executeUpdate(String sql) throws Exception { Statement statement = null;
[3/3] kylin git commit: KYLIN-3044 minor code review
KYLIN-3044 minor code review Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2 Branch: refs/heads/master Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d Parents: cac1f8b Author: Li Yang Authored: Sun Nov 26 13:04:27 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 13:04:27 2017 +0800 -- .../org/apache/kylin/common/KylinConfigBase.java | 2 +- .../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++- .../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +- .../apache/kylin/source/jdbc/JdbcHiveMRInput.java| 15 --- .../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +- .../org/apache/kylin/source/jdbc/SqlUtilTest.java| 6 +++--- 6 files changed, 16 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b2368ce..1a93dd4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4")); } -public String getFieldDelimiter() { +public String getSourceFieldDelimiter() { return getOptional("kylin.source.jdbc.field-delimiter", "|"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 0ab3d3d..d136ec6 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -85,7 +85,8 @@ public class JoinedFlatTable { public static String generateCreateTableStatement(IJoinedFlatTableDesc flatDesc, String storageDfsDir, String format) { -return generateCreateTableStatement(flatDesc, storageDfsDir, format, "|"); +String fieldDelimiter = flatDesc.getDataModel().getConfig().getSourceFieldDelimiter(); +return generateCreateTableStatement(flatDesc, storageDfsDir, format, fieldDelimiter); } public static String generateDropTableStatement(IJoinedFlatTableDesc flatDesc) { http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java index 1278128..2827b3d 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java @@ -112,7 +112,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(cname.toUpperCase()); -String kylinType = SqlUtil.jdbcTypetoKylinDataType(type); +String kylinType = SqlUtil.jdbcTypeToKylinDataType(type); int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1; int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1; http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java index e83518a..15259cc 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java @@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc; import java.util.List; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; import or
kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping [Forced Update!]
Repository: kylin Updated Branches: refs/heads/KYLIN-2966 0b6b0625d -> 85d7e6260 (forced update) KYLIN-2966 add pushdown jdbc columntype mapping This closes #82 Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85d7e626 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85d7e626 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85d7e626 Branch: refs/heads/KYLIN-2966 Commit: 85d7e6260e61ab004186bdd26cfb9543ed3b795d Parents: d837e18 Author: zhaiyuyong Authored: Wed Oct 25 17:06:04 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 14:12:59 2017 +0800 -- .../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++- 1 file changed, 44 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/85d7e626/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java -- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java index 503e273..7283d66 100644 --- a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java +++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java @@ -23,6 +23,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import java.util.LinkedList; import java.util.List; @@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, -metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), +metaData.getPrecision(i), metaData.getScale(i), toSqlType(metaData.getColumnTypeName(i)), metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false)); } } finally { @@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { } } +// calcite does not understand all java SqlTypes, for example LONGNVARCHAR -16, thus need this mapping (KYLIN-2966) +public static int toSqlType(String type) throws SQLException { +if ("string".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("varchar".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("char".equalsIgnoreCase(type)) { +return Types.CHAR; +} else if ("float".equalsIgnoreCase(type)) { +return Types.FLOAT; +} else if ("real".equalsIgnoreCase(type)) { +return Types.REAL; +} else if ("double".equalsIgnoreCase(type)) { +return Types.DOUBLE; +} else if ("boolean".equalsIgnoreCase(type)) { +return Types.BOOLEAN; +} else if ("tinyint".equalsIgnoreCase(type)) { +return Types.TINYINT; +} else if ("smallint".equalsIgnoreCase(type)) { +return Types.SMALLINT; +} else if ("int".equalsIgnoreCase(type)) { +return Types.INTEGER; +} else if ("bigint".equalsIgnoreCase(type)) { +return Types.BIGINT; +} else if ("date".equalsIgnoreCase(type)) { +return Types.DATE; +} else if ("timestamp".equalsIgnoreCase(type)) { +return Types.TIMESTAMP; +} else if ("decimal".equalsIgnoreCase(type)) { +return Types.DECIMAL; +} else if ("binary".equalsIgnoreCase(type)) { +return Types.BINARY; +} else if ("map".equalsIgnoreCase(type)) { +return Types.JAVA_OBJECT; +} else if ("array".equalsIgnoreCase(type)) { +return Types.ARRAY; +} else if ("struct".equalsIgnoreCase(type)) { +return Types.STRUCT; +} +throw new SQLException("Unrecognized column type: " + type); +} + @Override public void executeUpdate(String sql) throws Exception { Statement statement = null;
[2/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
KYLIN-3044 support SQLServer as kylin data source Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cac1f8bb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cac1f8bb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cac1f8bb Branch: refs/heads/master Commit: cac1f8bb48239fe5a49ca720575915f3507a4010 Parents: d837e18 Author: etherge Authored: Wed Nov 22 00:26:49 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 13:03:47 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 31 +- .../main/resources/kylin-defaults.properties| 9 + .../org/apache/kylin/job/JoinedFlatTable.java | 45 +-- pom.xml | 2 +- source-hive/pom.xml | 25 ++ .../apache/kylin/source/jdbc/JdbcDialect.java | 26 ++ .../apache/kylin/source/jdbc/JdbcExplorer.java | 288 --- .../kylin/source/jdbc/JdbcHiveMRInput.java | 129 +++-- .../kylin/source/jdbc/JdbcTableReader.java | 32 ++- .../org/apache/kylin/source/jdbc/SqlUtil.java | 140 ++--- .../jdbc/metadata/DefaultJdbcMetadata.java | 76 + .../source/jdbc/metadata/IJdbcMetadata.java | 33 +++ .../jdbc/metadata/JdbcMetadataFactory.java | 35 +++ .../source/jdbc/metadata/MySQLJdbcMetadata.java | 69 + .../jdbc/metadata/SQLServerJdbcMetadata.java| 61 .../kylin/source/jdbc/JdbcExplorerTest.java | 156 ++ .../apache/kylin/source/jdbc/SqlUtilTest.java | 46 +++ .../jdbc/metadata/DefaultJdbcMetadataTest.java | 126 .../jdbc/metadata/JdbcMetadataFactoryTest.java | 35 +++ .../jdbc/metadata/MySQLJdbcMetadataTest.java| 104 +++ .../metadata/SQLServerJdbcMetadataTest.java | 68 + 21 files changed, 1258 insertions(+), 278 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 3d67ee3..b2368ce 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -287,7 +287,7 @@ abstract public class KylinConfigBase implements Serializable { r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html return r; } - + public String getDataModelImpl() { return getOptional("kylin.metadata.data-model-impl", null); } @@ -295,7 +295,7 @@ abstract public class KylinConfigBase implements Serializable { public String getDataModelManagerImpl() { return getOptional("kylin.metadata.data-model-manager-impl", null); } - + public String[] getRealizationProviders() { return getOptionalStringArray("kylin.metadata.realization-providers", // new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); @@ -314,7 +314,7 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } - + public String getHBaseMappingAdapter() { return getOptional("kylin.metadata.hbasemapping-adapter"); } @@ -431,11 +431,11 @@ abstract public class KylinConfigBase implements Serializable { // // Cube Planner // - + public boolean isCubePlannerEnabled() { return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled", "false")); } - + public boolean isCubePlannerEnabledForExistingCube() { return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube", "false")); } @@ -724,23 +724,23 @@ abstract public class KylinConfigBase implements Serializable { // SOURCE.JDBC // -public String getJdbcConnectionUrl() { +public String getJdbcSourceConnectionUrl() { return getOptional("kylin.source.jdbc.connection-url"); } -public String getJdbcDriver() { +public String getJdbcSourceDriver() { return getOptional("kylin.source.jdbc.driver"); } -
[1/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
Repository: kylin Updated Branches: refs/heads/master d837e18eb -> 695971a2a http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java -- diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java new file mode 100644 index 000..a5516ab --- /dev/null +++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.source.jdbc.metadata; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +import org.apache.kylin.source.hive.DBConnConf; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class SQLServerJdbcMetadataTest extends DefaultJdbcMetadataTest { + +@Before +public void setup() { +dbConnConf = new DBConnConf(); +dbConnConf.setUrl("jdbc:sqlserver://fakehost:1433;database=testdb"); +dbConnConf.setDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver"); +dbConnConf.setUser("user"); +dbConnConf.setPass("pass"); +jdbcMetadata = new SQLServerJdbcMetadata(dbConnConf); + +setupProperties(); +} + +@Test +public void testListDatabases() throws SQLException { +ResultSet rs = mock(ResultSet.class); +when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false); + when(rs.getString("TABLE_SCHEM")).thenReturn("schema1").thenReturn("schema2"); + when(rs.getString("TABLE_CATALOG")).thenReturn("catalog1").thenReturn("testdb"); + +when(connection.getCatalog()).thenReturn("testdb"); +when(connection.getMetaData()).thenReturn(dbmd); +when(dbmd.getSchemas("testdb", "%")).thenReturn(rs); + +List dbs = jdbcMetadata.listDatabases(); + +Assert.assertEquals(1, dbs.size()); +Assert.assertEquals("schema2", dbs.get(0)); +} + +@Test(expected = IllegalArgumentException.class) +public void testListDatabasesWithoutSpecificDB() throws SQLException { +when(connection.getCatalog()).thenReturn(""); +jdbcMetadata.listDatabases(); +} +}
[kylin] Git Push Summary
Repository: kylin Updated Branches: refs/heads/KYLIN-3044 [deleted] 695971a2a
[3/4] kylin git commit: KYLIN-3044 minor code review
KYLIN-3044 minor code review Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2 Branch: refs/heads/KYLIN-2966 Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d Parents: cac1f8b Author: Li Yang Authored: Sun Nov 26 13:04:27 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 13:04:27 2017 +0800 -- .../org/apache/kylin/common/KylinConfigBase.java | 2 +- .../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++- .../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +- .../apache/kylin/source/jdbc/JdbcHiveMRInput.java| 15 --- .../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +- .../org/apache/kylin/source/jdbc/SqlUtilTest.java| 6 +++--- 6 files changed, 16 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b2368ce..1a93dd4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4")); } -public String getFieldDelimiter() { +public String getSourceFieldDelimiter() { return getOptional("kylin.source.jdbc.field-delimiter", "|"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java -- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 0ab3d3d..d136ec6 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -85,7 +85,8 @@ public class JoinedFlatTable { public static String generateCreateTableStatement(IJoinedFlatTableDesc flatDesc, String storageDfsDir, String format) { -return generateCreateTableStatement(flatDesc, storageDfsDir, format, "|"); +String fieldDelimiter = flatDesc.getDataModel().getConfig().getSourceFieldDelimiter(); +return generateCreateTableStatement(flatDesc, storageDfsDir, format, fieldDelimiter); } public static String generateDropTableStatement(IJoinedFlatTableDesc flatDesc) { http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java index 1278128..2827b3d 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java @@ -112,7 +112,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(cname.toUpperCase()); -String kylinType = SqlUtil.jdbcTypetoKylinDataType(type); +String kylinType = SqlUtil.jdbcTypeToKylinDataType(type); int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1; int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1; http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java -- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java index e83518a..15259cc 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java @@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc; import java.util.List; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; impor
[1/4] kylin git commit: KYLIN-3044 support SQLServer as kylin data source [Forced Update!]
Repository: kylin Updated Branches: refs/heads/KYLIN-2966 85d7e6260 -> b40d0e06f (forced update) http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java -- diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java new file mode 100644 index 000..a5516ab --- /dev/null +++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.source.jdbc.metadata; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +import org.apache.kylin.source.hive.DBConnConf; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class SQLServerJdbcMetadataTest extends DefaultJdbcMetadataTest { + +@Before +public void setup() { +dbConnConf = new DBConnConf(); +dbConnConf.setUrl("jdbc:sqlserver://fakehost:1433;database=testdb"); +dbConnConf.setDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver"); +dbConnConf.setUser("user"); +dbConnConf.setPass("pass"); +jdbcMetadata = new SQLServerJdbcMetadata(dbConnConf); + +setupProperties(); +} + +@Test +public void testListDatabases() throws SQLException { +ResultSet rs = mock(ResultSet.class); +when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false); + when(rs.getString("TABLE_SCHEM")).thenReturn("schema1").thenReturn("schema2"); + when(rs.getString("TABLE_CATALOG")).thenReturn("catalog1").thenReturn("testdb"); + +when(connection.getCatalog()).thenReturn("testdb"); +when(connection.getMetaData()).thenReturn(dbmd); +when(dbmd.getSchemas("testdb", "%")).thenReturn(rs); + +List dbs = jdbcMetadata.listDatabases(); + +Assert.assertEquals(1, dbs.size()); +Assert.assertEquals("schema2", dbs.get(0)); +} + +@Test(expected = IllegalArgumentException.class) +public void testListDatabasesWithoutSpecificDB() throws SQLException { +when(connection.getCatalog()).thenReturn(""); +jdbcMetadata.listDatabases(); +} +}
[2/4] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
KYLIN-3044 support SQLServer as kylin data source Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cac1f8bb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cac1f8bb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cac1f8bb Branch: refs/heads/KYLIN-2966 Commit: cac1f8bb48239fe5a49ca720575915f3507a4010 Parents: d837e18 Author: etherge Authored: Wed Nov 22 00:26:49 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 13:03:47 2017 +0800 -- .../apache/kylin/common/KylinConfigBase.java| 31 +- .../main/resources/kylin-defaults.properties| 9 + .../org/apache/kylin/job/JoinedFlatTable.java | 45 +-- pom.xml | 2 +- source-hive/pom.xml | 25 ++ .../apache/kylin/source/jdbc/JdbcDialect.java | 26 ++ .../apache/kylin/source/jdbc/JdbcExplorer.java | 288 --- .../kylin/source/jdbc/JdbcHiveMRInput.java | 129 +++-- .../kylin/source/jdbc/JdbcTableReader.java | 32 ++- .../org/apache/kylin/source/jdbc/SqlUtil.java | 140 ++--- .../jdbc/metadata/DefaultJdbcMetadata.java | 76 + .../source/jdbc/metadata/IJdbcMetadata.java | 33 +++ .../jdbc/metadata/JdbcMetadataFactory.java | 35 +++ .../source/jdbc/metadata/MySQLJdbcMetadata.java | 69 + .../jdbc/metadata/SQLServerJdbcMetadata.java| 61 .../kylin/source/jdbc/JdbcExplorerTest.java | 156 ++ .../apache/kylin/source/jdbc/SqlUtilTest.java | 46 +++ .../jdbc/metadata/DefaultJdbcMetadataTest.java | 126 .../jdbc/metadata/JdbcMetadataFactoryTest.java | 35 +++ .../jdbc/metadata/MySQLJdbcMetadataTest.java| 104 +++ .../metadata/SQLServerJdbcMetadataTest.java | 68 + 21 files changed, 1258 insertions(+), 278 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 3d67ee3..b2368ce 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -287,7 +287,7 @@ abstract public class KylinConfigBase implements Serializable { r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html return r; } - + public String getDataModelImpl() { return getOptional("kylin.metadata.data-model-impl", null); } @@ -295,7 +295,7 @@ abstract public class KylinConfigBase implements Serializable { public String getDataModelManagerImpl() { return getOptional("kylin.metadata.data-model-manager-impl", null); } - + public String[] getRealizationProviders() { return getOptionalStringArray("kylin.metadata.realization-providers", // new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); @@ -314,7 +314,7 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } - + public String getHBaseMappingAdapter() { return getOptional("kylin.metadata.hbasemapping-adapter"); } @@ -431,11 +431,11 @@ abstract public class KylinConfigBase implements Serializable { // // Cube Planner // - + public boolean isCubePlannerEnabled() { return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled", "false")); } - + public boolean isCubePlannerEnabledForExistingCube() { return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube", "false")); } @@ -724,23 +724,23 @@ abstract public class KylinConfigBase implements Serializable { // SOURCE.JDBC // -public String getJdbcConnectionUrl() { +public String getJdbcSourceConnectionUrl() { return getOptional("kylin.source.jdbc.connection-url"); } -public String getJdbcDriver() { +public String getJdbcSourceDriver() { return getOptional("kylin.source.jdbc.driver"); }
[4/4] kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping
KYLIN-2966 add pushdown jdbc columntype mapping This closes #82 Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b40d0e06 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b40d0e06 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b40d0e06 Branch: refs/heads/KYLIN-2966 Commit: b40d0e06f8dddf0916a40eb01a49417d4b65826b Parents: 695971a Author: zhaiyuyong Authored: Wed Oct 25 17:06:04 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 14:16:04 2017 +0800 -- .../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++- 1 file changed, 44 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/b40d0e06/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java -- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java index 503e273..7283d66 100644 --- a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java +++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java @@ -23,6 +23,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import java.util.LinkedList; import java.util.List; @@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, -metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), +metaData.getPrecision(i), metaData.getScale(i), toSqlType(metaData.getColumnTypeName(i)), metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false)); } } finally { @@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { } } +// calcite does not understand all java SqlTypes, for example LONGNVARCHAR -16, thus need this mapping (KYLIN-2966) +public static int toSqlType(String type) throws SQLException { +if ("string".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("varchar".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("char".equalsIgnoreCase(type)) { +return Types.CHAR; +} else if ("float".equalsIgnoreCase(type)) { +return Types.FLOAT; +} else if ("real".equalsIgnoreCase(type)) { +return Types.REAL; +} else if ("double".equalsIgnoreCase(type)) { +return Types.DOUBLE; +} else if ("boolean".equalsIgnoreCase(type)) { +return Types.BOOLEAN; +} else if ("tinyint".equalsIgnoreCase(type)) { +return Types.TINYINT; +} else if ("smallint".equalsIgnoreCase(type)) { +return Types.SMALLINT; +} else if ("int".equalsIgnoreCase(type)) { +return Types.INTEGER; +} else if ("bigint".equalsIgnoreCase(type)) { +return Types.BIGINT; +} else if ("date".equalsIgnoreCase(type)) { +return Types.DATE; +} else if ("timestamp".equalsIgnoreCase(type)) { +return Types.TIMESTAMP; +} else if ("decimal".equalsIgnoreCase(type)) { +return Types.DECIMAL; +} else if ("binary".equalsIgnoreCase(type)) { +return Types.BINARY; +} else if ("map".equalsIgnoreCase(type)) { +return Types.JAVA_OBJECT; +} else if ("array".equalsIgnoreCase(type)) { +return Types.ARRAY; +} else if ("struct".equalsIgnoreCase(type)) { +return Types.STRUCT; +} +throw new SQLException("Unrecognized column type: " + type); +} + @Override public void executeUpdate(String sql) throws Exception { Statement statement = null;
[kylin] Git Push Summary
Repository: kylin Updated Branches: refs/heads/KYLIN-2966 [deleted] b40d0e06f
kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping
Repository: kylin Updated Branches: refs/heads/master 695971a2a -> b40d0e06f KYLIN-2966 add pushdown jdbc columntype mapping This closes #82 Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b40d0e06 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b40d0e06 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b40d0e06 Branch: refs/heads/master Commit: b40d0e06f8dddf0916a40eb01a49417d4b65826b Parents: 695971a Author: zhaiyuyong Authored: Wed Oct 25 17:06:04 2017 +0800 Committer: Li Yang Committed: Sun Nov 26 14:16:04 2017 +0800 -- .../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++- 1 file changed, 44 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/b40d0e06/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java -- diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java index 503e273..7283d66 100644 --- a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java +++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java @@ -23,6 +23,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import java.util.LinkedList; import java.util.List; @@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, -metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), +metaData.getPrecision(i), metaData.getScale(i), toSqlType(metaData.getColumnTypeName(i)), metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false)); } } finally { @@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner { } } +// calcite does not understand all java SqlTypes, for example LONGNVARCHAR -16, thus need this mapping (KYLIN-2966) +public static int toSqlType(String type) throws SQLException { +if ("string".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("varchar".equalsIgnoreCase(type)) { +return Types.VARCHAR; +} else if ("char".equalsIgnoreCase(type)) { +return Types.CHAR; +} else if ("float".equalsIgnoreCase(type)) { +return Types.FLOAT; +} else if ("real".equalsIgnoreCase(type)) { +return Types.REAL; +} else if ("double".equalsIgnoreCase(type)) { +return Types.DOUBLE; +} else if ("boolean".equalsIgnoreCase(type)) { +return Types.BOOLEAN; +} else if ("tinyint".equalsIgnoreCase(type)) { +return Types.TINYINT; +} else if ("smallint".equalsIgnoreCase(type)) { +return Types.SMALLINT; +} else if ("int".equalsIgnoreCase(type)) { +return Types.INTEGER; +} else if ("bigint".equalsIgnoreCase(type)) { +return Types.BIGINT; +} else if ("date".equalsIgnoreCase(type)) { +return Types.DATE; +} else if ("timestamp".equalsIgnoreCase(type)) { +return Types.TIMESTAMP; +} else if ("decimal".equalsIgnoreCase(type)) { +return Types.DECIMAL; +} else if ("binary".equalsIgnoreCase(type)) { +return Types.BINARY; +} else if ("map".equalsIgnoreCase(type)) { +return Types.JAVA_OBJECT; +} else if ("array".equalsIgnoreCase(type)) { +return Types.ARRAY; +} else if ("struct".equalsIgnoreCase(type)) { +return Types.STRUCT; +} +throw new SQLException("Unrecognized column type: " + type); +} + @Override public void executeUpdate(String sql) throws Exception { Statement statement = null;