[1/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
Repository: kylin
Updated Branches:
refs/heads/KYLIN-3044 [created] 695971a2a
http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
--
diff --git
a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
new file mode 100644
index 000..a5516ab
--- /dev/null
+++
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.source.jdbc.metadata;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.kylin.source.hive.DBConnConf;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SQLServerJdbcMetadataTest extends DefaultJdbcMetadataTest {
+
+@Before
+public void setup() {
+dbConnConf = new DBConnConf();
+dbConnConf.setUrl("jdbc:sqlserver://fakehost:1433;database=testdb");
+dbConnConf.setDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver");
+dbConnConf.setUser("user");
+dbConnConf.setPass("pass");
+jdbcMetadata = new SQLServerJdbcMetadata(dbConnConf);
+
+setupProperties();
+}
+
+@Test
+public void testListDatabases() throws SQLException {
+ResultSet rs = mock(ResultSet.class);
+when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(rs.getString("TABLE_SCHEM")).thenReturn("schema1").thenReturn("schema2");
+
when(rs.getString("TABLE_CATALOG")).thenReturn("catalog1").thenReturn("testdb");
+
+when(connection.getCatalog()).thenReturn("testdb");
+when(connection.getMetaData()).thenReturn(dbmd);
+when(dbmd.getSchemas("testdb", "%")).thenReturn(rs);
+
+List dbs = jdbcMetadata.listDatabases();
+
+Assert.assertEquals(1, dbs.size());
+Assert.assertEquals("schema2", dbs.get(0));
+}
+
+@Test(expected = IllegalArgumentException.class)
+public void testListDatabasesWithoutSpecificDB() throws SQLException {
+when(connection.getCatalog()).thenReturn("");
+jdbcMetadata.listDatabases();
+}
+}
[2/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
KYLIN-3044 support SQLServer as kylin data source
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cac1f8bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cac1f8bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cac1f8bb
Branch: refs/heads/KYLIN-3044
Commit: cac1f8bb48239fe5a49ca720575915f3507a4010
Parents: d837e18
Author: etherge
Authored: Wed Nov 22 00:26:49 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 13:03:47 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 31 +-
.../main/resources/kylin-defaults.properties| 9 +
.../org/apache/kylin/job/JoinedFlatTable.java | 45 +--
pom.xml | 2 +-
source-hive/pom.xml | 25 ++
.../apache/kylin/source/jdbc/JdbcDialect.java | 26 ++
.../apache/kylin/source/jdbc/JdbcExplorer.java | 288 ---
.../kylin/source/jdbc/JdbcHiveMRInput.java | 129 +++--
.../kylin/source/jdbc/JdbcTableReader.java | 32 ++-
.../org/apache/kylin/source/jdbc/SqlUtil.java | 140 ++---
.../jdbc/metadata/DefaultJdbcMetadata.java | 76 +
.../source/jdbc/metadata/IJdbcMetadata.java | 33 +++
.../jdbc/metadata/JdbcMetadataFactory.java | 35 +++
.../source/jdbc/metadata/MySQLJdbcMetadata.java | 69 +
.../jdbc/metadata/SQLServerJdbcMetadata.java| 61
.../kylin/source/jdbc/JdbcExplorerTest.java | 156 ++
.../apache/kylin/source/jdbc/SqlUtilTest.java | 46 +++
.../jdbc/metadata/DefaultJdbcMetadataTest.java | 126
.../jdbc/metadata/JdbcMetadataFactoryTest.java | 35 +++
.../jdbc/metadata/MySQLJdbcMetadataTest.java| 104 +++
.../metadata/SQLServerJdbcMetadataTest.java | 68 +
21 files changed, 1258 insertions(+), 278 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3d67ee3..b2368ce 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -287,7 +287,7 @@ abstract public class KylinConfigBase implements
Serializable {
r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); //
note the naming convention --
http://kylin.apache.org/development/coding_naming_convention.html
return r;
}
-
+
public String getDataModelImpl() {
return getOptional("kylin.metadata.data-model-impl", null);
}
@@ -295,7 +295,7 @@ abstract public class KylinConfigBase implements
Serializable {
public String getDataModelManagerImpl() {
return getOptional("kylin.metadata.data-model-manager-impl", null);
}
-
+
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.metadata.realization-providers",
//
new String[] { "org.apache.kylin.cube.CubeManager",
"org.apache.kylin.storage.hybrid.HybridManager" });
@@ -314,7 +314,7 @@ abstract public class KylinConfigBase implements
Serializable {
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
-
+
public String getHBaseMappingAdapter() {
return getOptional("kylin.metadata.hbasemapping-adapter");
}
@@ -431,11 +431,11 @@ abstract public class KylinConfigBase implements
Serializable {
//
// Cube Planner
//
-
+
public boolean isCubePlannerEnabled() {
return
Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled", "false"));
}
-
+
public boolean isCubePlannerEnabledForExistingCube() {
return
Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube",
"false"));
}
@@ -724,23 +724,23 @@ abstract public class KylinConfigBase implements
Serializable {
// SOURCE.JDBC
//
-public String getJdbcConnectionUrl() {
+public String getJdbcSourceConnectionUrl() {
return getOptional("kylin.source.jdbc.connection-url");
}
-public String getJdbcDriver() {
+public String getJdbcSourceDriver() {
return getOptional("kylin.source.jdbc.driver");
}
[3/3] kylin git commit: KYLIN-3044 minor code review
KYLIN-3044 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2
Branch: refs/heads/KYLIN-3044
Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d
Parents: cac1f8b
Author: Li Yang
Authored: Sun Nov 26 13:04:27 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 13:04:27 2017 +0800
--
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++-
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +-
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java| 15 ---
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +-
.../org/apache/kylin/source/jdbc/SqlUtilTest.java| 6 +++---
6 files changed, 16 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b2368ce..1a93dd4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements
Serializable {
return
Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4"));
}
-public String getFieldDelimiter() {
+public String getSourceFieldDelimiter() {
return getOptional("kylin.source.jdbc.field-delimiter", "|");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 0ab3d3d..d136ec6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -85,7 +85,8 @@ public class JoinedFlatTable {
public static String generateCreateTableStatement(IJoinedFlatTableDesc
flatDesc, String storageDfsDir,
String format) {
-return generateCreateTableStatement(flatDesc, storageDfsDir, format,
"|");
+String fieldDelimiter =
flatDesc.getDataModel().getConfig().getSourceFieldDelimiter();
+return generateCreateTableStatement(flatDesc, storageDfsDir, format,
fieldDelimiter);
}
public static String generateDropTableStatement(IJoinedFlatTableDesc
flatDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 1278128..2827b3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -112,7 +112,7 @@ public class JdbcExplorer implements
ISourceMetadataExplorer, ISampleDataDeploye
ColumnDesc cdesc = new ColumnDesc();
cdesc.setName(cname.toUpperCase());
-String kylinType = SqlUtil.jdbcTypetoKylinDataType(type);
+String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
int precision = (SqlUtil.isPrecisionApplicable(kylinType) &&
csize > 0) ? csize : -1;
int scale = (SqlUtil.isScaleApplicable(kylinType) && digits >
0) ? digits : -1;
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
index e83518a..15259cc 100644
---
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
+++
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
@@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc;
import java.util.List;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
impor
kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2966 [created] 0b6b0625d
KYLIN-2966 add pushdown jdbc columntype mapping
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0b6b0625
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0b6b0625
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0b6b0625
Branch: refs/heads/KYLIN-2966
Commit: 0b6b0625d83b357ceae7aeab3d3e303d6fdd6f47
Parents: d837e18
Author: zhaiyuyong
Authored: Wed Oct 25 17:06:04 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 14:03:11 2017 +0800
--
.../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++-
1 file changed, 44 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0b6b0625/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index 503e273..7283d66 100644
---
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
+++
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -23,6 +23,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Types;
import java.util.LinkedList;
import java.util.List;
@@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
columnMetas.add(new
SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i),
false,
metaData.isCurrency(i), metaData.isNullable(i), false,
metaData.getColumnDisplaySize(i),
metaData.getColumnLabel(i), metaData.getColumnName(i),
null, null, null,
-metaData.getPrecision(i), metaData.getScale(i),
metaData.getColumnType(i),
+metaData.getPrecision(i), metaData.getScale(i),
toSqlType(metaData.getColumnTypeName(i)),
metaData.getColumnTypeName(i), metaData.isReadOnly(i),
false, false));
}
} finally {
@@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
}
}
+// calcite does not understand all java SqlTypes, for example LONGNVARCHAR
-16, thus need this mapping (KYLIN-2966)
+public static int toSqlType(String type) throws SQLException {
+if ("string".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("varchar".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("char".equalsIgnoreCase(type)) {
+return Types.CHAR;
+} else if ("float".equalsIgnoreCase(type)) {
+return Types.FLOAT;
+} else if ("real".equalsIgnoreCase(type)) {
+return Types.REAL;
+} else if ("double".equalsIgnoreCase(type)) {
+return Types.DOUBLE;
+} else if ("boolean".equalsIgnoreCase(type)) {
+return Types.BOOLEAN;
+} else if ("tinyint".equalsIgnoreCase(type)) {
+return Types.TINYINT;
+} else if ("smallint".equalsIgnoreCase(type)) {
+return Types.SMALLINT;
+} else if ("int".equalsIgnoreCase(type)) {
+return Types.INTEGER;
+} else if ("bigint".equalsIgnoreCase(type)) {
+return Types.BIGINT;
+} else if ("date".equalsIgnoreCase(type)) {
+return Types.DATE;
+} else if ("timestamp".equalsIgnoreCase(type)) {
+return Types.TIMESTAMP;
+} else if ("decimal".equalsIgnoreCase(type)) {
+return Types.DECIMAL;
+} else if ("binary".equalsIgnoreCase(type)) {
+return Types.BINARY;
+} else if ("map".equalsIgnoreCase(type)) {
+return Types.JAVA_OBJECT;
+} else if ("array".equalsIgnoreCase(type)) {
+return Types.ARRAY;
+} else if ("struct".equalsIgnoreCase(type)) {
+return Types.STRUCT;
+}
+throw new SQLException("Unrecognized column type: " + type);
+}
+
@Override
public void executeUpdate(String sql) throws Exception {
Statement statement = null;
[3/3] kylin git commit: KYLIN-3044 minor code review
KYLIN-3044 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2
Branch: refs/heads/master
Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d
Parents: cac1f8b
Author: Li Yang
Authored: Sun Nov 26 13:04:27 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 13:04:27 2017 +0800
--
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++-
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +-
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java| 15 ---
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +-
.../org/apache/kylin/source/jdbc/SqlUtilTest.java| 6 +++---
6 files changed, 16 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b2368ce..1a93dd4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements
Serializable {
return
Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4"));
}
-public String getFieldDelimiter() {
+public String getSourceFieldDelimiter() {
return getOptional("kylin.source.jdbc.field-delimiter", "|");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 0ab3d3d..d136ec6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -85,7 +85,8 @@ public class JoinedFlatTable {
public static String generateCreateTableStatement(IJoinedFlatTableDesc
flatDesc, String storageDfsDir,
String format) {
-return generateCreateTableStatement(flatDesc, storageDfsDir, format,
"|");
+String fieldDelimiter =
flatDesc.getDataModel().getConfig().getSourceFieldDelimiter();
+return generateCreateTableStatement(flatDesc, storageDfsDir, format,
fieldDelimiter);
}
public static String generateDropTableStatement(IJoinedFlatTableDesc
flatDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 1278128..2827b3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -112,7 +112,7 @@ public class JdbcExplorer implements
ISourceMetadataExplorer, ISampleDataDeploye
ColumnDesc cdesc = new ColumnDesc();
cdesc.setName(cname.toUpperCase());
-String kylinType = SqlUtil.jdbcTypetoKylinDataType(type);
+String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
int precision = (SqlUtil.isPrecisionApplicable(kylinType) &&
csize > 0) ? csize : -1;
int scale = (SqlUtil.isScaleApplicable(kylinType) && digits >
0) ? digits : -1;
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
index e83518a..15259cc 100644
---
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
+++
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
@@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc;
import java.util.List;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import or
kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2966 0b6b0625d -> 85d7e6260 (forced update)
KYLIN-2966 add pushdown jdbc columntype mapping
This closes #82
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85d7e626
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85d7e626
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85d7e626
Branch: refs/heads/KYLIN-2966
Commit: 85d7e6260e61ab004186bdd26cfb9543ed3b795d
Parents: d837e18
Author: zhaiyuyong
Authored: Wed Oct 25 17:06:04 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 14:12:59 2017 +0800
--
.../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++-
1 file changed, 44 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/85d7e626/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index 503e273..7283d66 100644
---
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
+++
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -23,6 +23,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Types;
import java.util.LinkedList;
import java.util.List;
@@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
columnMetas.add(new
SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i),
false,
metaData.isCurrency(i), metaData.isNullable(i), false,
metaData.getColumnDisplaySize(i),
metaData.getColumnLabel(i), metaData.getColumnName(i),
null, null, null,
-metaData.getPrecision(i), metaData.getScale(i),
metaData.getColumnType(i),
+metaData.getPrecision(i), metaData.getScale(i),
toSqlType(metaData.getColumnTypeName(i)),
metaData.getColumnTypeName(i), metaData.isReadOnly(i),
false, false));
}
} finally {
@@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
}
}
+// calcite does not understand all java SqlTypes, for example LONGNVARCHAR
-16, thus need this mapping (KYLIN-2966)
+public static int toSqlType(String type) throws SQLException {
+if ("string".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("varchar".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("char".equalsIgnoreCase(type)) {
+return Types.CHAR;
+} else if ("float".equalsIgnoreCase(type)) {
+return Types.FLOAT;
+} else if ("real".equalsIgnoreCase(type)) {
+return Types.REAL;
+} else if ("double".equalsIgnoreCase(type)) {
+return Types.DOUBLE;
+} else if ("boolean".equalsIgnoreCase(type)) {
+return Types.BOOLEAN;
+} else if ("tinyint".equalsIgnoreCase(type)) {
+return Types.TINYINT;
+} else if ("smallint".equalsIgnoreCase(type)) {
+return Types.SMALLINT;
+} else if ("int".equalsIgnoreCase(type)) {
+return Types.INTEGER;
+} else if ("bigint".equalsIgnoreCase(type)) {
+return Types.BIGINT;
+} else if ("date".equalsIgnoreCase(type)) {
+return Types.DATE;
+} else if ("timestamp".equalsIgnoreCase(type)) {
+return Types.TIMESTAMP;
+} else if ("decimal".equalsIgnoreCase(type)) {
+return Types.DECIMAL;
+} else if ("binary".equalsIgnoreCase(type)) {
+return Types.BINARY;
+} else if ("map".equalsIgnoreCase(type)) {
+return Types.JAVA_OBJECT;
+} else if ("array".equalsIgnoreCase(type)) {
+return Types.ARRAY;
+} else if ("struct".equalsIgnoreCase(type)) {
+return Types.STRUCT;
+}
+throw new SQLException("Unrecognized column type: " + type);
+}
+
@Override
public void executeUpdate(String sql) throws Exception {
Statement statement = null;
[2/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
KYLIN-3044 support SQLServer as kylin data source
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cac1f8bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cac1f8bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cac1f8bb
Branch: refs/heads/master
Commit: cac1f8bb48239fe5a49ca720575915f3507a4010
Parents: d837e18
Author: etherge
Authored: Wed Nov 22 00:26:49 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 13:03:47 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 31 +-
.../main/resources/kylin-defaults.properties| 9 +
.../org/apache/kylin/job/JoinedFlatTable.java | 45 +--
pom.xml | 2 +-
source-hive/pom.xml | 25 ++
.../apache/kylin/source/jdbc/JdbcDialect.java | 26 ++
.../apache/kylin/source/jdbc/JdbcExplorer.java | 288 ---
.../kylin/source/jdbc/JdbcHiveMRInput.java | 129 +++--
.../kylin/source/jdbc/JdbcTableReader.java | 32 ++-
.../org/apache/kylin/source/jdbc/SqlUtil.java | 140 ++---
.../jdbc/metadata/DefaultJdbcMetadata.java | 76 +
.../source/jdbc/metadata/IJdbcMetadata.java | 33 +++
.../jdbc/metadata/JdbcMetadataFactory.java | 35 +++
.../source/jdbc/metadata/MySQLJdbcMetadata.java | 69 +
.../jdbc/metadata/SQLServerJdbcMetadata.java| 61
.../kylin/source/jdbc/JdbcExplorerTest.java | 156 ++
.../apache/kylin/source/jdbc/SqlUtilTest.java | 46 +++
.../jdbc/metadata/DefaultJdbcMetadataTest.java | 126
.../jdbc/metadata/JdbcMetadataFactoryTest.java | 35 +++
.../jdbc/metadata/MySQLJdbcMetadataTest.java| 104 +++
.../metadata/SQLServerJdbcMetadataTest.java | 68 +
21 files changed, 1258 insertions(+), 278 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3d67ee3..b2368ce 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -287,7 +287,7 @@ abstract public class KylinConfigBase implements
Serializable {
r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); //
note the naming convention --
http://kylin.apache.org/development/coding_naming_convention.html
return r;
}
-
+
public String getDataModelImpl() {
return getOptional("kylin.metadata.data-model-impl", null);
}
@@ -295,7 +295,7 @@ abstract public class KylinConfigBase implements
Serializable {
public String getDataModelManagerImpl() {
return getOptional("kylin.metadata.data-model-manager-impl", null);
}
-
+
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.metadata.realization-providers",
//
new String[] { "org.apache.kylin.cube.CubeManager",
"org.apache.kylin.storage.hybrid.HybridManager" });
@@ -314,7 +314,7 @@ abstract public class KylinConfigBase implements
Serializable {
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
-
+
public String getHBaseMappingAdapter() {
return getOptional("kylin.metadata.hbasemapping-adapter");
}
@@ -431,11 +431,11 @@ abstract public class KylinConfigBase implements
Serializable {
//
// Cube Planner
//
-
+
public boolean isCubePlannerEnabled() {
return
Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled", "false"));
}
-
+
public boolean isCubePlannerEnabledForExistingCube() {
return
Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube",
"false"));
}
@@ -724,23 +724,23 @@ abstract public class KylinConfigBase implements
Serializable {
// SOURCE.JDBC
//
-public String getJdbcConnectionUrl() {
+public String getJdbcSourceConnectionUrl() {
return getOptional("kylin.source.jdbc.connection-url");
}
-public String getJdbcDriver() {
+public String getJdbcSourceDriver() {
return getOptional("kylin.source.jdbc.driver");
}
-
[1/3] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
Repository: kylin
Updated Branches:
refs/heads/master d837e18eb -> 695971a2a
http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
--
diff --git
a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
new file mode 100644
index 000..a5516ab
--- /dev/null
+++
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.source.jdbc.metadata;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.kylin.source.hive.DBConnConf;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SQLServerJdbcMetadataTest extends DefaultJdbcMetadataTest {
+
+@Before
+public void setup() {
+dbConnConf = new DBConnConf();
+dbConnConf.setUrl("jdbc:sqlserver://fakehost:1433;database=testdb");
+dbConnConf.setDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver");
+dbConnConf.setUser("user");
+dbConnConf.setPass("pass");
+jdbcMetadata = new SQLServerJdbcMetadata(dbConnConf);
+
+setupProperties();
+}
+
+@Test
+public void testListDatabases() throws SQLException {
+ResultSet rs = mock(ResultSet.class);
+when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(rs.getString("TABLE_SCHEM")).thenReturn("schema1").thenReturn("schema2");
+
when(rs.getString("TABLE_CATALOG")).thenReturn("catalog1").thenReturn("testdb");
+
+when(connection.getCatalog()).thenReturn("testdb");
+when(connection.getMetaData()).thenReturn(dbmd);
+when(dbmd.getSchemas("testdb", "%")).thenReturn(rs);
+
+List dbs = jdbcMetadata.listDatabases();
+
+Assert.assertEquals(1, dbs.size());
+Assert.assertEquals("schema2", dbs.get(0));
+}
+
+@Test(expected = IllegalArgumentException.class)
+public void testListDatabasesWithoutSpecificDB() throws SQLException {
+when(connection.getCatalog()).thenReturn("");
+jdbcMetadata.listDatabases();
+}
+}
[kylin] Git Push Summary
Repository: kylin Updated Branches: refs/heads/KYLIN-3044 [deleted] 695971a2a
[3/4] kylin git commit: KYLIN-3044 minor code review
KYLIN-3044 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2
Branch: refs/heads/KYLIN-2966
Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d
Parents: cac1f8b
Author: Li Yang
Authored: Sun Nov 26 13:04:27 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 13:04:27 2017 +0800
--
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++-
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +-
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java| 15 ---
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +-
.../org/apache/kylin/source/jdbc/SqlUtilTest.java| 6 +++---
6 files changed, 16 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b2368ce..1a93dd4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements
Serializable {
return
Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4"));
}
-public String getFieldDelimiter() {
+public String getSourceFieldDelimiter() {
return getOptional("kylin.source.jdbc.field-delimiter", "|");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 0ab3d3d..d136ec6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -85,7 +85,8 @@ public class JoinedFlatTable {
public static String generateCreateTableStatement(IJoinedFlatTableDesc
flatDesc, String storageDfsDir,
String format) {
-return generateCreateTableStatement(flatDesc, storageDfsDir, format,
"|");
+String fieldDelimiter =
flatDesc.getDataModel().getConfig().getSourceFieldDelimiter();
+return generateCreateTableStatement(flatDesc, storageDfsDir, format,
fieldDelimiter);
}
public static String generateDropTableStatement(IJoinedFlatTableDesc
flatDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 1278128..2827b3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -112,7 +112,7 @@ public class JdbcExplorer implements
ISourceMetadataExplorer, ISampleDataDeploye
ColumnDesc cdesc = new ColumnDesc();
cdesc.setName(cname.toUpperCase());
-String kylinType = SqlUtil.jdbcTypetoKylinDataType(type);
+String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
int precision = (SqlUtil.isPrecisionApplicable(kylinType) &&
csize > 0) ? csize : -1;
int scale = (SqlUtil.isScaleApplicable(kylinType) && digits >
0) ? digits : -1;
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
index e83518a..15259cc 100644
---
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
+++
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
@@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc;
import java.util.List;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
impor
[1/4] kylin git commit: KYLIN-3044 support SQLServer as kylin data source [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2966 85d7e6260 -> b40d0e06f (forced update)
http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
--
diff --git
a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
new file mode 100644
index 000..a5516ab
--- /dev/null
+++
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.source.jdbc.metadata;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.kylin.source.hive.DBConnConf;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SQLServerJdbcMetadataTest extends DefaultJdbcMetadataTest {
+
+@Before
+public void setup() {
+dbConnConf = new DBConnConf();
+dbConnConf.setUrl("jdbc:sqlserver://fakehost:1433;database=testdb");
+dbConnConf.setDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver");
+dbConnConf.setUser("user");
+dbConnConf.setPass("pass");
+jdbcMetadata = new SQLServerJdbcMetadata(dbConnConf);
+
+setupProperties();
+}
+
+@Test
+public void testListDatabases() throws SQLException {
+ResultSet rs = mock(ResultSet.class);
+when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(rs.getString("TABLE_SCHEM")).thenReturn("schema1").thenReturn("schema2");
+
when(rs.getString("TABLE_CATALOG")).thenReturn("catalog1").thenReturn("testdb");
+
+when(connection.getCatalog()).thenReturn("testdb");
+when(connection.getMetaData()).thenReturn(dbmd);
+when(dbmd.getSchemas("testdb", "%")).thenReturn(rs);
+
+List dbs = jdbcMetadata.listDatabases();
+
+Assert.assertEquals(1, dbs.size());
+Assert.assertEquals("schema2", dbs.get(0));
+}
+
+@Test(expected = IllegalArgumentException.class)
+public void testListDatabasesWithoutSpecificDB() throws SQLException {
+when(connection.getCatalog()).thenReturn("");
+jdbcMetadata.listDatabases();
+}
+}
[2/4] kylin git commit: KYLIN-3044 support SQLServer as kylin data source
KYLIN-3044 support SQLServer as kylin data source
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cac1f8bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cac1f8bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cac1f8bb
Branch: refs/heads/KYLIN-2966
Commit: cac1f8bb48239fe5a49ca720575915f3507a4010
Parents: d837e18
Author: etherge
Authored: Wed Nov 22 00:26:49 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 13:03:47 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 31 +-
.../main/resources/kylin-defaults.properties| 9 +
.../org/apache/kylin/job/JoinedFlatTable.java | 45 +--
pom.xml | 2 +-
source-hive/pom.xml | 25 ++
.../apache/kylin/source/jdbc/JdbcDialect.java | 26 ++
.../apache/kylin/source/jdbc/JdbcExplorer.java | 288 ---
.../kylin/source/jdbc/JdbcHiveMRInput.java | 129 +++--
.../kylin/source/jdbc/JdbcTableReader.java | 32 ++-
.../org/apache/kylin/source/jdbc/SqlUtil.java | 140 ++---
.../jdbc/metadata/DefaultJdbcMetadata.java | 76 +
.../source/jdbc/metadata/IJdbcMetadata.java | 33 +++
.../jdbc/metadata/JdbcMetadataFactory.java | 35 +++
.../source/jdbc/metadata/MySQLJdbcMetadata.java | 69 +
.../jdbc/metadata/SQLServerJdbcMetadata.java| 61
.../kylin/source/jdbc/JdbcExplorerTest.java | 156 ++
.../apache/kylin/source/jdbc/SqlUtilTest.java | 46 +++
.../jdbc/metadata/DefaultJdbcMetadataTest.java | 126
.../jdbc/metadata/JdbcMetadataFactoryTest.java | 35 +++
.../jdbc/metadata/MySQLJdbcMetadataTest.java| 104 +++
.../metadata/SQLServerJdbcMetadataTest.java | 68 +
21 files changed, 1258 insertions(+), 278 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/cac1f8bb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3d67ee3..b2368ce 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -287,7 +287,7 @@ abstract public class KylinConfigBase implements
Serializable {
r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); //
note the naming convention --
http://kylin.apache.org/development/coding_naming_convention.html
return r;
}
-
+
public String getDataModelImpl() {
return getOptional("kylin.metadata.data-model-impl", null);
}
@@ -295,7 +295,7 @@ abstract public class KylinConfigBase implements
Serializable {
public String getDataModelManagerImpl() {
return getOptional("kylin.metadata.data-model-manager-impl", null);
}
-
+
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.metadata.realization-providers",
//
new String[] { "org.apache.kylin.cube.CubeManager",
"org.apache.kylin.storage.hybrid.HybridManager" });
@@ -314,7 +314,7 @@ abstract public class KylinConfigBase implements
Serializable {
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
-
+
public String getHBaseMappingAdapter() {
return getOptional("kylin.metadata.hbasemapping-adapter");
}
@@ -431,11 +431,11 @@ abstract public class KylinConfigBase implements
Serializable {
//
// Cube Planner
//
-
+
public boolean isCubePlannerEnabled() {
return
Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled", "false"));
}
-
+
public boolean isCubePlannerEnabledForExistingCube() {
return
Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube",
"false"));
}
@@ -724,23 +724,23 @@ abstract public class KylinConfigBase implements
Serializable {
// SOURCE.JDBC
//
-public String getJdbcConnectionUrl() {
+public String getJdbcSourceConnectionUrl() {
return getOptional("kylin.source.jdbc.connection-url");
}
-public String getJdbcDriver() {
+public String getJdbcSourceDriver() {
return getOptional("kylin.source.jdbc.driver");
}
[4/4] kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping
KYLIN-2966 add pushdown jdbc columntype mapping
This closes #82
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b40d0e06
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b40d0e06
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b40d0e06
Branch: refs/heads/KYLIN-2966
Commit: b40d0e06f8dddf0916a40eb01a49417d4b65826b
Parents: 695971a
Author: zhaiyuyong
Authored: Wed Oct 25 17:06:04 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 14:16:04 2017 +0800
--
.../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++-
1 file changed, 44 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b40d0e06/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index 503e273..7283d66 100644
---
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
+++
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -23,6 +23,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Types;
import java.util.LinkedList;
import java.util.List;
@@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
columnMetas.add(new
SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i),
false,
metaData.isCurrency(i), metaData.isNullable(i), false,
metaData.getColumnDisplaySize(i),
metaData.getColumnLabel(i), metaData.getColumnName(i),
null, null, null,
-metaData.getPrecision(i), metaData.getScale(i),
metaData.getColumnType(i),
+metaData.getPrecision(i), metaData.getScale(i),
toSqlType(metaData.getColumnTypeName(i)),
metaData.getColumnTypeName(i), metaData.isReadOnly(i),
false, false));
}
} finally {
@@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
}
}
+// calcite does not understand all java SqlTypes, for example LONGNVARCHAR
-16, thus need this mapping (KYLIN-2966)
+public static int toSqlType(String type) throws SQLException {
+if ("string".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("varchar".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("char".equalsIgnoreCase(type)) {
+return Types.CHAR;
+} else if ("float".equalsIgnoreCase(type)) {
+return Types.FLOAT;
+} else if ("real".equalsIgnoreCase(type)) {
+return Types.REAL;
+} else if ("double".equalsIgnoreCase(type)) {
+return Types.DOUBLE;
+} else if ("boolean".equalsIgnoreCase(type)) {
+return Types.BOOLEAN;
+} else if ("tinyint".equalsIgnoreCase(type)) {
+return Types.TINYINT;
+} else if ("smallint".equalsIgnoreCase(type)) {
+return Types.SMALLINT;
+} else if ("int".equalsIgnoreCase(type)) {
+return Types.INTEGER;
+} else if ("bigint".equalsIgnoreCase(type)) {
+return Types.BIGINT;
+} else if ("date".equalsIgnoreCase(type)) {
+return Types.DATE;
+} else if ("timestamp".equalsIgnoreCase(type)) {
+return Types.TIMESTAMP;
+} else if ("decimal".equalsIgnoreCase(type)) {
+return Types.DECIMAL;
+} else if ("binary".equalsIgnoreCase(type)) {
+return Types.BINARY;
+} else if ("map".equalsIgnoreCase(type)) {
+return Types.JAVA_OBJECT;
+} else if ("array".equalsIgnoreCase(type)) {
+return Types.ARRAY;
+} else if ("struct".equalsIgnoreCase(type)) {
+return Types.STRUCT;
+}
+throw new SQLException("Unrecognized column type: " + type);
+}
+
@Override
public void executeUpdate(String sql) throws Exception {
Statement statement = null;
[kylin] Git Push Summary
Repository: kylin Updated Branches: refs/heads/KYLIN-2966 [deleted] b40d0e06f
kylin git commit: KYLIN-2966 add pushdown jdbc columntype mapping
Repository: kylin
Updated Branches:
refs/heads/master 695971a2a -> b40d0e06f
KYLIN-2966 add pushdown jdbc columntype mapping
This closes #82
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b40d0e06
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b40d0e06
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b40d0e06
Branch: refs/heads/master
Commit: b40d0e06f8dddf0916a40eb01a49417d4b65826b
Parents: 695971a
Author: zhaiyuyong
Authored: Wed Oct 25 17:06:04 2017 +0800
Committer: Li Yang
Committed: Sun Nov 26 14:16:04 2017 +0800
--
.../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +++-
1 file changed, 44 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b40d0e06/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index 503e273..7283d66 100644
---
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
+++
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -23,6 +23,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Types;
import java.util.LinkedList;
import java.util.List;
@@ -66,7 +67,7 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
columnMetas.add(new
SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i),
false,
metaData.isCurrency(i), metaData.isNullable(i), false,
metaData.getColumnDisplaySize(i),
metaData.getColumnLabel(i), metaData.getColumnName(i),
null, null, null,
-metaData.getPrecision(i), metaData.getScale(i),
metaData.getColumnType(i),
+metaData.getPrecision(i), metaData.getScale(i),
toSqlType(metaData.getColumnTypeName(i)),
metaData.getColumnTypeName(i), metaData.isReadOnly(i),
false, false));
}
} finally {
@@ -76,6 +77,48 @@ public class PushDownRunnerJdbcImpl implements
IPushDownRunner {
}
}
+// calcite does not understand all java SqlTypes, for example LONGNVARCHAR
-16, thus need this mapping (KYLIN-2966)
+public static int toSqlType(String type) throws SQLException {
+if ("string".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("varchar".equalsIgnoreCase(type)) {
+return Types.VARCHAR;
+} else if ("char".equalsIgnoreCase(type)) {
+return Types.CHAR;
+} else if ("float".equalsIgnoreCase(type)) {
+return Types.FLOAT;
+} else if ("real".equalsIgnoreCase(type)) {
+return Types.REAL;
+} else if ("double".equalsIgnoreCase(type)) {
+return Types.DOUBLE;
+} else if ("boolean".equalsIgnoreCase(type)) {
+return Types.BOOLEAN;
+} else if ("tinyint".equalsIgnoreCase(type)) {
+return Types.TINYINT;
+} else if ("smallint".equalsIgnoreCase(type)) {
+return Types.SMALLINT;
+} else if ("int".equalsIgnoreCase(type)) {
+return Types.INTEGER;
+} else if ("bigint".equalsIgnoreCase(type)) {
+return Types.BIGINT;
+} else if ("date".equalsIgnoreCase(type)) {
+return Types.DATE;
+} else if ("timestamp".equalsIgnoreCase(type)) {
+return Types.TIMESTAMP;
+} else if ("decimal".equalsIgnoreCase(type)) {
+return Types.DECIMAL;
+} else if ("binary".equalsIgnoreCase(type)) {
+return Types.BINARY;
+} else if ("map".equalsIgnoreCase(type)) {
+return Types.JAVA_OBJECT;
+} else if ("array".equalsIgnoreCase(type)) {
+return Types.ARRAY;
+} else if ("struct".equalsIgnoreCase(type)) {
+return Types.STRUCT;
+}
+throw new SQLException("Unrecognized column type: " + type);
+}
+
@Override
public void executeUpdate(String sql) throws Exception {
Statement statement = null;
