KYLIN-3044 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2 Branch: refs/heads/KYLIN-3044 Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d Parents: cac1f8b Author: Li Yang <liy...@apache.org> Authored: Sun Nov 26 13:04:27 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sun Nov 26 13:04:27 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfigBase.java | 2 +- .../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++- .../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +- .../apache/kylin/source/jdbc/JdbcHiveMRInput.java | 15 ++++++++------- .../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +- .../org/apache/kylin/source/jdbc/SqlUtilTest.java | 6 +++--- 6 files changed, 16 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b2368ce..1a93dd4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4")); } - public String getFieldDelimiter() { + public String getSourceFieldDelimiter() { return getOptional("kylin.source.jdbc.field-delimiter", "|"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 0ab3d3d..d136ec6 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -85,7 +85,8 @@ public class JoinedFlatTable { public static String generateCreateTableStatement(IJoinedFlatTableDesc flatDesc, String storageDfsDir, String format) { - return generateCreateTableStatement(flatDesc, storageDfsDir, format, "|"); + String fieldDelimiter = flatDesc.getDataModel().getConfig().getSourceFieldDelimiter(); + return generateCreateTableStatement(flatDesc, storageDfsDir, format, fieldDelimiter); } public static String generateDropTableStatement(IJoinedFlatTableDesc flatDesc) { http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java index 1278128..2827b3d 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java @@ -112,7 +112,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(cname.toUpperCase()); - String kylinType = SqlUtil.jdbcTypetoKylinDataType(type); + String kylinType = SqlUtil.jdbcTypeToKylinDataType(type); int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1; int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1; http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java index e83518a..15259cc 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java @@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc; import java.util.List; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; import org.apache.kylin.job.JoinedFlatTable; import org.apache.kylin.job.constant.ExecutableConstants; @@ -51,6 +50,10 @@ public class JdbcHiveMRInput extends HiveMRInput { public BatchCubingInputSide(IJoinedFlatTableDesc flatDesc) { super(flatDesc); } + + private KylinConfig getConfig() { + return flatDesc.getDataModel().getConfig(); + } @Override protected void addStepPhase1_DoCreateFlatTable(DefaultChainedExecutable jobFlow) { @@ -64,8 +67,7 @@ public class JdbcHiveMRInput extends HiveMRInput { private AbstractExecutable createFlatHiveTableFromFiles(String hiveInitStatements, String jobWorkingDir) { final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc); - KylinConfig config = KylinConfig.getInstanceFromEnv(); - String filedDelimiter = config.getFieldDelimiter(); + String filedDelimiter = getConfig().getSourceFieldDelimiter(); // Sqoop does not support exporting SEQUENSEFILE to Hive now SQOOP-869 final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir, "TEXTFILE", filedDelimiter); @@ -97,7 +99,7 @@ public class JdbcHiveMRInput extends HiveMRInput { return partitionDesc.getPartitionDateColumnRef(); } TblColRef splitColumn = null; - TableMetadataManager tblManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + TableMetadataManager tblManager = TableMetadataManager.getInstance(getConfig()); long maxCardinality = 0; for (TableRef tableRef : flatDesc.getDataModel().getAllTables()) { TableExtDesc tableExtDesc = tblManager.getTableExt(tableRef.getTableDesc()); @@ -123,8 +125,7 @@ public class JdbcHiveMRInput extends HiveMRInput { } private AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) { - KylinConfig config = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName) - .getConfig(); + KylinConfig config = getConfig(); PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc(); String partCol = null; String partitionString = null; @@ -151,7 +152,7 @@ public class JdbcHiveMRInput extends HiveMRInput { String jdbcUser = config.getJdbcSourceUser(); String jdbcPass = config.getJdbcSourcePass(); String sqoopHome = config.getSqoopHome(); - String filedDelimiter = config.getFieldDelimiter(); + String filedDelimiter = config.getSourceFieldDelimiter(); int mapperNum = config.getSqoopMapperNum(); String bquery = String.format("SELECT min(%s), max(%s) FROM %s.%s", splitColumn, splitColumn, splitDatabase, http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java index 79fab7d..715bb99 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java @@ -96,7 +96,7 @@ public class SqlUtil { return con; } - public static String jdbcTypetoKylinDataType(int sqlType) { + public static String jdbcTypeToKylinDataType(int sqlType) { String result = "any"; switch (sqlType) { http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java ---------------------------------------------------------------------- diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java index d952675..7a7fd33 100644 --- a/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java +++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java @@ -27,9 +27,9 @@ public class SqlUtilTest { @Test public void testJdbcTypetoKylinDataType() { this.getClass().getClassLoader().toString(); - Assert.assertEquals("double", SqlUtil.jdbcTypetoKylinDataType(Types.FLOAT)); - Assert.assertEquals("varchar", SqlUtil.jdbcTypetoKylinDataType(Types.NVARCHAR)); - Assert.assertEquals("any", SqlUtil.jdbcTypetoKylinDataType(Types.ARRAY)); + Assert.assertEquals("double", SqlUtil.jdbcTypeToKylinDataType(Types.FLOAT)); + Assert.assertEquals("varchar", SqlUtil.jdbcTypeToKylinDataType(Types.NVARCHAR)); + Assert.assertEquals("any", SqlUtil.jdbcTypeToKylinDataType(Types.ARRAY)); } @Test
