KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e419681e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e419681e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e419681e Branch: refs/heads/v1.6.0-rc1-cdh5.7 Commit: e419681e8c9c6135af40dcdc2b9b3b367b3e2456 Parents: 46a894d Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Oct 28 18:15:33 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Oct 28 22:11:33 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/StringUtil.java | 13 +++++++++ .../engine/mr/common/AbstractHadoopJob.java | 29 ++++++-------------- 2 files changed, 22 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java index 15872aa..b336e4b 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java @@ -124,4 +124,17 @@ public class StringUtil { return result; } + public static void appendWithSeparator(StringBuilder src, String append) { + if (src == null) { + throw new IllegalArgumentException(); + } + if (src.length() > 0 && src.toString().endsWith(",") == false) { + src.append(","); + } + + if (StringUtils.isBlank(append) == false) { + src.append(append); + } + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index a5b2d2e..417d63d 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.common.util.StringSplitter; +import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.engine.mr.HadoopUtil; @@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { String filteredHive = filterKylinHiveDependency(kylinHiveDependency); logger.info("Hive Dependencies After Filtered: " + filteredHive); - if (kylinDependency.length() > 0) - kylinDependency.append(","); - kylinDependency.append(filteredHive); + StringUtil.appendWithSeparator(kylinDependency, filteredHive); } else { logger.info("No hive dependency jars set in the environment, will find them from jvm:"); try { String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver")); - kylinDependency.append(hiveExecJarPath).append(","); + + StringUtil.appendWithSeparator(kylinDependency, hiveExecJarPath); logger.info("hive-exec jar file: " + hiveExecJarPath); String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat")); - kylinDependency.append(hiveHCatJarPath).append(","); + StringUtil.appendWithSeparator(kylinDependency, hiveHCatJarPath); logger.info("hive-catalog jar file: " + hiveHCatJarPath); String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table")); - kylinDependency.append(hiveMetaStoreJarPath).append(","); + StringUtil.appendWithSeparator(kylinDependency, hiveMetaStoreJarPath); logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath); } catch (ClassNotFoundException e) { logger.error("Cannot found hive dependency jars: " + e); @@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { // for kafka dependencies if (kylinKafkaDependency != null) { kylinKafkaDependency = kylinKafkaDependency.replace(":", ","); - logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency); - - if (kylinDependency.length() > 0) - kylinDependency.append(","); - kylinDependency.append(kylinKafkaDependency); + StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency); } else { - logger.info("No Kafka dependency jars set in the environment, will find them from jvm:"); - try { String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer")); - kylinDependency.append(kafkaClientJarPath).append(","); + StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath); logger.info("kafka jar file: " + kafkaClientJarPath); } catch (ClassNotFoundException e) { @@ -251,12 +245,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { // for KylinJobMRLibDir String mrLibDir = kylinConf.getKylinJobMRLibDir(); - if (!StringUtils.isBlank(mrLibDir)) { - if (kylinDependency.length() > 0) { - kylinDependency.append(","); - } - kylinDependency.append(mrLibDir); - } + StringUtil.appendWithSeparator(kylinDependency, mrLibDir); setJobTmpJarsAndFiles(job, kylinDependency.toString());