KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/066a38e3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/066a38e3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/066a38e3 Branch: refs/heads/yang21-hbase1.x Commit: 066a38e3b7b1c2345f9f12106173eccbcfbfb0e7 Parents: 9e1e70a Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Oct 28 18:15:33 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Oct 28 22:09:13 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/StringUtil.java | 13 +++++++++ .../engine/mr/common/AbstractHadoopJob.java | 28 +++++++------------- 2 files changed, 23 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/066a38e3/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java index 15872aa..b336e4b 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java @@ -124,4 +124,17 @@ public class StringUtil { return result; } + public static void appendWithSeparator(StringBuilder src, String append) { + if (src == null) { + throw new IllegalArgumentException(); + } + if (src.length() > 0 && src.toString().endsWith(",") == false) { + src.append(","); + } + + if (StringUtils.isBlank(append) == false) { + src.append(append); + } + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/066a38e3/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index af2ed9f..b77b6b0 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.common.util.StringSplitter; +import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.engine.mr.HadoopUtil; @@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { String filteredHive = filterKylinHiveDependency(kylinHiveDependency); logger.info("Hive Dependencies After Filtered: " + filteredHive); - if (kylinDependency.length() > 0) - kylinDependency.append(","); - kylinDependency.append(filteredHive); + StringUtil.appendWithSeparator(kylinDependency, filteredHive); } else { logger.info("No hive dependency jars set in the environment, will find them from jvm:"); try { String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver")); - kylinDependency.append(hiveExecJarPath).append(","); + + StringUtil.appendWithSeparator(kylinDependency, hiveExecJarPath); logger.info("hive-exec jar file: " + hiveExecJarPath); String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat")); - kylinDependency.append(hiveHCatJarPath).append(","); + StringUtil.appendWithSeparator(kylinDependency, hiveHCatJarPath); logger.info("hive-catalog jar file: " + hiveHCatJarPath); String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table")); - kylinDependency.append(hiveMetaStoreJarPath).append(","); + StringUtil.appendWithSeparator(kylinDependency, hiveMetaStoreJarPath); logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath); } catch (ClassNotFoundException e) { logger.error("Cannot found hive dependency jars: " + e); @@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { // for hive dependencies if (kylinKafkaDependency != null) { kylinKafkaDependency = kylinKafkaDependency.replace(":", ","); - - logger.info("Kafka Dependencies Before Filtered: " + kylinHiveDependency); - - if (kylinDependency.length() > 0) - kylinDependency.append(","); - kylinDependency.append(kylinKafkaDependency); + logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency); + StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency); } else { - logger.info("No Kafka dependency jars set in the environment, will find them from jvm:"); - try { String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer")); - kylinDependency.append(kafkaClientJarPath).append(","); + StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath); logger.info("kafka jar file: " + kafkaClientJarPath); } catch (ClassNotFoundException e) { @@ -254,9 +248,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { if (!StringUtils.isBlank(mrLibDir)) { File dirFileMRLIB = new File(mrLibDir); if (dirFileMRLIB.exists()) { - if (kylinDependency.length() > 0) - kylinDependency.append(","); - kylinDependency.append(mrLibDir); + StringUtil.appendWithSeparator(kylinDependency, mrLibDir); } else { logger.info("The directory '" + mrLibDir + "' for 'kylin.job.mr.lib.dir' does not exist!!!"); }