Repository: kylin Updated Branches: refs/heads/yang21-hbase1.x a694c8be2 -> 90027c2cc (forced update)
AbstractHadoopJob: set hive dependency filter as a config entry Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d155263a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d155263a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d155263a Branch: refs/heads/yang21-hbase1.x Commit: d155263a21c2a51f3fa0f693d0fb7db194dd50ec Parents: 28fb758 Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Oct 31 17:03:42 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Mon Oct 31 17:03:42 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++++++--- .../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 11 ++++------- 2 files changed, 11 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d155263a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 784b2b1..e703a07 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -22,9 +22,9 @@ import java.io.File; import java.io.IOException; import java.io.Serializable; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.SortedSet; -import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -426,7 +426,7 @@ abstract public class KylinConfigBase implements Serializable { public int getCubeAlgorithmAutoMapperLimit() { return Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500")); } - + @Deprecated public int getCubeAggrGroupMaxSize() { return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12")); @@ -500,7 +500,7 @@ abstract public class KylinConfigBase implements Serializable { public boolean getBadQueryPersistentEnabled() { return Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable", "true")); } - + public String[] getQueryTransformers() { return getOptionalStringArray("kylin.query.transformers", new String[0]); } @@ -629,6 +629,10 @@ abstract public class KylinConfigBase implements Serializable { return this.getOptional("kylin.job.hive.database.for.intermediatetable", "default"); } + public String getHiveDependencyFilterList() { + return this.getOptional("kylin.job.dependency.filterlist", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar"); + } + public String getKylinOwner() { return this.getOptional("kylin.owner", ""); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d155263a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 5d85093..11d995c 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -80,7 +80,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job name. For example, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME); protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME); protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID); -// @Deprecated + // @Deprecated protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create(BatchConstants.ARG_SEGMENT_NAME); protected static final Option OPTION_SEGMENT_ID = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_ID).hasArg().isRequired(true).withDescription("Cube segment id").create(BatchConstants.ARG_SEGMENT_ID); protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input path").create(BatchConstants.ARG_INPUT); @@ -96,8 +96,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath"; - private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar"; - protected static void runJob(Tool job, String[] args) { try { int exitCode = ToolRunner.run(job, args); @@ -200,8 +198,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { kylinHiveDependency = kylinHiveDependency.replace(":", ","); logger.info("Hive Dependencies Before Filtered: " + kylinHiveDependency); - //String filteredHive = filterKylinHiveDependency(kylinHiveDependency); - String filteredHive = kylinHiveDependency; + String filteredHive = filterKylinHiveDependency(kylinHiveDependency, kylinConf); logger.info("Hive Dependencies After Filtered: " + filteredHive); StringUtil.appendWithSeparator(kylinDependency, filteredHive); @@ -266,13 +263,13 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { } } - private String filterKylinHiveDependency(String kylinHiveDependency) { + private String filterKylinHiveDependency(String kylinHiveDependency, KylinConfig config) { if (StringUtils.isBlank(kylinHiveDependency)) return ""; StringBuilder jarList = new StringBuilder(); - Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS); + Pattern hivePattern = Pattern.compile(config.getHiveDependencyFilterList()); Matcher matcher = hivePattern.matcher(kylinHiveDependency); while (matcher.find()) {
