Repository: oozie Updated Branches: refs/heads/master c1fe02cff -> fe3dc9cd0
OOZIE-2869 amend Ability to add/remove MapReduce jars from action classpath (andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/fe3dc9cd Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/fe3dc9cd Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/fe3dc9cd Branch: refs/heads/master Commit: fe3dc9cd0745fdcdcb93deae84ad38927a31a97e Parents: c1fe02c Author: Andras Piros <andras.pi...@cloudera.com> Authored: Thu Dec 7 17:38:44 2017 +0100 Committer: Andras Piros <andras.pi...@cloudera.com> Committed: Thu Dec 7 17:38:44 2017 +0100 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 23 ++++++++++++++++---- .../oozie/service/ConfigurationService.java | 19 ++++++++++++++++ core/src/main/resources/oozie-default.xml | 16 ++++++++++++++ release-log.txt | 1 + 4 files changed, 55 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/fe3dc9cd/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index cab0d8b..5c8acae 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -1558,12 +1558,27 @@ public class JavaActionExecutor extends ActionExecutor { * If returns true, it means that we have to add Hadoop MR jars to the classpath. * Subclasses should override this method if necessary. By default we don't add * MR jars to the classpath. + * + * <p>Configuration properties are read either from {@code launcherConf}, or, if not present, falling back to + * {@link ConfigurationService#getBoolean(Configuration, String)}. + * * @return false by default - * @param launcherConf + * @param launcherConf the launcher {@link Configuration} that is used on first lookup */ - private boolean needToAddMapReduceToClassPath(Configuration launcherConf) { - boolean defaultValue = launcherConf.getBoolean(OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY, false); - return launcherConf.getBoolean(OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY + "." + getType(), defaultValue); + private boolean needToAddMapReduceToClassPath(final Configuration launcherConf) { + LOG.debug("Calculating whether to add MapReduce JARs to classpath. [type={0};this={1}]", getType(), this); + + final boolean defaultValue = ConfigurationService.getBooleanOrDefault( + launcherConf, + OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY, + false); + LOG.debug("Default value for [{0}] is [{1}]", OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY, defaultValue); + + final String configurationKey = OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY + "." + getType(); + final boolean configuredValue = ConfigurationService.getBooleanOrDefault(launcherConf, configurationKey, defaultValue); + LOG.debug("Configured value for [{0}] is [{1}]", configurationKey, configuredValue); + + return configuredValue; } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/fe3dc9cd/core/src/main/java/org/apache/oozie/service/ConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java index 4707252..15076de 100644 --- a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java +++ b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java @@ -18,6 +18,7 @@ package org.apache.oozie.service; +import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.util.ConfigUtils; import org.apache.oozie.util.Instrumentable; @@ -537,6 +538,24 @@ public class ConfigurationService implements Service, Instrumentable { return conf.getBoolean(name, ConfigUtils.BOOLEAN_DEFAULT); } + /** + * Get the {@code boolean} value for {@code name} from {@code conf}, or the default {@link Configuration} coming from + * {@code oozie-site.xml}, or {@code defaultValue}, if no previous occurrences present. + * + * @param conf the {@link Configuration} for primary lookup + * @param name name of the parameter to look up + * @param defaultValue default value to return when every other possibility is exhausted + * @return a {@code boolean} given above lookup order + */ + public static boolean getBooleanOrDefault(final Configuration conf, final String name, final boolean defaultValue) { + if (Strings.isNullOrEmpty(conf.get(name))) { + final Configuration defaultConf = Services.get().getConf(); + return defaultConf.getBoolean(name, defaultValue); + } + + return conf.getBoolean(name, defaultValue); + } + public static int getInt(String name) { Configuration conf = Services.get().getConf(); return getInt(conf, name); http://git-wip-us.apache.org/repos/asf/oozie/blob/fe3dc9cd/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 7ec64bc..1c34800 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -3261,6 +3261,22 @@ will be the requeue interval for the actions which are waiting for a long time w </property> <property> + <name>oozie.action.mapreduce.needed.for.hive2</name> + <value>true</value> + <description> + Whether to add MapReduce jars to the Hive2 action's classpath's by default. + </description> + </property> + + <property> + <name>oozie.action.mapreduce.needed.for.java</name> + <value>true</value> + <description> + Whether to add MapReduce jars to the Java action's classpath's by default. + </description> + </property> + + <property> <name>oozie.action.mapreduce.needed.for.map-reduce</name> <value>true</value> <description> http://git-wip-us.apache.org/repos/asf/oozie/blob/fe3dc9cd/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 74e6f28..6d2181d 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2869 amend Ability to add/remove MapReduce jars from action classpath (andras.piros) OOZIE-2969 Drop support for Java 1.7 (dbist13 via gezapeti, pbacsko) OOZIE-2681 Fix javadoc to compile on JDK8 (dbist13, abhisekbafna, gezapeti) OOZIE-3112 SparkConfigrationService overwrites properties provided via --properties-file option in SparkAction (gezapeti)
