Repository: oozie Updated Branches: refs/heads/oya 5dd08b159 -> be32cda04
Address code review comments Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/be32cda0 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/be32cda0 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/be32cda0 Branch: refs/heads/oya Commit: be32cda04577b46fe4df292f859214b739e5a39b Parents: 5dd08b1 Author: Peter Bacsko <pbac...@cloudera.com> Authored: Thu May 18 16:58:18 2017 +0200 Committer: Peter Bacsko <pbac...@cloudera.com> Committed: Thu May 18 16:58:18 2017 +0200 ---------------------------------------------------------------------- .../oozie/action/hadoop/DistcpActionExecutor.java | 2 +- .../oozie/action/hadoop/HiveActionExecutor.java | 2 +- .../oozie/action/hadoop/JavaActionExecutor.java | 14 ++------------ .../oozie/action/hadoop/MapReduceActionExecutor.java | 2 +- .../apache/oozie/action/hadoop/PigActionExecutor.java | 2 +- .../oozie/action/hadoop/ShellActionExecutor.java | 8 ++++++++ .../oozie/action/hadoop/SparkActionExecutor.java | 2 +- .../oozie/action/hadoop/SqoopActionExecutor.java | 2 +- .../oozie/action/hadoop/TestJavaActionExecutor.java | 6 ++++++ .../oozie/action/hadoop/TestShellActionExecutor.java | 2 +- .../test/java/org/apache/oozie/test/XTestCase.java | 2 +- sharelib/oozie/pom.xml | 1 + 12 files changed, 25 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java index fb26399..20f47d5 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java @@ -107,7 +107,7 @@ public class DistcpActionExecutor extends JavaActionExecutor{ } @Override - protected boolean needToAddMRJars() { + protected boolean needToAddMapReduceToClassPath() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index 012d1ec..22378fc 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -132,7 +132,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { } @Override - protected boolean needToAddMRJars() { + protected boolean needToAddMapReduceToClassPath() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 87deb69..c879c90 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -314,14 +314,6 @@ public class JavaActionExecutor extends ActionExecutor { } } - void updateConfForJavaTmpDir(Configuration conf) { - String amChildOpts = conf.get(YARN_AM_COMMAND_OPTS); - String oozieJavaTmpDirSetting = "-Djava.io.tmpdir=./tmp"; - if (amChildOpts != null && !amChildOpts.contains(JAVA_TMP_DIR_SETTINGS)) { - conf.set(YARN_AM_COMMAND_OPTS, amChildOpts + " " + oozieJavaTmpDirSetting); - } - } - public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException { parseJobXmlAndConfiguration(context, element, appPath, conf, false); @@ -402,7 +394,6 @@ public class JavaActionExecutor extends ActionExecutor { // set cancel.delegation.token in actionConf that child job doesn't cancel delegation token actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); - updateConfForJavaTmpDir(actionConf); setRootLoggerLevel(actionConf); return actionConf; } @@ -862,7 +853,6 @@ public class JavaActionExecutor extends ActionExecutor { launcherJobConf.set(HADOOP_CHILD_JAVA_OPTS, opts.toString().trim()); launcherJobConf.set(HADOOP_MAP_JAVA_OPTS, opts.toString().trim()); - updateConfForJavaTmpDir(launcherJobConf); injectLauncherTimelineServiceEnabled(launcherJobConf, actionConf); // properties from action that are needed by the launcher (e.g. QUEUE NAME, ACLs) @@ -1085,7 +1075,7 @@ public class JavaActionExecutor extends ActionExecutor { // This adds the Hadoop jars to the classpath in the Launcher JVM ClasspathUtils.setupClasspath(env, launcherJobConf); - if (needToAddMRJars()) { + if (needToAddMapReduceToClassPath()) { ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf); } @@ -1343,7 +1333,7 @@ public class JavaActionExecutor extends ActionExecutor { * MR jars to the classpath. * @return false by default */ - protected boolean needToAddMRJars() { + protected boolean needToAddMapReduceToClassPath() { return false; } http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index 2050591..ca7c7f2 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -342,7 +342,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { } @Override - protected boolean needToAddMRJars() { + protected boolean needToAddMapReduceToClassPath() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java index 17508eb..8a24ac3 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java @@ -123,7 +123,7 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor { } @Override - protected boolean needToAddMRJars() { + protected boolean needToAddMapReduceToClassPath() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java index 9d71b78..63b8722 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java @@ -19,10 +19,13 @@ package org.apache.oozie.action.hadoop; +import java.io.File; import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.util.Apps; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.service.ConfigurationService; import org.jdom.Element; @@ -148,6 +151,11 @@ public class ShellActionExecutor extends JavaActionExecutor { updateProperty(conf, OOZIE_LAUNCHER_CHILD_ENV, envValues); } + @Override + protected void addActionSpecificEnvVars(Map<String, String> env) { + Apps.setEnvFromInputString(env, "PATH=.:$PATH", File.pathSeparator); + } + /** * Utility method to append the new value to any property. * http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 2161af0..00497a7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -143,7 +143,7 @@ public class SparkActionExecutor extends JavaActionExecutor { } @Override - protected boolean needToAddMRJars() { + protected boolean needToAddMapReduceToClassPath() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index 73cc190..955f3b7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@ -237,7 +237,7 @@ public class SqoopActionExecutor extends JavaActionExecutor { } @Override - protected boolean needToAddMRJars() { + protected boolean needToAddMapReduceToClassPath() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 972cc1c..f14f060 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -31,11 +31,14 @@ import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.examples.SleepJob; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -43,8 +46,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.action.ActionExecutor; @@ -58,6 +63,7 @@ import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.Services; import org.apache.oozie.service.ShareLibService; import org.apache.oozie.service.UUIDService; +import org.apache.oozie.service.UserGroupInformationService; import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.service.WorkflowStoreService; import org.apache.oozie.util.IOUtils; http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java index 1c3b33e..72be0a2 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java @@ -42,7 +42,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { private static final String SHELL_EXEC = Shell.WINDOWS ? "cmd.exe" : "sh"; private static final String SHELL_PARAM = Shell.WINDOWS ? "/c" : "-c"; - private static final String SHELL_SCRIPTNAME = Shell.WINDOWS ? "script.cmd" : "./script.sh"; + private static final String SHELL_SCRIPTNAME = Shell.WINDOWS ? "script.cmd" : "script.sh"; private static final String SHELL_SCRIPT_CONTENT = Shell.WINDOWS ? "dir /s /b\necho %1 %2\necho %PATH%\ntype %0" : "ls -ltr\necho $1 $2\necho $PATH\npwd\ntype sh"; http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 72d646d..d7ce9b2 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -1235,7 +1235,7 @@ public abstract class XTestCase extends TestCase { final ApplicationId appId = ConverterUtils.toApplicationId(externalId); final MutableObject<YarnApplicationState> finalState = new MutableObject<YarnApplicationState>(); - Configuration conf = Services.get().get(HadoopAccessorService.class).getCachedConf(); + Configuration conf = Services.get().get(HadoopAccessorService.class).createJobConf(getJobTrackerUri()); final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), conf); try { http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/sharelib/oozie/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml index 0ccca1d..12f5cdd 100644 --- a/sharelib/oozie/pom.xml +++ b/sharelib/oozie/pom.xml @@ -64,6 +64,7 @@ <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> + <scope>test</scope> </dependency> <dependency>