OOZIE-2598 refactoring classes so that Shell action doesn't need MR jars Change-Id: I48f5abe8a8beb68ea96fdb1d72b6601e561074ea
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/165847cc Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/165847cc Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/165847cc Branch: refs/heads/oya Commit: 165847ccbc0b1ee0bc084648173de561fe519259 Parents: b1069b2 Author: Peter Bacsko <pbac...@cloudera.com> Authored: Tue Oct 4 15:02:53 2016 +0200 Committer: Peter Bacsko <pbac...@cloudera.com> Committed: Tue Oct 4 15:02:53 2016 +0200 ---------------------------------------------------------------------- .../action/hadoop/Hive2ActionExecutor.java | 4 +- .../oozie/action/hadoop/HiveActionExecutor.java | 4 +- .../oozie/action/hadoop/JavaActionExecutor.java | 1 + .../action/hadoop/MapReduceActionExecutor.java | 4 +- .../oozie/action/hadoop/PigActionExecutor.java | 4 +- .../action/hadoop/ShellActionExecutor.java | 2 +- .../action/hadoop/SqoopActionExecutor.java | 2 +- .../wf/SubmitScriptLanguageXCommand.java | 6 +-- .../oozie/command/wf/SubmitSqoopXCommand.java | 4 +- .../oozie/action/hadoop/TestShellMain.java | 5 +- .../command/wf/TestSubmitHiveXCommand.java | 6 +-- .../oozie/command/wf/TestSubmitPigXCommand.java | 8 ++-- .../command/wf/TestSubmitSqoopXCommand.java | 4 +- .../apache/oozie/action/hadoop/HiveMain.java | 4 +- .../oozie/action/hadoop/TestHiveMain.java | 4 +- .../apache/oozie/action/hadoop/Hive2Main.java | 4 +- .../apache/oozie/action/hadoop/ActionUtils.java | 49 ++++++++++++++++++++ .../apache/oozie/action/hadoop/LauncherAM.java | 6 --- .../oozie/action/hadoop/MapReduceMain.java | 29 ++++-------- .../apache/oozie/action/hadoop/ShellMain.java | 8 ++-- .../org/apache/oozie/action/hadoop/PigMain.java | 4 +- .../oozie/action/hadoop/PigMainWithOldAPI.java | 4 +- .../action/hadoop/TestPigMainWithOldAPI.java | 5 +- .../apache/oozie/action/hadoop/SqoopMain.java | 2 +- .../oozie/action/hadoop/StreamingMain.java | 4 +- 25 files changed, 104 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java index ddb1504..35277ae 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java @@ -106,7 +106,7 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor { for (int i = 0; i < params.size(); i++) { strParams[i] = params.get(i).getTextTrim(); } - MapReduceMain.setStrings(conf, HIVE2_PARAMS, strParams); + ActionUtils.setStrings(conf, HIVE2_PARAMS, strParams); String[] strArgs = null; List<Element> eArgs = actionXml.getChildren("argument", ns); @@ -116,7 +116,7 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor { strArgs[i] = eArgs.get(i).getTextTrim(); } } - MapReduceMain.setStrings(conf, HIVE2_ARGS, strArgs); + ActionUtils.setStrings(conf, HIVE2_ARGS, strArgs); return conf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index d05edb8..82a955c 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -98,7 +98,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { for (int i = 0; i < params.size(); i++) { strParams[i] = params.get(i).getTextTrim(); } - MapReduceMain.setStrings(conf, HIVE_PARAMS, strParams); + ActionUtils.setStrings(conf, HIVE_PARAMS, strParams); String[] strArgs = null; List<Element> eArgs = actionXml.getChildren("argument", ns); @@ -108,7 +108,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { strArgs[i] = eArgs.get(i).getTextTrim(); } } - MapReduceMain.setStrings(conf, HIVE_ARGS, strArgs); + ActionUtils.setStrings(conf, HIVE_ARGS, strArgs); return conf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 0515d39..284690b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -1169,6 +1169,7 @@ public class JavaActionExecutor extends ActionExecutor { // FIXME: move this to specific places where it's actually needed - keeping it here for now ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf); + // FIXME: Pyspark fix // FIXME: Do we want to support mapred.child.env? env.put("SPARK_HOME", "."); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index 89f30f1..f7d230f 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -309,8 +309,8 @@ public class MapReduceActionExecutor extends JavaActionExecutor { if (recordReader != null) { conf.set("oozie.streaming.record-reader", recordReader); } - MapReduceMain.setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping); - MapReduceMain.setStrings(conf, "oozie.streaming.env", env); + ActionUtils.setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping); + ActionUtils.setStrings(conf, "oozie.streaming.env", env); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java index 65e9cbf..cbb7436 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java @@ -98,8 +98,8 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor { public static void setPigScript(Configuration conf, String script, String[] params, String[] args) { conf.set(PIG_SCRIPT, script); - MapReduceMain.setStrings(conf, PIG_PARAMS, params); - MapReduceMain.setStrings(conf, PIG_ARGS, args); + ActionUtils.setStrings(conf, PIG_PARAMS, params); + ActionUtils.setStrings(conf, PIG_ARGS, args); } http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java index 5acd581..9d71b78 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java @@ -112,7 +112,7 @@ public class ShellActionExecutor extends JavaActionExecutor { } } } - MapReduceMain.setStrings(actionConf, key, strTagValue); + ActionUtils.setStrings(actionConf, key, strTagValue); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index b0fc83a..c6266df 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@ -108,7 +108,7 @@ public class SqoopActionExecutor extends JavaActionExecutor { } private void setSqoopCommand(Configuration conf, String[] args) { - MapReduceMain.setStrings(conf, SQOOP_ARGS, args); + ActionUtils.setStrings(conf, SQOOP_ARGS, args); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java index b82369c..fab4398 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java @@ -19,7 +19,7 @@ package org.apache.oozie.command.wf; import org.apache.hadoop.conf.Configuration; -import org.apache.oozie.action.hadoop.MapReduceMain; +import org.apache.oozie.action.hadoop.ActionUtils; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.command.CommandException; import org.jdom.Element; @@ -58,7 +58,7 @@ public abstract class SubmitScriptLanguageXCommand extends SubmitHttpXCommand { List<String> Dargs = new ArrayList<String>(); List<String> otherArgs = new ArrayList<String>(); - String[] args = MapReduceMain.getStrings(conf, getOptions()); + String[] args = ActionUtils.getStrings(conf, getOptions()); for (String arg : args) { if (arg.startsWith("-D")) { Dargs.add(arg); @@ -67,7 +67,7 @@ public abstract class SubmitScriptLanguageXCommand extends SubmitHttpXCommand { otherArgs.add(arg); } } - String [] params = MapReduceMain.getStrings(conf, getScriptParamters()); + String [] params = ActionUtils.getStrings(conf, getScriptParamters()); // configuration section if (Dargs.size() > 0) { http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java index 51f3d03..c5574c5 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.command.CommandException; -import org.apache.oozie.action.hadoop.MapReduceMain; +import org.apache.oozie.action.hadoop.ActionUtils; import org.jdom.Namespace; import org.jdom.Element; @@ -57,7 +57,7 @@ public class SubmitSqoopXCommand extends SubmitHttpXCommand { ele.addContent(nn); List<String> Dargs = new ArrayList<String>(); - String[] args = MapReduceMain.getStrings(conf, getOptions()); + String[] args = ActionUtils.getStrings(conf, getOptions()); for (String arg : args) { if (arg.startsWith("-D")) { Dargs.add(arg); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java index e757e54..a7d6c18 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java @@ -25,7 +25,6 @@ import java.io.FileWriter; import java.io.Writer; import java.util.Properties; -import org.apache.hadoop.fs.Path; import org.apache.oozie.util.XConfiguration; //Test cases are mainly implemented in the Base class @@ -53,8 +52,8 @@ public class TestShellMain extends ShellTestCase { jobConf.set(ShellMain.CONF_OOZIE_SHELL_EXEC, SHELL_COMMAND_NAME); String[] args = new String[] { SHELL_COMMAND_SCRIPTFILE_OPTION, script.toString(), "A", "B" }; - MapReduceMain.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ARGS, args); - MapReduceMain.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ENVS, + ActionUtils.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ARGS, args); + ActionUtils.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ENVS, new String[] { "var1=value1", "var2=value2" }); File actionXml = new File(getTestCaseDir(), "action.xml"); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java index 014daff..98c94a7 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java @@ -21,7 +21,7 @@ package org.apache.oozie.command.wf; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.client.OozieClient; import org.apache.oozie.local.LocalOozie; -import org.apache.oozie.action.hadoop.MapReduceMain; +import org.apache.oozie.action.hadoop.ActionUtils; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.test.XFsTestCase; import org.apache.oozie.util.XLog; @@ -54,9 +54,9 @@ public class TestSubmitHiveXCommand extends XFsTestCase { String hiveArgsStr = "-a aaa -b bbb -c ccc -M -Da=aaa -Db=bbb -param input=abc"; String[] args = hiveArgsStr.split(" "); - MapReduceMain.setStrings(conf, XOozieClient.HIVE_OPTIONS, args); + ActionUtils.setStrings(conf, XOozieClient.HIVE_OPTIONS, args); String[] params = new String[]{"INPUT=/some/path", "OUTPUT=/some/other/path", "abc=xyz"}; - MapReduceMain.setStrings(conf, XOozieClient.HIVE_SCRIPT_PARAMS, params); + ActionUtils.setStrings(conf, XOozieClient.HIVE_SCRIPT_PARAMS, params); SubmitHiveXCommand submitHiveCmd = new SubmitHiveXCommand(conf); String xml = submitHiveCmd.getWorkflowXml(conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java index e3ce56b..c3cd1aa 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java @@ -21,7 +21,7 @@ package org.apache.oozie.command.wf; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.client.OozieClient; import org.apache.oozie.local.LocalOozie; -import org.apache.oozie.action.hadoop.MapReduceMain; +import org.apache.oozie.action.hadoop.ActionUtils; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.test.XFsTestCase; import org.apache.oozie.util.XLog; @@ -55,9 +55,9 @@ public class TestSubmitPigXCommand extends XFsTestCase { String pigArgsStr = "-a aaa -b bbb -c ccc -M -Da=aaa -Db=bbb -param input=abc"; String[] args = pigArgsStr.split(" "); - MapReduceMain.setStrings(conf, XOozieClient.PIG_OPTIONS, args); + ActionUtils.setStrings(conf, XOozieClient.PIG_OPTIONS, args); String[] params = new String[]{"INPUT=/some/path", "OUTPUT=/some/other/path", "abc=xyz"}; - MapReduceMain.setStrings(conf, XOozieClient.PIG_SCRIPT_PARAMS, params); + ActionUtils.setStrings(conf, XOozieClient.PIG_SCRIPT_PARAMS, params); SubmitPigXCommand submitPigCmd = new SubmitPigXCommand(conf); String xml = submitPigCmd.getWorkflowXml(conf); @@ -128,7 +128,7 @@ public class TestSubmitPigXCommand extends XFsTestCase { String[] args = new String[2]; args[0] = "-a"; args[1] = "aaa bbb"; - MapReduceMain.setStrings(conf, XOozieClient.PIG_OPTIONS, args); + ActionUtils.setStrings(conf, XOozieClient.PIG_OPTIONS, args); SubmitPigXCommand submitPigCmd = new SubmitPigXCommand(conf); String xml = submitPigCmd.getWorkflowXml(conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java index 05fef6a..f2f248a 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java @@ -21,7 +21,7 @@ package org.apache.oozie.command.wf; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.client.OozieClient; import org.apache.oozie.local.LocalOozie; -import org.apache.oozie.action.hadoop.MapReduceMain; +import org.apache.oozie.action.hadoop.ActionUtils; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.test.XFsTestCase; import org.apache.oozie.util.XLog; @@ -54,7 +54,7 @@ public class TestSubmitSqoopXCommand extends XFsTestCase { String sqoopArgsStr = "-Da=aaa -Db=bbb"; String[] args = sqoopArgsStr.split(" "); - MapReduceMain.setStrings(conf, XOozieClient.SQOOP_OPTIONS, args); + ActionUtils.setStrings(conf, XOozieClient.SQOOP_OPTIONS, args); SubmitSqoopXCommand submitSqoopCmd = new SubmitSqoopXCommand(conf); String xml = submitSqoopCmd.getWorkflowXml(conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index 1cf44db..f59750b 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -272,7 +272,7 @@ public class HiveMain extends LauncherMain { } // Pass any parameters to Hive via arguments - String[] params = MapReduceMain.getStrings(hiveConf, HiveActionExecutor.HIVE_PARAMS); + String[] params = ActionUtils.getStrings(hiveConf, HiveActionExecutor.HIVE_PARAMS); if (params.length > 0) { System.out.println("Parameters:"); System.out.println("------------------------"); @@ -292,7 +292,7 @@ public class HiveMain extends LauncherMain { System.out.println(); } - String[] hiveArgs = MapReduceMain.getStrings(hiveConf, HiveActionExecutor.HIVE_ARGS); + String[] hiveArgs = ActionUtils.getStrings(hiveConf, HiveActionExecutor.HIVE_ARGS); for (String hiveArg : hiveArgs) { if (DISALLOWED_HIVE_OPTIONS.contains(hiveArg)) { throw new RuntimeException("Error: Hive argument " + hiveArg + " is not supported"); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java index 879ae2d..b724db3 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java @@ -102,10 +102,10 @@ public class TestHiveMain extends MainTestCase { SharelibUtils.addToDistributedCache("hive", fs, getFsTestCaseDir(), jobConf); jobConf.set(HiveActionExecutor.HIVE_SCRIPT, script.toString()); - MapReduceMain.setStrings(jobConf, HiveActionExecutor.HIVE_PARAMS, new String[]{ + ActionUtils.setStrings(jobConf, HiveActionExecutor.HIVE_PARAMS, new String[]{ "IN=" + inputDir.toUri().getPath(), "OUT=" + outputDir.toUri().getPath()}); - MapReduceMain.setStrings(jobConf, HiveActionExecutor.HIVE_ARGS, + ActionUtils.setStrings(jobConf, HiveActionExecutor.HIVE_ARGS, new String[]{ "-v" }); File actionXml = new File(getTestCaseDir(), "action.xml"); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java index 157fbef..ccf2aff 100644 --- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java +++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java @@ -179,7 +179,7 @@ public class Hive2Main extends LauncherMain { } // Pass any parameters to Beeline via arguments - String[] params = MapReduceMain.getStrings(actionConf, Hive2ActionExecutor.HIVE2_PARAMS); + String[] params = ActionUtils.getStrings(actionConf, Hive2ActionExecutor.HIVE2_PARAMS); if (params.length > 0) { System.out.println("Parameters:"); System.out.println("------------------------"); @@ -204,7 +204,7 @@ public class Hive2Main extends LauncherMain { arguments.add("-a"); arguments.add("delegationToken"); - String[] beelineArgs = MapReduceMain.getStrings(actionConf, Hive2ActionExecutor.HIVE2_ARGS); + String[] beelineArgs = ActionUtils.getStrings(actionConf, Hive2ActionExecutor.HIVE2_ARGS); for (String beelineArg : beelineArgs) { if (DISALLOWED_BEELINE_OPTIONS.contains(beelineArg)) { throw new RuntimeException("Error: Beeline argument " + beelineArg + " is not supported"); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java new file mode 100644 index 0000000..3002ad5 --- /dev/null +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.conf.Configuration; + +public final class ActionUtils { + + private ActionUtils() { + // no instances + } + + public static void setStrings(Configuration conf, String key, String[] values) { + if (values != null) { + conf.setInt(key + ".size", values.length); + for (int i = 0; i < values.length; i++) { + conf.set(key + "." + i, values[i]); + } + } + } + + public static String[] getStrings(Configuration conf, String key) { + String[] values = new String[conf.getInt(key + ".size", 0)]; + for (int i = 0; i < values.length; i++) { + values[i] = conf.get(key + "." + i); + if (values[i] == null) { + values[i] = ""; + } + } + return values; + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index d51dcf7..85d78c6 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -40,18 +40,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.SaslRpcServer.AuthMethod; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.exceptions.YarnException; http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java index b808535..16cf4b1 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java @@ -123,31 +123,20 @@ public class MapReduceMain extends LauncherMain { return runJob; } - protected JobClient createJobClient(JobConf jobConf) throws IOException { - return new JobClient(jobConf); - } - - // allows any character in the value, the conf.setStrings() does not allow - // commas + @Deprecated public static void setStrings(Configuration conf, String key, String[] values) { - if (values != null) { - conf.setInt(key + ".size", values.length); - for (int i = 0; i < values.length; i++) { - conf.set(key + "." + i, values[i]); - } - } + ActionUtils.setStrings(conf, key, values); } + @Deprecated public static String[] getStrings(Configuration conf, String key) { - String[] values = new String[conf.getInt(key + ".size", 0)]; - for (int i = 0; i < values.length; i++) { - values[i] = conf.get(key + "." + i); - if (values[i] == null) { - values[i] = ""; - } - } - return values; + return ActionUtils.getStrings(conf, key); + } + + protected JobClient createJobClient(JobConf jobConf) throws IOException { + return new JobClient(jobConf); } + /** * Will run the user specified OozieActionConfigurator subclass (if one is provided) to update the action configuration. * http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java index f109318..0ee35e8 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java @@ -24,10 +24,10 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; -import java.io.PrintWriter; -import java.io.StringReader; import java.io.IOException; import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringReader; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -186,7 +186,7 @@ public class ShellMain extends LauncherMain { */ private Map<String, String> getEnvMap(Map<String, String> envp, Configuration actionConf) { // Adding user-specified environments - String[] envs = MapReduceMain.getStrings(actionConf, CONF_OOZIE_SHELL_ENVS); + String[] envs = ActionUtils.getStrings(actionConf, CONF_OOZIE_SHELL_ENVS); for (String env : envs) { String[] varValue = env.split("=",2); // Error case is handled in // ShellActionExecutor @@ -339,7 +339,7 @@ public class ShellMain extends LauncherMain { */ protected List<String> getShellArguments(Configuration actionConf) { List<String> arguments = new ArrayList<String>(); - String[] scrArgs = MapReduceMain.getStrings(actionConf, CONF_OOZIE_SHELL_ARGS); + String[] scrArgs = ActionUtils.getStrings(actionConf, CONF_OOZIE_SHELL_ARGS); for (String scrArg : scrArgs) { arguments.add(scrArg); } http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index c00b263..0029dd0 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -150,7 +150,7 @@ public class PigMain extends LauncherMain { arguments.add("-file"); arguments.add(script); - String[] params = MapReduceMain.getStrings(actionConf, PigActionExecutor.PIG_PARAMS); + String[] params = ActionUtils.getStrings(actionConf, PigActionExecutor.PIG_PARAMS); for (String param : params) { arguments.add("-param"); arguments.add(param); @@ -204,7 +204,7 @@ public class PigMain extends LauncherMain { arguments.add("-logfile"); arguments.add(pigLog); - String[] pigArgs = MapReduceMain.getStrings(actionConf, PigActionExecutor.PIG_ARGS); + String[] pigArgs = ActionUtils.getStrings(actionConf, PigActionExecutor.PIG_ARGS); for (String pigArg : pigArgs) { if (DISALLOWED_PIG_OPTIONS.contains(pigArg)) { throw new RuntimeException("Error: Pig argument " + pigArg + " is not supported"); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java index b84e2b5..52aabe7 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java @@ -136,7 +136,7 @@ public class PigMainWithOldAPI extends LauncherMain { arguments.add("-file"); arguments.add(script); - String[] params = MapReduceMain.getStrings(actionConf, "oozie.pig.params"); + String[] params = ActionUtils.getStrings(actionConf, "oozie.pig.params"); for (String param : params) { arguments.add("-param"); arguments.add(param); @@ -188,7 +188,7 @@ public class PigMainWithOldAPI extends LauncherMain { arguments.add("-logfile"); arguments.add(pigLog); - String[] pigArgs = MapReduceMain.getStrings(actionConf, "oozie.pig.args"); + String[] pigArgs = ActionUtils.getStrings(actionConf, "oozie.pig.args"); for (String pigArg : pigArgs) { if (DISALLOWED_PIG_OPTIONS.contains(pigArg)) { throw new RuntimeException("Error: Pig argument " + pigArg + " is not supported"); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java index e52e6fd..74de433 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java @@ -21,7 +21,6 @@ package org.apache.oozie.action.hadoop; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.oozie.action.hadoop.MainTestCase; -import org.apache.oozie.action.hadoop.MapReduceMain; import org.apache.oozie.action.hadoop.PigMainWithOldAPI; import org.apache.oozie.action.hadoop.SharelibUtils; import org.apache.oozie.test.XFsTestCase; @@ -97,9 +96,9 @@ public class TestPigMainWithOldAPI extends XFsTestCase implements Callable<Void> SharelibUtils.addToDistributedCache("pig", fs, getFsTestCaseDir(), jobConfiguration); String[] params = { "IN=" + inputDir.toUri().getPath(), "OUT=" + outputDir.toUri().getPath() }; - MapReduceMain.setStrings(jobConfiguration, "oozie.pig.params", params); + ActionUtils.setStrings(jobConfiguration, "oozie.pig.params", params); String[] args = { "-v" }; - MapReduceMain.setStrings(jobConfiguration, "oozie.pig.args", args); + ActionUtils.setStrings(jobConfiguration, "oozie.pig.args", args); File actionXml = new File(getTestCaseDir(), "action.xml"); OutputStream os = new FileOutputStream(actionXml); http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index 9e53af5..9c6ec6c 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -161,7 +161,7 @@ public class SqoopMain extends LauncherMain { Configuration sqoopConf = setUpSqoopSite(); String logFile = setUpSqoopLog4J(sqoopConf); - String[] sqoopArgs = MapReduceMain.getStrings(sqoopConf, SqoopActionExecutor.SQOOP_ARGS); + String[] sqoopArgs = ActionUtils.getStrings(sqoopConf, SqoopActionExecutor.SQOOP_ARGS); if (sqoopArgs == null) { throw new RuntimeException("Action Configuration does not have [" + SqoopActionExecutor.SQOOP_ARGS + "] property"); } http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java ---------------------------------------------------------------------- diff --git a/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java b/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java index 991bf7e..cc55166 100644 --- a/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java +++ b/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java @@ -56,12 +56,12 @@ public class StreamingMain extends MapReduceMain { if (value != null) { jobConf.set("stream.recordreader.class", value); } - String[] values = getStrings(actionConf, "oozie.streaming.record-reader-mapping"); + String[] values = ActionUtils.getStrings(actionConf, "oozie.streaming.record-reader-mapping"); for (String s : values) { String[] kv = s.split("="); jobConf.set("stream.recordreader." + kv[0], kv[1]); } - values = getStrings(actionConf, "oozie.streaming.env"); + values = ActionUtils.getStrings(actionConf, "oozie.streaming.env"); value = jobConf.get("stream.addenvironment", ""); if (value.length() > 0) { value = value + " ";