OOZIE-2918 Delete LauncherMapper and its test (asasvari via pbacsko)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c09561ff Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c09561ff Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c09561ff Branch: refs/heads/master Commit: c09561ff263f38ff4ecdf81f71359ae2ae4c90d5 Parents: 79ccfdd Author: Peter Bacsko <pbac...@cloudera.com> Authored: Tue Jul 4 14:30:11 2017 +0200 Committer: Peter Bacsko <pbac...@cloudera.com> Committed: Tue Jul 4 14:30:11 2017 +0200 ---------------------------------------------------------------------- .../action/hadoop/DistcpActionExecutor.java | 2 +- .../oozie/action/hadoop/FsActionExecutor.java | 2 +- .../oozie/action/hadoop/FsELFunctions.java | 2 +- .../action/hadoop/Hive2ActionExecutor.java | 4 +- .../oozie/action/hadoop/HiveActionExecutor.java | 2 +- .../oozie/action/hadoop/JavaActionExecutor.java | 13 +- .../oozie/action/hadoop/LauncherHelper.java | 62 +- .../hadoop/LauncherInputFormatClassLocator.java | 84 --- .../action/hadoop/MapReduceActionExecutor.java | 27 +- .../oozie/action/hadoop/PigActionExecutor.java | 2 +- .../action/hadoop/ShellActionExecutor.java | 2 +- .../action/hadoop/SparkActionExecutor.java | 4 +- .../action/hadoop/SqoopActionExecutor.java | 3 +- core/src/main/resources/oozie-default.xml | 2 +- .../oozie/action/hadoop/LauncherMainTester.java | 2 +- .../oozie/action/hadoop/TestDistcpMain.java | 6 +- .../action/hadoop/TestJavaActionExecutor.java | 8 +- .../oozie/action/hadoop/TestLauncher.java | 245 ------- .../TestLauncherInputFormatClassLocator.java | 57 -- .../oozie/action/hadoop/TestMapReduceMain.java | 2 +- .../action/hadoop/TestShellActionExecutor.java | 2 +- .../oozie/service/TestConfigurationService.java | 7 +- release-log.txt | 1 + .../apache/oozie/action/hadoop/DistcpMain.java | 4 +- .../apache/oozie/action/hadoop/HiveMain.java | 2 +- .../action/hadoop/TestHiveActionExecutor.java | 6 +- .../oozie/action/hadoop/TestHiveMain.java | 13 +- .../action/hadoop/FSLauncherURIHandler.java | 2 +- .../apache/oozie/action/hadoop/JavaMain.java | 2 +- .../apache/oozie/action/hadoop/LauncherAM.java | 9 +- .../oozie/action/hadoop/LauncherAMUtils.java | 108 +++ .../oozie/action/hadoop/LauncherMapper.java | 715 ------------------- .../oozie/action/hadoop/MapReduceMain.java | 4 +- .../action/hadoop/OozieLauncherInputFormat.java | 119 --- .../hadoop/OozieLauncherOutputCommitter.java | 6 +- .../hadoop/OozieLauncherOutputFormat.java | 48 -- .../apache/oozie/action/hadoop/ShellMain.java | 2 +- .../oozie/action/hadoop/TestLauncherAM.java | 6 +- .../action/hadoop/TestLauncherAMUtils.java | 129 ++++ .../oozie/action/hadoop/TestLauncherMapper.java | 117 --- .../org/apache/oozie/action/hadoop/PigMain.java | 23 +- .../oozie/action/hadoop/PigMainWithOldAPI.java | 12 +- .../action/hadoop/TestPigActionExecutor.java | 2 +- .../apache/oozie/action/hadoop/TestPigMain.java | 15 - .../action/hadoop/TestPigMainWithOldAPI.java | 18 +- .../apache/oozie/action/hadoop/SparkMain.java | 4 +- .../apache/oozie/action/hadoop/SqoopMain.java | 4 +- .../hadoop/TestMapReduceActionExecutor.java | 4 +- .../oozie/action/hadoop/TestStreamingMain.java | 2 +- 49 files changed, 355 insertions(+), 1562 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java index 20f47d5..961994e 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java @@ -113,6 +113,6 @@ public class DistcpActionExecutor extends JavaActionExecutor{ @Override protected String getLauncherMain(Configuration launcherConf, Element actionXml) { - return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS); + return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java index 0515db6..c2a6192 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java @@ -65,7 +65,7 @@ public class FsActionExecutor extends ActionExecutor { public FsActionExecutor() { super(ACTION_TYPE); - maxGlobCount = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX); + maxGlobCount = ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java index 295f81a..031be25 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java @@ -170,7 +170,7 @@ public class FsELFunctions { int count = 0; int globMax = Integer.MAX_VALUE; public FSPathFilter() { - globMax = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX); + globMax = ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX); } @Override public boolean accept(Path p) { http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java index 35277ae..0480f06 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java @@ -27,7 +27,7 @@ import org.jdom.Namespace; import java.util.ArrayList; import java.util.List; -import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; +import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS; public class Hive2ActionExecutor extends ScriptLanguageActionExecutor { @@ -142,4 +142,4 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor { return new String[] { "hive-site.xml" }; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index 22378fc..d8c613b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -30,7 +30,7 @@ import org.jdom.Namespace; import java.util.ArrayList; import java.util.List; -import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; +import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS; public class HiveActionExecutor extends ScriptLanguageActionExecutor { http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index eac8e62..fa83f1e 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -199,7 +199,7 @@ public class JavaActionExecutor extends ActionExecutor { maxExternalStatsSize = ConfigurationService.getInt(JavaActionExecutor.MAX_EXTERNAL_STATS_SIZE); maxExternalStatsSize = (maxExternalStatsSize == -1) ? Integer.MAX_VALUE : maxExternalStatsSize; //Get the limit for the maximum number of globbed files/dirs for FS operation - maxFSGlobMax = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX); + maxFSGlobMax = ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX); registerError(UnknownHostException.class.getName(), ActionExecutorException.ErrorType.TRANSIENT, "JA001"); registerError(AccessControlException.class.getName(), ActionExecutorException.ErrorType.NON_TRANSIENT, @@ -414,8 +414,8 @@ public class JavaActionExecutor extends ActionExecutor { * @param actionConf */ void setRootLoggerLevel(Configuration actionConf) { - String oozieActionTypeRootLogger = "oozie.action." + getType() + LauncherMapper.ROOT_LOGGER_LEVEL; - String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL; + String oozieActionTypeRootLogger = "oozie.action." + getType() + LauncherAMUtils.ROOT_LOGGER_LEVEL; + String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL; // check if root log level has already mentioned in action configuration String rootLogLevel = actionConf.get(oozieActionTypeRootLogger, actionConf.get(oozieActionRootLogger)); @@ -842,11 +842,8 @@ public class JavaActionExecutor extends ActionExecutor { } LauncherHelper.setupMainArguments(launcherJobConf, args); // backward compatibility flag - see OOZIE-2872 - if (ConfigurationService.getBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED)) { - launcherJobConf.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, true); - } else { - launcherJobConf.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, false); - } + boolean nullArgsAllowed = ConfigurationService.getBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED); + launcherJobConf.setBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed); // Make mapred.child.java.opts and mapreduce.map.java.opts equal, but give values from the latter priority; also append // <java-opt> and <java-opts> and give those highest priority http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java index f80141c..f9e0aef 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java @@ -55,8 +55,6 @@ public class LauncherHelper { public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag"; - private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator(); - public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws HadoopAccessorException, IOException { String jobId = null; @@ -78,7 +76,7 @@ public class LauncherHelper { // Only set the javaMainClass if its not null or empty string, this way the user can override the action's main class via // <configuration> property if (javaMainClass != null && !javaMainClass.equals("")) { - launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass); + launcherConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass); } } @@ -89,14 +87,14 @@ public class LauncherHelper { } public static void setupMainArguments(Configuration launcherConf, String[] args) { - launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length); + launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length); for (int i = 0; i < args.length; i++) { - launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]); + launcherConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]); } } public static void setupMaxOutputData(Configuration launcherConf, int maxOutputData) { - launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData); + launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData); } /** @@ -106,7 +104,7 @@ public class LauncherHelper { * @param maxStatsData the maximum allowed size of stats data */ public static void setupMaxExternalStatsSize(Configuration launcherConf, int maxStatsData){ - launcherConf.setInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData); + launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData); } /** @@ -116,20 +114,20 @@ public class LauncherHelper { * @param fsGlobMax the maximum number of files/dirs for FS operation */ public static void setupMaxFSGlob(Configuration launcherConf, int fsGlobMax){ - launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax); + launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax); } public static void setupLauncherInfo(Configuration launcherConf, String jobId, String actionId, Path actionDir, String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException { - launcherConf.set(LauncherMapper.OOZIE_JOB_ID, jobId); - launcherConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId); - launcherConf.set(LauncherMapper.OOZIE_ACTION_DIR_PATH, actionDir.toString()); - launcherConf.set(LauncherMapper.OOZIE_ACTION_RECOVERY_ID, recoveryId); - launcherConf.set(LauncherMapper.ACTION_PREPARE_XML, prepareXML); + launcherConf.set(LauncherAMUtils.OOZIE_JOB_ID, jobId); + launcherConf.set(LauncherAMUtils.OOZIE_ACTION_ID, actionId); + launcherConf.set(LauncherAMUtils.OOZIE_ACTION_DIR_PATH, actionDir.toString()); + launcherConf.set(LauncherAMUtils.OOZIE_ACTION_RECOVERY_ID, recoveryId); + launcherConf.set(LauncherAMUtils.ACTION_PREPARE_XML, prepareXML); - actionConf.set(LauncherMapper.OOZIE_JOB_ID, jobId); - actionConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId); + actionConf.set(LauncherAMUtils.OOZIE_JOB_ID, jobId); + actionConf.set(LauncherAMUtils.OOZIE_ACTION_ID, actionId); if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) { List<String> purgedEntries = new ArrayList<String>(); @@ -171,9 +169,9 @@ public class LauncherHelper { if (succeeded) { Counters counters = runningJob.getCounters(); if (counters != null) { - Counters.Group group = counters.getGroup(LauncherMapper.COUNTER_GROUP); + Counters.Group group = counters.getGroup(LauncherAMUtils.COUNTER_GROUP); if (group != null) { - succeeded = group.getCounter(LauncherMapper.COUNTER_LAUNCHER_ERROR) == 0; + succeeded = group.getCounter(LauncherAMUtils.COUNTER_LAUNCHER_ERROR) == 0; } } } @@ -187,7 +185,7 @@ public class LauncherHelper { * @throws IOException */ public static boolean hasExternalChildJobs(Map<String, String> actionData) throws IOException { - return actionData.containsKey(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS); + return actionData.containsKey(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS); } /** @@ -197,7 +195,7 @@ public class LauncherHelper { * @throws IOException */ public static boolean hasOutputData(Map<String, String> actionData) throws IOException { - return actionData.containsKey(LauncherMapper.ACTION_DATA_OUTPUT_PROPS); + return actionData.containsKey(LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS); } /** @@ -207,7 +205,7 @@ public class LauncherHelper { * @throws IOException */ public static boolean hasStatsData(Map<String, String> actionData) throws IOException{ - return actionData.containsKey(LauncherMapper.ACTION_DATA_STATS); + return actionData.containsKey(LauncherAMUtils.ACTION_DATA_STATS); } /** @@ -217,7 +215,7 @@ public class LauncherHelper { * @throws IOException */ public static boolean hasIdSwap(Map<String, String> actionData) throws IOException { - return actionData.containsKey(LauncherMapper.ACTION_DATA_NEW_ID); + return actionData.containsKey(LauncherAMUtils.ACTION_DATA_NEW_ID); } /** @@ -226,7 +224,7 @@ public class LauncherHelper { * @return */ public static Path getActionDataSequenceFilePath(Path actionDir) { - return new Path(actionDir, LauncherMapper.ACTION_DATA_SEQUENCE_FILE); + return new Path(actionDir, LauncherAMUtils.ACTION_DATA_SEQUENCE_FILE); } /** @@ -269,35 +267,35 @@ public class LauncherHelper { if (file.equals(new Path(actionDir, "externalChildIds.properties"))) { is = fs.open(file); reader = new BufferedReader(new InputStreamReader(is)); - ret.put(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS, + ret.put(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS, IOUtils.getReaderAsString(reader, -1)); } else if (file.equals(new Path(actionDir, "newId.properties"))) { is = fs.open(file); reader = new BufferedReader(new InputStreamReader(is)); props = PropertiesUtils.readProperties(reader, -1); - ret.put(LauncherMapper.ACTION_DATA_NEW_ID, props.getProperty("id")); + ret.put(LauncherAMUtils.ACTION_DATA_NEW_ID, props.getProperty("id")); } - else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_OUTPUT_PROPS))) { - int maxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, + else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS))) { + int maxOutputData = conf.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024); is = fs.open(file); reader = new BufferedReader(new InputStreamReader(is)); - ret.put(LauncherMapper.ACTION_DATA_OUTPUT_PROPS, PropertiesUtils + ret.put(LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS, PropertiesUtils .propertiesToString(PropertiesUtils.readProperties(reader, maxOutputData))); } - else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_STATS))) { - int statsMaxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, + else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_STATS))) { + int statsMaxOutputData = conf.getInt(LauncherAMUtils.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, Integer.MAX_VALUE); is = fs.open(file); reader = new BufferedReader(new InputStreamReader(is)); - ret.put(LauncherMapper.ACTION_DATA_STATS, PropertiesUtils + ret.put(LauncherAMUtils.ACTION_DATA_STATS, PropertiesUtils .propertiesToString(PropertiesUtils.readProperties(reader, statsMaxOutputData))); } - else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_ERROR_PROPS))) { + else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_ERROR_PROPS))) { is = fs.open(file); reader = new BufferedReader(new InputStreamReader(is)); - ret.put(LauncherMapper.ACTION_DATA_ERROR_PROPS, IOUtils.getReaderAsString(reader, -1)); + ret.put(LauncherAMUtils.ACTION_DATA_ERROR_PROPS, IOUtils.getReaderAsString(reader, -1)); } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java deleted file mode 100644 index 4dba776..0000000 --- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import org.apache.hadoop.mapred.InputFormat; -import org.apache.oozie.service.ConfigurationService; -import org.apache.oozie.util.XLog; - -class LauncherInputFormatClassLocator { - private final XLog LOG = XLog.getLog(getClass()); - - static final String HADOOP_INPUT_FORMAT_CLASSNAME = "mapreduce.input.format.class"; - - private Class<? extends InputFormat> launcherInputFormatClass; - - /** - * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration. - * <p/> - * Priority: - * <ol> - * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li> - * <li><tt>OozieLauncherInputFormat</tt></li> - * </ol> - * - * @return - */ - Class<? extends InputFormat> locateOrGet() { - if (launcherInputFormatClass == null) { - launcherInputFormatClass = locate(); - } - - return launcherInputFormatClass; - } - - /** - * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration. - * <p/> - * Priority: - * <ol> - * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li> - * <li><tt>OozieLauncherInputFormat</tt></li> - * </ol> - * - * @return - */ - private Class<? extends InputFormat> locate() { - String inputFormatClassName = OozieLauncherInputFormat.class.getSimpleName(); - - final String configuredClassName = ConfigurationService.get( - JavaActionExecutor.OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_INPUT_FORMAT_CLASSNAME); - if (configuredClassName != null) { - inputFormatClassName = configuredClassName; - } - - try { - LOG.debug("Locating launcher input format class [{0}].", inputFormatClassName); - - final Class inputFormatClass = Class.forName(inputFormatClassName); - LOG.debug("Launcher input format class [{0}] located successfully.", inputFormatClassName); - - return inputFormatClass; - } catch (final ClassNotFoundException | ClassCastException e) { - LOG.warn("Could not load class [{0}], located [{1}] instead.", inputFormatClassName, - OozieLauncherInputFormat.class.getSimpleName()); - return OozieLauncherInputFormat.class; - } - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index 634a1cb..4427688 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -25,8 +25,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import javax.annotation.Nonnull; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -35,14 +33,6 @@ import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.client.ClientRMProxy; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; @@ -52,11 +42,6 @@ import org.apache.oozie.util.XmlUtils; import org.jdom.Element; import org.jdom.Namespace; -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; -import com.google.common.io.Closeables; - public class MapReduceActionExecutor extends JavaActionExecutor { public static final String OOZIE_ACTION_EXTERNAL_STATS_WRITE = "oozie.action.external.stats.write"; @@ -99,14 +84,14 @@ public class MapReduceActionExecutor extends JavaActionExecutor { String mainClass; Namespace ns = actionXml.getNamespace(); if (actionXml.getChild("streaming", ns) != null) { - mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, STREAMING_MAIN_CLASS_NAME); + mainClass = launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, STREAMING_MAIN_CLASS_NAME); } else { if (actionXml.getChild("pipes", ns) != null) { - mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName()); + mainClass = launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName()); } else { - mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName()); + mainClass = launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName()); } } return mainClass; @@ -125,7 +110,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { // Inject config-class for launcher to use for action Element e = actionXml.getChild("config-class", actionXml.getNamespace()); if (e != null) { - conf.set(LauncherMapper.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim()); + conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim()); } } @@ -363,7 +348,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { throw convertException(e); } - final String newId = actionData.get(LauncherMapper.ACTION_DATA_NEW_ID); + final String newId = actionData.get(LauncherAMUtils.ACTION_DATA_NEW_ID); // check the Hadoop job if newID is defined (which should be the case here) - otherwise perform the normal check() if (newId != null) { @@ -403,7 +388,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { } // run original check() if the MR action is completed or there are errors - otherwise mark it as RUNNING - if (jobCompleted || actionData.containsKey(LauncherMapper.ACTION_DATA_ERROR_PROPS)) { + if (jobCompleted || actionData.containsKey(LauncherAMUtils.ACTION_DATA_ERROR_PROPS)) { super.check(context, action); } else { context.setExternalStatus(RUNNING); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java index 8a24ac3..6541d52 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java @@ -60,7 +60,7 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor { @Override protected String getLauncherMain(Configuration launcherConf, Element actionXml) { - return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PIG_MAIN_CLASS_NAME); + return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, PIG_MAIN_CLASS_NAME); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java index d44bbc5..9591cd9 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java @@ -44,7 +44,7 @@ public class ShellActionExecutor extends JavaActionExecutor { @Override protected String getLauncherMain(Configuration launcherConf, Element actionXml) { - return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, ShellMain.class.getName()); + return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, ShellMain.class.getName()); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 00497a7..80d64ec 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -19,10 +19,8 @@ package org.apache.oozie.action.hadoop; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.oozie.action.ActionExecutorException; -import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.Services; import org.apache.oozie.service.SparkConfigurationService; @@ -154,7 +152,7 @@ public class SparkActionExecutor extends JavaActionExecutor { @Override protected String getLauncherMain(Configuration launcherConf, Element actionXml) { - return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME); + return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index 955f3b7..8fdc50c 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; @@ -64,7 +63,7 @@ public class SqoopActionExecutor extends JavaActionExecutor { @Override protected String getLauncherMain(Configuration launcherConf, Element actionXml) { - return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SQOOP_MAIN_CLASS_NAME); + return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, SQOOP_MAIN_CLASS_NAME); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index c60a458..5629a89 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1804,7 +1804,7 @@ will be the requeue interval for the actions which are waiting for a long time w </description> </property> - <!-- LauncherMapper --> + <!-- LauncherAMUtils --> <property> <name>oozie.action.max.output.data</name> <value>2048</value> http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java index 16ab729..6cee7a8 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java @@ -69,7 +69,7 @@ public class LauncherMainTester { } // by using NULL as permission, if an underlaying SecurityManager is in place // a security exception will be thrown. As there is not underlaying SecurityManager - // this tests that the delegation logic of the LauncherMapper SecurityManager is + // this tests that the delegation logic of the LauncherAMUtils SecurityManager is // correct for both checkPermission() signatures. sm.checkPermission(null); sm.checkPermission(null, sm.getSecurityContext()); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java index 199962e..3f000d5 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java @@ -31,8 +31,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import org.apache.oozie.action.hadoop.DistcpMain; - public class TestDistcpMain extends MainTestCase { @Override @@ -49,7 +47,7 @@ public class TestDistcpMain extends MainTestCase { writer.close(); Path outputDir = new Path(getFsTestCaseDir(), "output"); - jobConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, "org.apache.hadoop.tools.DistCp"); + jobConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, "org.apache.hadoop.tools.DistCp"); jobConf.set("mapreduce.job.tags", "" + System.currentTimeMillis()); setSystemProperty("oozie.job.launch.time", "" + System.currentTimeMillis()); @@ -75,7 +73,7 @@ public class TestDistcpMain extends MainTestCase { DistcpMain.main(new String[]{inputDir.toString(), outputDir.toString()}); assertTrue(getFileSystem().exists(outputDir)); assertTrue(hadoopIdsFile.exists()); - assertNotNull(LauncherMapper.getLocalFileContentStr(hadoopIdsFile, "", -1)); + assertNotNull(LauncherAMUtils.getLocalFileContentStr(hadoopIdsFile, "", -1)); fs.delete(outputDir,true); // Check exception handling http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 67c7d06..59a21c4 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -264,7 +264,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("org.apache.oozie.action.hadoop.JavaMain", ae.getLauncherMain(conf, actionXml)); assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPTS")); assertTrue(conf.get("mapreduce.map.java.opts").contains("JAVA-OPTS")); - assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherMapper.getMainArguments(conf))); + assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherAMUtils.getMainArguments(conf))); actionXml = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node> <configuration>" + @@ -2177,8 +2177,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } public void testSetRootLoggerLevel() throws Exception { - String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL; - String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherMapper.ROOT_LOGGER_LEVEL; + String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL; + String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherAMUtils.ROOT_LOGGER_LEVEL; // default should be INFO JavaActionExecutor jae = new JavaActionExecutor(); @@ -2211,7 +2211,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { private void testEmptyArgs(boolean nullArgsAllowed, String expectedExternalStatus, WorkflowAction.Status expectedStatus) throws Exception { - ConfigurationService.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed); + ConfigurationService.setBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed); String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java index 1b7724e..a3b3b7b 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java @@ -38,12 +38,6 @@ import java.io.Writer; import java.net.URI; import java.util.Map; -// TODO -// this whole class can be deleted - for now, just renamed the tests that fail -// These tests mostly validate LaunhcherMapper - with OOYA, LauncherMapper should be eliminated, too - -// With Hadoop 2.4.0, things work slightly differently (there is an exception in LauncherMapper.map()), also, -// SequenceFile.Reader got deprecated public class TestLauncher extends XFsTestCase { @Override @@ -58,245 +52,6 @@ public class TestLauncher extends XFsTestCase { super.tearDown(); } - private RunningJob _test(String... arg) throws Exception { - Path actionDir = getFsTestCaseDir(); - - File jar = IOUtils.createJar(new File(getTestCaseDir()), "launcher.jar", LauncherMapper.class, - LauncherMainException.class, - LauncherSecurityManager.class, LauncherException.class, LauncherMainTester.class); - - FileSystem fs = getFileSystem(); - - Path launcherJar = new Path(actionDir, "launcher.jar"); - fs.copyFromLocalFile(new Path(jar.toString()), launcherJar); - - Configuration appConf = Services.get().get(HadoopAccessorService.class). - createConfiguration(new URI(getNameNodeUri()).getAuthority()); - appConf.set("user.name", getTestUser()); - appConf.setInt("mapred.map.tasks", 1); - appConf.setInt("mapred.map.max.attempts", 1); - appConf.setInt("mapred.reduce.max.attempts", 1); - - appConf.set("mapreduce.framework.name", "yarn"); - appConf.set("mapred.job.tracker", getJobTrackerUri()); - appConf.set("fs.default.name", getNameNodeUri()); - - - LauncherHelper.setupMainClass(appConf, LauncherMainTester.class.getName()); - LauncherHelper.setupMainArguments(appConf, arg); - - Configuration actionConf = new XConfiguration(); - LauncherHelper.setupLauncherInfo(appConf, "1", "1@a", actionDir, "1@a-0", actionConf, ""); - LauncherHelper.setupYarnRestartHandling(appConf, appConf, "1@a", System.currentTimeMillis()); - - assertEquals("1", actionConf.get("oozie.job.id")); - assertEquals("1@a", actionConf.get("oozie.action.id")); - - DistributedCache.addFileToClassPath(new Path(launcherJar.toUri().getPath()), appConf); - - JobClient jobClient = createJobClient(); - - final RunningJob runningJob = jobClient.submitJob(new JobConf(appConf)); - - System.out.println("Action Dir: " + actionDir); - System.out.println("LauncherMapper ID: " + runningJob.getJobID().toString()); - - waitFor(180 * 1000, new Predicate() { - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - - assertTrue(appConf.get("oozie.action.prepare.xml").equals("")); - return runningJob; - - } - - public void ___testEmpty() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test(); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertFalse(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertTrue(LauncherHelper.isMainSuccessful(runningJob)); - assertFalse(LauncherHelper.hasOutputData(actionData)); - assertFalse(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - } - - public void ___testExit0() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test("exit0"); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertFalse(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertTrue(LauncherHelper.isMainSuccessful(runningJob)); - assertFalse(LauncherHelper.hasOutputData(actionData)); - assertFalse(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - } - - public void ___testExit1() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test("exit1"); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertFalse(LauncherHelper.isMainSuccessful(runningJob)); - assertFalse(LauncherHelper.hasOutputData(actionData)); - assertFalse(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertTrue(actionData.containsKey(LauncherMapper.ACTION_DATA_ERROR_PROPS)); - } - - public void ___testException() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test("exception"); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertFalse(LauncherHelper.isMainSuccessful(runningJob)); - assertFalse(LauncherHelper.hasOutputData(actionData)); - assertFalse(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - } - - public void __testThrowable() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test("throwable"); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertFalse(LauncherHelper.isMainSuccessful(runningJob)); - assertFalse(LauncherHelper.hasOutputData(actionData)); - assertFalse(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - } - - public void __testOutput() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test("out"); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertTrue(LauncherHelper.isMainSuccessful(runningJob)); - assertTrue(LauncherHelper.hasOutputData(actionData)); - assertFalse(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - } - - public void __testNewId() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test("id"); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertTrue(LauncherHelper.isMainSuccessful(runningJob)); - assertFalse(LauncherHelper.hasOutputData(actionData)); - assertTrue(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - } - - public void __testSecurityManager() throws Exception { - Path actionDir = getFsTestCaseDir(); - FileSystem fs = getFileSystem(); - final RunningJob runningJob = _test("securityManager"); - waitFor(2000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); - - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf); - assertFalse(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir))); - assertTrue(LauncherHelper.isMainDone(runningJob)); - assertTrue(LauncherHelper.isMainSuccessful(runningJob)); - assertFalse(LauncherHelper.hasOutputData(actionData)); - assertFalse(LauncherHelper.hasIdSwap(actionData)); - assertTrue(LauncherHelper.isMainDone(runningJob)); - } - // Test to ensure that the property value "oozie.action.prepare.xml" in the configuration of the job is an empty // string when there is no prepare block in workflow XML or there is one with no prepare actions in it public void testSetupLauncherInfoWithEmptyPrepareXML() throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java deleted file mode 100644 index a6ccc5a..0000000 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.oozie.service.ConfigurationService; - -public class TestLauncherInputFormatClassLocator extends ActionExecutorTestCase { - - public void testInputFormatClassnameNotSet() { - ConfigurationService.set( - "oozie.action.launcher.mapreduce.input.format.class", - ""); - - final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet(); - - assertEquals("default launcher input format is found when configuration is left empty", - OozieLauncherInputFormat.class, - inputFormatClass); - } - - public void testInputFormatClassnameDefault() { - final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet(); - - assertEquals("default launcher input format is found when configuration is left on default", - OozieLauncherInputFormat.class, - inputFormatClass); - } - - public void testInputFormatClassnameSet() { - ConfigurationService.set( - "oozie.action.launcher.mapreduce.input.format.class", - InputFormat.class.getName()); - - final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet(); - - assertEquals("Hadoop launcher input format is found when configuration is set", - InputFormat.class, - inputFormatClass); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java index a4b9943..8c716e8 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java @@ -70,7 +70,7 @@ public class TestMapReduceMain extends MainTestCase { MapReduceMain.main(new String[0]); assertTrue(newId.exists()); - assertNotNull(LauncherMapper.getLocalFileContentStr(newId, "", -1)); + assertNotNull(LauncherAMUtils.getLocalFileContentStr(newId, "", -1)); return null; } http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java index 72be0a2..8d0b441 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java @@ -268,7 +268,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { public void testEnvVar() throws Exception { Services.get().destroy(); Services services = new Services(); - services.getConf().setInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 8 * 1042); + services.getConf().setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 8 * 1042); services.init(); FileSystem fs = getFileSystem(); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java index 7bc81c8..4cb2530 100644 --- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java +++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java @@ -21,8 +21,7 @@ package org.apache.oozie.service; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.action.hadoop.CredentialsProviderFactory; import org.apache.oozie.action.hadoop.DistcpActionExecutor; -import org.apache.oozie.action.hadoop.JavaActionExecutor; -import org.apache.oozie.action.hadoop.LauncherMapper; +import org.apache.oozie.action.hadoop.LauncherAMUtils; import org.apache.oozie.command.coord.CoordActionInputCheckXCommand; import org.apache.oozie.command.coord.CoordSubmitXCommand; import org.apache.oozie.command.wf.JobXCommand; @@ -210,7 +209,7 @@ public class TestConfigurationService extends XTestCase { assertEquals("300000", ConfigurationService.get(JPAService.CONF_VALIDATE_DB_CONN_EVICTION_INTERVAL).trim()); assertEquals("10", ConfigurationService.get(JPAService.CONF_VALIDATE_DB_CONN_EVICTION_NUM).trim()); - assertEquals(2048, ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA)); + assertEquals(2048, ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA)); assertEquals("http://0.0.0.0:11000/oozie?job=";, ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL)); assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED)); @@ -245,7 +244,7 @@ public class TestConfigurationService extends XTestCase { assertEquals("org.apache.oozie.dependency.FSURIHandler", ConfigurationService.getStrings(URIHandlerService.URI_HANDLERS)[0]); assertEquals(cl.getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false), - ConfigurationService.getBoolean(LauncherMapper.HADOOP2_WORKAROUND_DISTRIBUTED_CACHE)); + ConfigurationService.getBoolean(LauncherAMUtils.HADOOP2_WORKAROUND_DISTRIBUTED_CACHE)); assertEquals("org.apache.oozie.event.MemoryEventQueue", (ConfigurationService.getClass(cl.getConf(), EventHandlerService.CONF_EVENT_QUEUE).getName())); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 3cd95af..de6aff5 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2918 Delete LauncherMapper and its test (asasvari via pbacsko) OOZIE-2733 change org.apache.hadoop.fs.permission.AccessControlException to org.apache.hadoop.security.AccessControlException (gezapeti) OOZIE-2884 consolidate hadoop versions in pomfiles (Artem Ervits via gezapeti) OOZIE-2794 Hive and Pig tests failing when multiple pre-commit jobs are running (gezapeti) http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java index d7cf9d9..d17d5be 100644 --- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java +++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java @@ -55,7 +55,7 @@ public class DistcpMain extends JavaMain { Configuration actionConf = loadActionConf(); LauncherMain.killChildYarnJobs(actionConf); String logFile = setUpDistcpLog4J(actionConf); - Class<?> klass = actionConf.getClass(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, + Class<?> klass = actionConf.getClass(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, org.apache.hadoop.tools.DistCp.class); System.out.println("Main class : " + klass.getName()); System.out.println("Arguments :"); @@ -125,7 +125,7 @@ public class DistcpMain extends JavaMain { String logFile = new File("distcp-oozie-" + hadoopJobId + ".log").getAbsolutePath(); String logLevel = distcpConf.get("oozie.distcp.log.level", "INFO"); - String rootLogLevel = distcpConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + String rootLogLevel = distcpConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO"); log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A"); log4jProperties.setProperty("log4j.logger.org.apache.hadoop.tools", logLevel + ", A, jobid"); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index 7b152f2..fb3fabf 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -136,7 +136,7 @@ public class HiveMain extends LauncherMain { String logFile = new File("hive-oozie-" + hadoopJobId + ".log").getAbsolutePath(); String logLevel = hiveConf.get("oozie.hive.log.level", "INFO"); - String rootLogLevel = hiveConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + String rootLogLevel = hiveConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO"); log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A"); log4jProperties.setProperty("log4j.logger.org.apache.hadoop.hive", logLevel + ", A"); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java index 71ee641..0e99fac 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java @@ -170,7 +170,7 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { assertNotNull(context.getExternalChildIDs()); //while this works in a real cluster, it does not with miniMR //assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0); - //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty()); + //assertTrue(!actionData.get(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty()); assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } @@ -193,7 +193,7 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { assertNotNull(context.getAction().getExternalChildIDs()); //while this works in a real cluster, it does not with miniMR //assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0); - //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty()); + //assertTrue(!actionData.get(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty()); assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } @@ -266,7 +266,7 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { Context context = createContext(getActionScriptXml()); Namespace ns = Namespace.getNamespace("uri:oozie:hive-action:0.2"); submitAction(context, ns); - FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML)); + FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherAMUtils.ACTION_CONF_XML)); XConfiguration conf = new XConfiguration(); conf.addResource(os); assertNull(conf.get("oozie.HadoopAccessorService.created")); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java index bbd6246..35c2865 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java @@ -135,7 +135,8 @@ public class TestHiveMain extends MainTestCase { setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath()); setSystemProperty("oozie.action.externalChildIDs", externalChildIdsFile.getAbsolutePath()); - new LauncherSecurityManager(); + LauncherAM.LauncherSecurityManager launcherSecurityManager = new LauncherAM.LauncherSecurityManager(); + launcherSecurityManager.enable(); String user = System.getProperty("user.name"); try { os = new FileOutputStream(hiveSite); @@ -145,10 +146,10 @@ public class TestHiveMain extends MainTestCase { HiveMain.main(null); } catch (SecurityException ex) { - if (LauncherSecurityManager.getExitInvoked()) { - System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")"); - System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")"); - if (LauncherSecurityManager.getExitCode() != 0) { + if (launcherSecurityManager.getExitInvoked()) { + System.out.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")"); + System.err.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")"); + if (launcherSecurityManager.getExitCode() != 0) { fail(); } } @@ -163,7 +164,7 @@ public class TestHiveMain extends MainTestCase { } assertTrue(externalChildIdsFile.exists()); - assertNotNull(LauncherMapper.getLocalFileContentStr(externalChildIdsFile, "", -1)); + assertNotNull(LauncherAMUtils.getLocalFileContentStr(externalChildIdsFile, "", -1)); //TODO: I cannot figure out why when log file is not created in this testcase, it works when running in Launcher // Properties props = new Properties(); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java index 512730a..fd6c283 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java @@ -58,7 +58,7 @@ public class FSLauncherURIHandler implements LauncherURIHandler { FileSystem fs = FileSystem.get(uri, conf); Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(getNormalizedPath(uri))); if (pathArr != null && pathArr.length > 0) { - int fsGlobMax = conf.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, 1000); + int fsGlobMax = conf.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX, 1000); if (pathArr.length > fsGlobMax) { throw new LauncherException("exceeds max number (" + fsGlobMax + ") of files/dirs to delete in <prepare>"); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java index c3e3d3f..f98c781 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@ -28,7 +28,7 @@ public class JavaMain extends LauncherMain { public static final String JAVA_MAIN_CLASS = "oozie.action.java.main"; /** - * @param args Invoked from LauncherMapper:map() + * @param args Invoked from LauncherAM:run() * @throws Exception */ public static void main(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 4f252d1..bac17b2 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -537,7 +537,7 @@ public class LauncherAM { } private String[] getMainArguments(Configuration conf) { - return LauncherMapper.getMainArguments(conf); + return LauncherAMUtils.getMainArguments(conf); } public static class LauncherSecurityManager extends SecurityManager { @@ -593,6 +593,11 @@ public class LauncherAM { System.setSecurityManager(originalSecurityManager); } } + + public void reset() { + exitInvoked = false; + exitCode = 0; + } } public enum OozieActionResult { @@ -611,4 +616,4 @@ public class LauncherAM { return yarnStatus; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java new file mode 100644 index 0000000..74c7af0 --- /dev/null +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import com.google.common.base.Strings; +import org.apache.hadoop.conf.Configuration; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; + +public class LauncherAMUtils { + + static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class"; + + static final String ACTION_PREFIX = "oozie.action."; + public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = ACTION_PREFIX + "max.output.data"; + static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = ACTION_PREFIX + "main.arg.count"; + static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = ACTION_PREFIX + "main.arg."; + static final String CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE = "oozie.external.stats.max.size"; + static final String OOZIE_ACTION_CONFIG_CLASS = ACTION_PREFIX + "config.class"; + static final String CONF_OOZIE_ACTION_FS_GLOB_MAX = ACTION_PREFIX + "fs.glob.max"; + static final String CONF_OOZIE_NULL_ARGS_ALLOWED = ACTION_PREFIX + "null.args.allowed"; + + static final String COUNTER_GROUP = "oozie.launcher"; + static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error"; + + static final String OOZIE_JOB_ID = "oozie.job.id"; + static final String OOZIE_ACTION_ID = ACTION_PREFIX + "id"; + static final String OOZIE_ACTION_RECOVERY_ID = ACTION_PREFIX + "recovery.id"; + + static final String OOZIE_ACTION_DIR_PATH = ACTION_PREFIX + "dir.path"; + static final String ACTION_CONF_XML = "action.xml"; + static final String ACTION_PREPARE_XML = "oozie.action.prepare.xml"; + static final String ACTION_DATA_SEQUENCE_FILE = "action-data.seq"; // COMBO FILE + static final String ACTION_DATA_EXTERNAL_CHILD_IDS = "externalChildIDs"; + static final String ACTION_DATA_OUTPUT_PROPS = "output.properties"; + static final String ACTION_DATA_STATS = "stats.properties"; + static final String ACTION_DATA_NEW_ID = "newId"; + static final String ACTION_DATA_ERROR_PROPS = "error.properties"; + public static final String HADOOP2_WORKAROUND_DISTRIBUTED_CACHE = "oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache"; + public static final String PROPAGATION_CONF_XML = "propagation-conf.xml"; + public static final String ROOT_LOGGER_LEVEL = "rootlogger.log.level"; + + + public static String getLocalFileContentStr(File file, String type, int maxLen) throws LauncherException, IOException { + StringBuffer sb = new StringBuffer(); + FileReader reader = new FileReader(file); + char[] buffer = new char[2048]; + int read; + int count = 0; + while ((read = reader.read(buffer)) > -1) { + count += read; + if (maxLen > -1 && count > maxLen) { + throw new LauncherException(type + " data exceeds its limit ["+ maxLen + "]"); + } + sb.append(buffer, 0, read); + } + reader.close(); + return sb.toString(); + } + + public static String[] getMainArguments(Configuration conf) { + String[] args = new String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)]; + + String[] retArray; + + if (conf.getBoolean(CONF_OOZIE_NULL_ARGS_ALLOWED, true)) { + for (int i = 0; i < args.length; i++) { + args[i] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i); + } + + retArray = args; + } else { + int pos = 0; + for (int i = 0; i < args.length; i++) { + String arg = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i); + if (!Strings.isNullOrEmpty(arg)) { + args[pos++] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i); + } + } + + // this is to skip null args, that is <arg></arg> in the workflow XML -- in this case, + // args[] might look like {"arg1", "arg2", null, null} at this point + retArray = new String[pos]; + System.arraycopy(args, 0, retArray, 0, pos); + } + + return retArray; + } +} +
