Repository: oozie Updated Branches: refs/heads/master 79ccfdd78 -> c09561ff2
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java deleted file mode 100644 index d0d3629..0000000 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java +++ /dev/null @@ -1,715 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.security.Permission; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.StringTokenizer; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reporter; -import org.xml.sax.SAXException; - -import com.google.common.base.Strings; -import javax.xml.parsers.ParserConfigurationException; - -// TODO: OYA: Delete :) -public class LauncherMapper<K1, V1, K2, V2> implements Runnable { - - static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class"; - - static final String ACTION_PREFIX = "oozie.action."; - public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = ACTION_PREFIX + "max.output.data"; - static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = ACTION_PREFIX + "main.arg.count"; - static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = ACTION_PREFIX + "main.arg."; - static final String CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE = "oozie.external.stats.max.size"; - static final String OOZIE_ACTION_CONFIG_CLASS = ACTION_PREFIX + "config.class"; - static final String CONF_OOZIE_ACTION_FS_GLOB_MAX = ACTION_PREFIX + "fs.glob.max"; - static final String CONF_OOZIE_NULL_ARGS_ALLOWED = ACTION_PREFIX + "null.args.allowed"; - - static final String COUNTER_GROUP = "oozie.launcher"; - static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error"; - - static final String OOZIE_JOB_ID = "oozie.job.id"; - static final String OOZIE_ACTION_ID = ACTION_PREFIX + "id"; - static final String OOZIE_ACTION_RECOVERY_ID = ACTION_PREFIX + "recovery.id"; - - static final String OOZIE_ACTION_DIR_PATH = ACTION_PREFIX + "dir.path"; - static final String ACTION_CONF_XML = "action.xml"; - static final String ACTION_PREPARE_XML = "oozie.action.prepare.xml"; - static final String ACTION_DATA_SEQUENCE_FILE = "action-data.seq"; // COMBO FILE - static final String ACTION_DATA_EXTERNAL_CHILD_IDS = "externalChildIDs"; - static final String ACTION_DATA_OUTPUT_PROPS = "output.properties"; - static final String ACTION_DATA_STATS = "stats.properties"; - static final String ACTION_DATA_NEW_ID = "newId"; - static final String ACTION_DATA_ERROR_PROPS = "error.properties"; - public static final String HADOOP2_WORKAROUND_DISTRIBUTED_CACHE = "oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache"; - public static final String PROPAGATION_CONF_XML = "propagation-conf.xml"; - public static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id"; - public static final String ROOT_LOGGER_LEVEL = "rootlogger.log.level"; - - private void setRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws LauncherException { - try { - String jobId = launcherConf.get("mapred.job.id"); - Path path = new Path(actionDir, recoveryId); - FileSystem fs = FileSystem.get(path.toUri(), launcherConf); - if (!fs.exists(path)) { - try { - java.io.Writer writer = new OutputStreamWriter(fs.create(path)); - writer.write(jobId); - writer.close(); - } - catch (IOException ex) { - failLauncher(0, "IO error", ex); - } - } - else { - InputStream is = fs.open(path); - BufferedReader reader = new BufferedReader(new InputStreamReader(is)); - String id = reader.readLine(); - reader.close(); - if (!jobId.equals(id)) { - failLauncher(0, MessageFormat.format( - "Hadoop job Id mismatch, action file [{0}] declares Id [{1}] current Id [{2}]", path, id, - jobId), null); - } - - } - } - catch (IOException ex) { - failLauncher(0, "IO error", ex); - } - } - - private Configuration appConf; - private Path actionDir; - private ScheduledThreadPoolExecutor timer; - - private boolean configFailure = false; - private LauncherException configureFailureEx; - private Map<String,String> actionData; - - public LauncherMapper() { - actionData = new HashMap<String,String>(); - } - - public void configure(Configuration appConf) { - System.out.println(); - System.out.println("Oozie Launcher starts"); - System.out.println(); - this.appConf = appConf; - actionDir = new Path(getAppConf().get(OOZIE_ACTION_DIR_PATH)); - String recoveryId = appConf.get(OOZIE_ACTION_RECOVERY_ID, null); - try { - setRecoveryId(appConf, actionDir, recoveryId); - } - catch (LauncherException ex) { - System.out.println("Launcher config error "+ex.getMessage()); - configureFailureEx = ex; - configFailure = true; - } - } - - - public void map(K1 key, V1 value, OutputCollector<K2, V2> collector, Reporter reporter) throws IOException { - SecurityManager initialSecurityManager = System.getSecurityManager(); - try { - if (configFailure) { - throw configureFailureEx; - } - else { - String mainClass = getAppConf().get(CONF_OOZIE_ACTION_MAIN_CLASS); - if (getAppConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) { - System.err.println("WARNING, workaround for Hadoop 2.0.2-alpha distributed cached issue (MAPREDUCE-4820) enabled"); - } - String msgPrefix = "Main class [" + mainClass + "], "; - int errorCode = 0; - Throwable errorCause = null; - String errorMessage = null; - try { - new LauncherSecurityManager(); - } - catch (SecurityException ex) { - errorMessage = "Could not set LauncherSecurityManager"; - errorCause = ex; - } - - try { - setupHeartBeater(reporter); - - setupMainConfiguration(); - - // Propagating the conf to use by child job. - propagateToHadoopConf(); - - try { - System.out.println("Starting the execution of prepare actions"); - executePrepare(); - System.out.println("Completed the execution of prepare actions successfully"); - } catch (Exception ex) { - System.out.println("Prepare execution in the Launcher Mapper has failed"); - throw new LauncherException(ex.getMessage(), ex); - } - - String[] args = getMainArguments(getAppConf()); - - printContentsOfCurrentDir(); - - System.out.println(); - System.out.println("Oozie Java/Map-Reduce/Pig action launcher-job configuration"); - System.out.println("================================================================="); - System.out.println("Workflow job id : " + System.getProperty("oozie.job.id")); - System.out.println("Workflow action id: " + System.getProperty("oozie.action.id")); - System.out.println(); - System.out.println("Classpath :"); - System.out.println("------------------------"); - StringTokenizer st = new StringTokenizer(System.getProperty("java.class.path"), ":"); - while (st.hasMoreTokens()) { - System.out.println(" " + st.nextToken()); - } - System.out.println("------------------------"); - System.out.println(); - System.out.println("Main class : " + mainClass); - System.out.println(); - System.out.println("Maximum output : " - + getAppConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024)); - System.out.println(); - printArgs("Arguments :", args); - - System.out.println(); - System.out.println("Java System Properties:"); - System.out.println("------------------------"); - System.getProperties().store(System.out, ""); - System.out.flush(); - System.out.println("------------------------"); - System.out.println(); - - System.out.println("================================================================="); - System.out.println(); - System.out.println(">>> Invoking Main class now >>>"); - System.out.println(); - System.out.flush(); - - try { - Class klass = getAppConf().getClass(CONF_OOZIE_ACTION_MAIN_CLASS, Object.class); - Method mainMethod = klass.getMethod("main", String[].class); - mainMethod.invoke(null, (Object) args); - } - catch (InvocationTargetException ex) { - // Get what actually caused the exception - Throwable cause = ex.getCause(); - // If we got a JavaMainException from JavaMain, then we need to unwrap it - if (JavaMain.JavaMainException.class.isInstance(cause)) { - cause = cause.getCause(); - } - if (LauncherMainException.class.isInstance(cause)) { - errorMessage = msgPrefix + "exit code [" +((LauncherMainException)ex.getCause()).getErrorCode() - + "]"; - errorCause = null; - } - else if (SecurityException.class.isInstance(cause)) { - if (LauncherSecurityManager.getExitInvoked()) { - System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() - + ")"); - System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() - + ")"); - // if 0 main() method finished successfully - // ignoring - errorCode = LauncherSecurityManager.getExitCode(); - if (errorCode != 0) { - errorMessage = msgPrefix + "exit code [" + errorCode + "]"; - errorCause = null; - } - } - } - else { - throw ex; - } - } - finally { - System.out.println(); - System.out.println("<<< Invocation of Main class completed <<<"); - System.out.println(); - } - if (errorMessage == null) { - handleActionData(); - if (actionData.get(ACTION_DATA_OUTPUT_PROPS) != null) { - System.out.println(); - System.out.println("Oozie Launcher, capturing output data:"); - System.out.println("======================="); - System.out.println(actionData.get(ACTION_DATA_OUTPUT_PROPS)); - System.out.println(); - System.out.println("======================="); - System.out.println(); - } - if (actionData.get(ACTION_DATA_NEW_ID) != null) { - System.out.println(); - System.out.println("Oozie Launcher, propagating new Hadoop job id to Oozie"); - System.out.println("======================="); - System.out.println(actionData.get(ACTION_DATA_NEW_ID)); - System.out.println("======================="); - System.out.println(); - } - } - } - catch (NoSuchMethodException ex) { - errorMessage = msgPrefix + "main() method not found"; - errorCause = ex; - } - catch (InvocationTargetException ex) { - errorMessage = msgPrefix + "main() threw exception"; - errorCause = ex.getTargetException(); - } - catch (Throwable ex) { - errorMessage = msgPrefix + "exception invoking main()"; - errorCause = ex; - } - finally { - destroyHeartBeater(); - if (errorMessage != null) { - failLauncher(errorCode, errorMessage, errorCause); - } - } - } - } - catch (LauncherException ex) { - reporter.incrCounter(COUNTER_GROUP, COUNTER_LAUNCHER_ERROR, 1); - System.out.println(); - System.out.println("Oozie Launcher failed, finishing Hadoop job gracefully"); - System.out.println(); - } - finally { - uploadActionDataToHDFS(); - resetSecurityManager(initialSecurityManager); - } - } - - public void close() throws IOException { - System.out.println(); - System.out.println("Oozie Launcher ends"); - System.out.println(); - } - - /** - * Pushing all important conf to hadoop conf for the action - */ - private void propagateToHadoopConf() throws IOException { - Configuration propagationConf = new Configuration(false); - if (System.getProperty(OOZIE_ACTION_ID) != null) { - propagationConf.set(OOZIE_ACTION_ID, System.getProperty(OOZIE_ACTION_ID)); - } - if (System.getProperty(OOZIE_JOB_ID) != null) { - propagationConf.set(OOZIE_JOB_ID, System.getProperty(OOZIE_JOB_ID)); - } - if(System.getProperty(OOZIE_LAUNCHER_JOB_ID) != null) { - propagationConf.set(OOZIE_LAUNCHER_JOB_ID, System.getProperty(OOZIE_LAUNCHER_JOB_ID)); - } - - // loading action conf prepared by Oozie - Configuration actionConf = LauncherMain.loadActionConf(); - - if(actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS) != null) { - propagationConf.set(LauncherMain.MAPREDUCE_JOB_TAGS, - actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS)); - } - - propagationConf.writeXml(new FileWriter(PROPAGATION_CONF_XML)); - Configuration.dumpConfiguration(propagationConf, new OutputStreamWriter(System.out)); - Configuration.addDefaultResource(PROPAGATION_CONF_XML); - } - - protected Configuration getAppConf() { - return appConf; - } - - private void handleActionData() throws IOException, LauncherException { - // external child IDs - String externalChildIdsProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS); - if (externalChildIdsProp != null) { - File externalChildIDs = new File(externalChildIdsProp); - if (externalChildIDs.exists()) { - actionData.put(ACTION_DATA_EXTERNAL_CHILD_IDS, getLocalFileContentStr(externalChildIDs, "", -1)); - } - } - - // external stats - String statsProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_STATS); - if (statsProp != null) { - File actionStatsData = new File(statsProp); - if (actionStatsData.exists()) { - int statsMaxOutputData = getAppConf().getInt(CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, Integer.MAX_VALUE); - actionData.put(ACTION_DATA_STATS, getLocalFileContentStr(actionStatsData, "Stats", statsMaxOutputData)); - } - } - - // output data - String outputProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS); - if (outputProp != null) { - File actionOutputData = new File(outputProp); - if (actionOutputData.exists()) { - int maxOutputData = getAppConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024); - actionData.put(ACTION_DATA_OUTPUT_PROPS, - getLocalFileContentStr(actionOutputData, "Output", maxOutputData)); - } - } - - // id swap - String newIdProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID); - if (newIdProp != null) { - File newId = new File(newIdProp); - if (newId.exists()) { - actionData.put(ACTION_DATA_NEW_ID, getLocalFileContentStr(newId, "", -1)); - } - } - } - - public static String getLocalFileContentStr(File file, String type, int maxLen) throws LauncherException, IOException { - StringBuffer sb = new StringBuffer(); - FileReader reader = new FileReader(file); - char[] buffer = new char[2048]; - int read; - int count = 0; - while ((read = reader.read(buffer)) > -1) { - count += read; - if (maxLen > -1 && count > maxLen) { - throw new LauncherException(type + " data exceeds its limit ["+ maxLen + "]"); - } - sb.append(buffer, 0, read); - } - reader.close(); - return sb.toString(); - } - - private void uploadActionDataToHDFS() throws IOException { - if (!actionData.isEmpty()) { - Path finalPath = new Path(actionDir, ACTION_DATA_SEQUENCE_FILE); - FileSystem fs = FileSystem.get(finalPath.toUri(), getAppConf()); - // upload into sequence file - System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: " - + new Path(actionDir, ACTION_DATA_SEQUENCE_FILE).toUri()); - - SequenceFile.Writer wr = null; - try { - wr = SequenceFile.createWriter(fs, getAppConf(), finalPath, Text.class, Text.class); - if (wr != null) { - for (Entry<String, String> entry : actionData.entrySet()) { - wr.append(new Text(entry.getKey()), new Text(entry.getValue())); - } - } - else { - throw new IOException("SequenceFile.Writer is null for " + finalPath); - } - } - catch(IOException e) { - e.printStackTrace(); - throw e; - } - finally { - if (wr != null) { - wr.close(); - } - } - } - } - - private void setupMainConfiguration() throws IOException { - Path pathNew = new Path(new Path(actionDir, ACTION_CONF_XML), - new Path(new File(ACTION_CONF_XML).getAbsolutePath())); - FileSystem fs = FileSystem.get(pathNew.toUri(), getAppConf()); - fs.copyToLocalFile(new Path(actionDir, ACTION_CONF_XML), new Path(new File(ACTION_CONF_XML).getAbsolutePath())); - - System.setProperty("oozie.launcher.job.id", getAppConf().get("mapred.job.id")); - System.setProperty(OOZIE_JOB_ID, getAppConf().get(OOZIE_JOB_ID)); - System.setProperty(OOZIE_ACTION_ID, getAppConf().get(OOZIE_ACTION_ID)); - System.setProperty("oozie.action.conf.xml", new File(ACTION_CONF_XML).getAbsolutePath()); - System.setProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS, - new File(ACTION_DATA_EXTERNAL_CHILD_IDS).getAbsolutePath()); - System.setProperty(ACTION_PREFIX + ACTION_DATA_STATS, new File(ACTION_DATA_STATS).getAbsolutePath()); - System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath()); - System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath()); - System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS, new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath()); - if (getAppConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME) != null) { - System.setProperty(LauncherMain.OOZIE_JOB_LAUNCH_TIME, - getAppConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME)); - } - - String actionConfigClass = getAppConf().get(OOZIE_ACTION_CONFIG_CLASS); - if (actionConfigClass != null) { - System.setProperty(OOZIE_ACTION_CONFIG_CLASS, actionConfigClass); - } - } - - // Method to execute the prepare actions - private void executePrepare() throws IOException, LauncherException, ParserConfigurationException, SAXException { - String prepareXML = getAppConf().get(ACTION_PREPARE_XML); - if (prepareXML != null) { - if (!prepareXML.equals("")) { - Configuration actionConf = new Configuration(getAppConf()); - String actionXml = System.getProperty("oozie.action.conf.xml"); - actionConf.addResource(new Path("file:///", actionXml)); - PrepareActionsDriver.doOperations(prepareXML, actionConf); - } else { - System.out.println("There are no prepare actions to execute."); - } - } - } - - public static String[] getMainArguments(Configuration conf) { - String[] args = new String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)]; - - String[] retArray; - - if (conf.getBoolean(CONF_OOZIE_NULL_ARGS_ALLOWED, true)) { - for (int i = 0; i < args.length; i++) { - args[i] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i); - } - - retArray = args; - } else { - int pos = 0; - for (int i = 0; i < args.length; i++) { - String arg = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i); - if (!Strings.isNullOrEmpty(arg)) { - args[pos++] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i); - } - } - - // this is to skip null args, that is <arg></arg> in the workflow XML -- in this case, - // args[] might look like {"arg1", "arg2", null, null} at this point - retArray = new String[pos]; - System.arraycopy(args, 0, retArray, 0, pos); - } - - return retArray; - } - - private void setupHeartBeater(Reporter reporter) { - timer = new ScheduledThreadPoolExecutor(1); - timer.scheduleAtFixedRate(new LauncherMapper(reporter), 0, 30, TimeUnit.SECONDS); - } - - private void destroyHeartBeater() { - timer.shutdownNow(); - } - - private Reporter reporter; - - private LauncherMapper(Reporter reporter) { - this.reporter = reporter; - } - - @Override - public void run() { - System.out.println("Heart beat"); - reporter.progress(); - } - - private void failLauncher(int errorCode, String reason, Throwable ex) throws LauncherException { - if (ex != null) { - reason += ", " + ex.getMessage(); - } - Properties errorProps = new Properties(); - errorProps.setProperty("error.code", Integer.toString(errorCode)); - errorProps.setProperty("error.reason", reason); - if (ex != null) { - if (ex.getMessage() != null) { - errorProps.setProperty("exception.message", ex.getMessage()); - } - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - ex.printStackTrace(pw); - pw.close(); - errorProps.setProperty("exception.stacktrace", sw.toString()); - } - StringWriter sw = new StringWriter(); - try { - errorProps.store(sw, ""); - sw.close(); - actionData.put(ACTION_DATA_ERROR_PROPS, sw.toString()); - - // external child IDs - String externalChildIdsProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS); - if (externalChildIdsProp != null) { - File externalChildIDs = new File(externalChildIdsProp); - if (externalChildIDs.exists()) { - actionData.put(ACTION_DATA_EXTERNAL_CHILD_IDS, getLocalFileContentStr(externalChildIDs, "", -1)); - } - } - } - catch (IOException ioe) { - throw new LauncherException(ioe.getMessage(), ioe); - } - finally { - System.out.print("Failing Oozie Launcher, " + reason + "\n"); - System.err.print("Failing Oozie Launcher, " + reason + "\n"); - if (ex != null) { - ex.printStackTrace(System.out); - ex.printStackTrace(System.err); - } - } - throw new LauncherException(reason, ex); - } - - /** - * Print files and directories in current directory. Will list files in the sub-directory (only 1 level deep) - */ - protected void printContentsOfCurrentDir() { - File folder = new File("."); - System.out.println(); - System.out.println("Files in current dir:" + folder.getAbsolutePath()); - System.out.println("======================"); - - File[] listOfFiles = folder.listFiles(); - - if (listOfFiles != null) { - for (File fileName : listOfFiles) { - if (fileName.isFile()) { - System.out.println("File: " + fileName.getName()); - } - else if (fileName.isDirectory()) { - System.out.println("Dir: " + fileName.getName()); - File subDir = new File(fileName.getName()); - File[] moreFiles = subDir.listFiles(); - - if (moreFiles != null) { - for (File subFileName : moreFiles) { - if (subFileName.isFile()) { - System.out.println(" File: " + subFileName.getName()); - } - else if (subFileName.isDirectory()) { - System.out.println(" Dir: " + subFileName.getName()); - } - } - } - } - } - } - } - - private void resetSecurityManager(SecurityManager initialSecurityManager) { - try { - SecurityManager prev = System.getSecurityManager(); - System.setSecurityManager(initialSecurityManager); - System.out - .println("Successfully reset security manager from " + prev + " to " + System.getSecurityManager()); - } - catch (Throwable t) { - System.err.println("Failed to reset security manager: " + t.getMessage()); - t.printStackTrace(System.err); - } - } - - /** - * Print arguments to standard output stream. Mask out argument values to option with name 'password' in them. - * @param banner source banner - * @param args arguments to be printed - */ - public static void printArgs(String banner, String[] args) { - System.out.println(banner); - boolean maskNextArg = false; - for (String arg : args) { - if (arg == null) { - arg = "null"; // prevent NPE in pwd masking - } - - if (maskNextArg) { - System.out.println(" " + "********"); - maskNextArg = false; - } - else { - System.out.println(" " + arg); - if (arg.toLowerCase().contains("password")) { - maskNextArg = true; - } - } - } - } -} - -class LauncherSecurityManager extends SecurityManager { - private static boolean exitInvoked; - private static int exitCode; - private SecurityManager securityManager; - - public LauncherSecurityManager() { - reset(); - securityManager = System.getSecurityManager(); - System.setSecurityManager(this); - } - - @Override - public void checkPermission(Permission perm, Object context) { - if (securityManager != null) { - // check everything with the original SecurityManager - securityManager.checkPermission(perm, context); - } - } - - @Override - public void checkPermission(Permission perm) { - if (securityManager != null) { - // check everything with the original SecurityManager - securityManager.checkPermission(perm); - } - } - - @Override - public void checkExit(int status) throws SecurityException { - exitInvoked = true; - exitCode = status; - throw new SecurityException("Intercepted System.exit(" + status + ")"); - } - - public static boolean getExitInvoked() { - return exitInvoked; - } - - public static int getExitCode() { - return exitCode; - } - - public static void reset() { - exitInvoked = false; - exitCode = 0; - } -} - http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java index e0974e8..77aaa62 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java @@ -66,7 +66,7 @@ public class MapReduceMain extends LauncherMain { logMasking("Map-Reduce job configuration:", new ArrayList<String>(), maskedJobConf); - File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID)); + File idFile = new File(System.getProperty(LauncherAMUtils.ACTION_PREFIX + LauncherAMUtils.ACTION_DATA_NEW_ID)); System.out.println("Submitting Oozie action Map-Reduce job"); System.out.println(); // submitting job @@ -143,7 +143,7 @@ public class MapReduceMain extends LauncherMain { * @throws OozieActionConfiguratorException */ private static void runConfigClass(JobConf actionConf) throws OozieActionConfiguratorException { - String configClass = actionConf.get(LauncherMapper.OOZIE_ACTION_CONFIG_CLASS); + String configClass = actionConf.get(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS); if (configClass != null) { try { Class<?> klass = Class.forName(configClass); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java deleted file mode 100644 index 3895538..0000000 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.oozie.action.hadoop; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.ObjectWritable; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; -/** - * Dummy input format implementation of Oozie launcher jobs. It returns only one record. - */ -public class OozieLauncherInputFormat implements InputFormat<Object, Object> { - - boolean isReadingDone = false; - - public RecordReader<Object, Object> getRecordReader(InputSplit arg0, JobConf arg1, Reporter arg2) - throws IOException { - return new RecordReader<Object, Object>() { - - @Override - public void close() throws IOException { - } - - @Override - public float getProgress() throws IOException { - if (isReadingDone) { - return 1.0f; - } - else - return 0.0f; - } - - @Override - public Object createKey() { - return new ObjectWritable(); - } - - @Override - public Object createValue() { - return new ObjectWritable(); - } - - @Override - public long getPos() throws IOException { - if (isReadingDone) { - return 1; - } - else { - return 0; - } - } - - @Override - public boolean next(Object arg0, Object arg1) throws IOException { - if (isReadingDone) { - return false; - } - else { - isReadingDone = true; - return true; - } - } - - }; - } - - @Override - public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException { - return new InputSplit[] { new EmptySplit() }; - } - - /** - * Empty Split implementation. - */ - public static class EmptySplit implements InputSplit { - - @Override - public void write(DataOutput out) throws IOException { - } - - @Override - public void readFields(DataInput in) throws IOException { - } - - @Override - public long getLength() { - return 0L; - } - - @Override - public String[] getLocations() { - return new String[0]; - } - } - -} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java index f13e8a0..27da385 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java @@ -29,13 +29,13 @@ import org.apache.hadoop.mapred.TaskAttemptContext; public class OozieLauncherOutputCommitter extends OutputCommitter { public OozieLauncherOutputCommitter() { - File propConf = new File(LauncherMapper.PROPAGATION_CONF_XML); + File propConf = new File(LauncherAMUtils.PROPAGATION_CONF_XML); if (!propConf.exists()) { try { propConf.createNewFile(); } catch (IOException e) { - System.out.println("Failed to create " + LauncherMapper.PROPAGATION_CONF_XML); + System.out.println("Failed to create " + LauncherAMUtils.PROPAGATION_CONF_XML); e.printStackTrace(System.err); } } @@ -76,4 +76,4 @@ public class OozieLauncherOutputCommitter extends OutputCommitter { public boolean isRecoverySupported(JobContext jobContext) throws IOException { return isRecoverySupported(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java deleted file mode 100644 index 9e18dd9..0000000 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputFormat; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.util.Progressable; - -public class OozieLauncherOutputFormat implements OutputFormat<Object, Object>{ - - @Override - public RecordWriter<Object, Object> getRecordWriter(FileSystem ignored, JobConf job, String name, - Progressable progress) throws IOException { - return new RecordWriter<Object, Object>() { - @Override - public void write(Object key, Object value) throws IOException { - } - @Override - public void close(Reporter reporter) throws IOException { - } - }; - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java index 0ee35e8..bde7f1d 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java @@ -52,7 +52,7 @@ public class ShellMain extends LauncherMain { private static String LOG4J_PROPERTIES = "log4j.properties"; /** - * @param args Invoked from LauncherMapper:map() + * @param args Invoked from LauncherAMUtils:map() * @throws Exception */ public static void main(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java index 9cdedb7..37af3dd 100644 --- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java @@ -137,7 +137,7 @@ public class TestLauncherAM { @Before public void setup() throws Exception { configureMocksForHappyPath(); - launcherJobConfig.set(LauncherMapper.OOZIE_ACTION_RECOVERY_ID, "1"); + launcherJobConfig.set(LauncherAMUtils.OOZIE_ACTION_RECOVERY_ID, "1"); instantiateLauncher(); } @@ -399,7 +399,7 @@ public class TestLauncherAM { @Test public void testRecoveryIdNotSet() throws Exception { - launcherJobConfig.unset(LauncherMapper.OOZIE_ACTION_RECOVERY_ID); + launcherJobConfig.unset(LauncherAMUtils.OOZIE_ACTION_RECOVERY_ID); instantiateLauncher(); executeLauncher(); @@ -638,4 +638,4 @@ public class TestLauncherAM { return this; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java new file mode 100644 index 0000000..43c4914 --- /dev/null +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_COUNT; +import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX; +import static org.junit.Assert.assertTrue; +import static org.mockito.BDDMockito.given; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.anyBoolean; + +import java.io.File; +import java.io.FileWriter; +import java.io.Writer; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RunningJob; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import com.google.common.collect.Lists; + +@RunWith(MockitoJUnitRunner.class) +public class TestLauncherAMUtils { + @Mock + private Configuration conf; // we have to use mock, because conf.set(null) throws exception + + @Test + public void testArgsHandlingWithoutNullsAndNullsNotAllowed() { + setupConf(Lists.newArrayList("a", "b", "c")); + setEnableNullArgsAllowed(false); + + String args[] = LauncherAMUtils.getMainArguments(conf); + + assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args)); + } + + @Test + public void testHandlingWhenArgsContainNullsAndNullsNotAllowed() { + setupConf(Lists.newArrayList("a", null, "b", null, "c")); + setEnableNullArgsAllowed(false); + + String args[] = LauncherAMUtils.getMainArguments(conf); + + assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args)); + } + + @Test + public void testArgsHandlingWhenArgsContainsNullsOnlyAndNullsNotAllowed() { + setupConf(Lists.<String>newArrayList(null, null, null)); + setEnableNullArgsAllowed(false); + + String args[] = LauncherAMUtils.getMainArguments(conf); + + assertTrue(Arrays.equals(new String[] {}, args)); + } + + @Test + public void testArgsHandlingWhenArgsContainsOneNullAndNullsNotAllowed() { + setupConf(Lists.<String>newArrayList((String) null)); + setEnableNullArgsAllowed(false); + + String args[] = LauncherAMUtils.getMainArguments(conf); + + assertTrue(Arrays.equals(new String[] {}, args)); + } + + @Test + public void testHandlingWhenArgsContainNullsAndNullAllowed() { + setupConf(Lists.newArrayList("a", null, "b", null, "c")); + setEnableNullArgsAllowed(true); + + String args[] = LauncherAMUtils.getMainArguments(conf); + + assertTrue(Arrays.equals(new String[] { "a", null, "b", null, "c"}, args)); + } + + @Test + public void testArgsHandlingWhenArgsContainsOneNullAndNullsAllowed() { + setupConf(Lists.<String>newArrayList((String) null)); + setEnableNullArgsAllowed(true); + + String args[] = LauncherAMUtils.getMainArguments(conf); + + assertTrue(Arrays.equals(new String[] { null }, args)); + } + + private void setupConf(List<String> argList) { + int argCount = argList.size(); + + given(conf.getInt(eq(CONF_OOZIE_ACTION_MAIN_ARG_COUNT), eq(0))).willReturn(argCount); + + for (int i = 0; i < argCount; i++) { + given(conf.get(eq(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i))).willReturn(argList.get(i)); + } + } + + private void setEnableNullArgsAllowed(boolean nullArgsAllowed) { + given(conf.getBoolean(eq(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED), anyBoolean())).willReturn(nullArgsAllowed); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java deleted file mode 100644 index 51b1d6f..0000000 --- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_COUNT; -import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX; -import static org.junit.Assert.assertTrue; -import static org.mockito.BDDMockito.given; -import static org.mockito.Matchers.eq; -import static org.mockito.Matchers.anyBoolean; - -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -import com.google.common.collect.Lists; - -@RunWith(MockitoJUnitRunner.class) -public class TestLauncherMapper { - @Mock - private Configuration conf; // we have to use mock, because conf.set(null) throws exception - - @Test - public void testArgsHandlingWithoutNullsAndNullsNotAllowed() { - setupConf(Lists.newArrayList("a", "b", "c")); - setEnableNullArgsAllowed(false); - - String args[] = LauncherMapper.getMainArguments(conf); - - assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args)); - } - - @Test - public void testHandlingWhenArgsContainNullsAndNullsNotAllowed() { - setupConf(Lists.newArrayList("a", null, "b", null, "c")); - setEnableNullArgsAllowed(false); - - String args[] = LauncherMapper.getMainArguments(conf); - - assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args)); - } - - @Test - public void testArgsHandlingWhenArgsContainsNullsOnlyAndNullsNotAllowed() { - setupConf(Lists.<String>newArrayList(null, null, null)); - setEnableNullArgsAllowed(false); - - String args[] = LauncherMapper.getMainArguments(conf); - - assertTrue(Arrays.equals(new String[] {}, args)); - } - - @Test - public void testArgsHandlingWhenArgsContainsOneNullAndNullsNotAllowed() { - setupConf(Lists.<String>newArrayList((String) null)); - setEnableNullArgsAllowed(false); - - String args[] = LauncherMapper.getMainArguments(conf); - - assertTrue(Arrays.equals(new String[] {}, args)); - } - - @Test - public void testHandlingWhenArgsContainNullsAndNullAllowed() { - setupConf(Lists.newArrayList("a", null, "b", null, "c")); - setEnableNullArgsAllowed(true); - - String args[] = LauncherMapper.getMainArguments(conf); - - assertTrue(Arrays.equals(new String[] { "a", null, "b", null, "c"}, args)); - } - - @Test - public void testArgsHandlingWhenArgsContainsOneNullAndNullsAllowed() { - setupConf(Lists.<String>newArrayList((String) null)); - setEnableNullArgsAllowed(true); - - String args[] = LauncherMapper.getMainArguments(conf); - - assertTrue(Arrays.equals(new String[] { null }, args)); - } - - private void setupConf(List<String> argList) { - int argCount = argList.size(); - - given(conf.getInt(eq(CONF_OOZIE_ACTION_MAIN_ARG_COUNT), eq(0))).willReturn(argCount); - - for (int i = 0; i < argCount; i++) { - given(conf.get(eq(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i))).willReturn(argList.get(i)); - } - } - - private void setEnableNullArgsAllowed(boolean nullArgsAllowed) { - given(conf.getBoolean(eq(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED), anyBoolean())).willReturn(nullArgsAllowed); - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index b6b30eb..4882981 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -53,7 +53,7 @@ import java.util.regex.Pattern; public class PigMain extends LauncherMain { private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<>(); public static final int STRING_BUFFER_SIZE = 100; - public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript"; + public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherAMUtils.ACTION_PREFIX + "pig.log.expandedscript"; @VisibleForTesting static final Pattern[] PIG_JOB_IDS_PATTERNS = { @@ -162,7 +162,7 @@ public class PigMain extends LauncherMain { String logFile = new File("pig-oozie-" + hadoopJobId + ".log").getAbsolutePath(); String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO"); - String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + String rootLogLevel = actionConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO"); // append required PIG properties to the default hadoop log4j file log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A, B"); @@ -378,29 +378,12 @@ public class PigMain extends LauncherMain { } } else { - try { System.out.println("Run pig script using Main.main() for Pig version before 0.8"); Main.main(args); - } - catch (SecurityException ex) { - if (resetSecurityManager) { - LauncherSecurityManager.reset(); - } - else { - if (LauncherSecurityManager.getExitInvoked()) { - if (LauncherSecurityManager.getExitCode() != 0) { - if (pigLog != null) { - handleError(pigLog); - } - throw ex; - } - } - } - } } } - // write external data(stats, hadoopIds) to the file which will be read by the LauncherMapper + // write external data(stats, hadoopIds) to the file which will be read by the LauncherAMUtils private static void writeExternalData(String data, File f) throws IOException { BufferedWriter out = null; try { http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java index df31655..7884cdd 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java @@ -153,7 +153,7 @@ public class PigMainWithOldAPI extends LauncherMain { String logFile = new File("pig-oozie-" + hadoopJobId + ".log").getAbsolutePath(); String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO"); - String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + String rootLogLevel = actionConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO"); // append required PIG properties to the default hadoop log4j file log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A, B"); @@ -200,22 +200,24 @@ public class PigMainWithOldAPI extends LauncherMain { System.out.println(">>> Invoking Pig command line now >>>"); System.out.println(); System.out.flush(); - + LauncherAM.LauncherSecurityManager launcherSecurityManager = new LauncherAM.LauncherSecurityManager(); + launcherSecurityManager.enable(); try { System.out.println(); runPigJob(new String[] { "-version" }); } catch (SecurityException ex) { - LauncherSecurityManager.reset(); + launcherSecurityManager.reset(); } System.out.println(); System.out.flush(); + try { runPigJob(arguments.toArray(new String[arguments.size()])); } catch (SecurityException ex) { - if (LauncherSecurityManager.getExitInvoked()) { - if (LauncherSecurityManager.getExitCode() != 0) { + if (launcherSecurityManager.getExitInvoked()) { + if (launcherSecurityManager.getExitCode() != 0) { System.err.println(); System.err.println("Pig logfile dump:"); System.err.println(); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java index 7c3c5bb..75727d2 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java @@ -225,7 +225,7 @@ public class TestPigActionExecutor extends ActionExecutorTestCase { Context context = createContext(actionXml); submitAction(context); FileSystem fs = getFileSystem(); - FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML)); + FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherAMUtils.ACTION_CONF_XML)); XConfiguration conf = new XConfiguration(); conf.addResource(os); assertNull(conf.get("oozie.HadoopAccessorService.created")); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java index 22e7952..9a185c9 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java @@ -115,7 +115,6 @@ public class TestPigMain extends PigTestCase { assertEquals(props.getProperty("oozie.pig.args.size"), "1"); File pigProps = new File(classPathDir, "pig.properties"); - new LauncherSecurityManager(); String user = System.getProperty("user.name"); try { Writer wr = new FileWriter(pigProps); @@ -123,20 +122,6 @@ public class TestPigMain extends PigTestCase { wr.close(); PigMain.main(null); } - catch (SecurityException ex) { - if (LauncherSecurityManager.getExitInvoked()) { - System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")"); - System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")"); - if (failOnException) { - if (LauncherSecurityManager.getExitCode() != 0) { - fail(); - } - } - } - else { - throw ex; - } - } finally { pigProps.delete(); System.setProperty("user.name", user); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java index 74de433..d5a5797 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java @@ -36,9 +36,12 @@ import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.Writer; import java.net.URL; +import java.security.Permission; import java.util.Properties; import java.util.concurrent.Callable; +import static groovy.util.GroovyTestCase.assertEquals; + /** * Test PigMainWithOldAPI class should run a Pig script and write results to @@ -118,29 +121,18 @@ public class TestPigMainWithOldAPI extends XFsTestCase implements Callable<Void> assertEquals(props.getProperty("oozie.pig.args.size"), "1"); File pigProps = new File(classPathDir, "pig.properties"); - new LauncherSecurityManager(); String user = System.getProperty("user.name"); ByteArrayOutputStream data = new ByteArrayOutputStream(); PrintStream oldPrintStream = System.out; System.setOut(new PrintStream(data)); + + try { Writer wr = new FileWriter(pigProps); props.store(wr, ""); wr.close(); PigMainWithOldAPI.main(null); } - catch (SecurityException ex) { - if (LauncherSecurityManager.getExitInvoked()) { - System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")"); - System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")"); - if (LauncherSecurityManager.getExitCode() != 0) { - fail("Exit code should be 0"); - } - } - else { - throw ex; - } - } finally { pigProps.delete(); System.setProperty("user.name", user); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 674839a..8aa8343 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -191,7 +191,7 @@ public class SparkMain extends LauncherMain { final String logFile = new File("spark-oozie-" + hadoopJobId + ".log").getAbsolutePath(); final String logLevel = actionConf.get("oozie.spark.log.level", "INFO"); - final String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + final String rootLogLevel = actionConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO"); log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A"); log4jProperties.setProperty("log4j.logger.org.apache.spark", logLevel + ", A, jobid"); @@ -272,4 +272,4 @@ public class SparkMain extends LauncherMain { // to null. HiveConf.setHiveSiteLocation(HiveConf.class.getClassLoader().getResource("hive-site.xml")); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index 416f1ec..e889ef9 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -22,9 +22,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.net.URL; import java.util.Map; -import java.util.Properties; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; @@ -116,7 +114,7 @@ public class SqoopMain extends LauncherMain { String logFile = new File("sqoop-oozie-" + hadoopJobId + ".log").getAbsolutePath(); String logLevel = sqoopConf.get("oozie.sqoop.log.level", "INFO"); - String rootLogLevel = sqoopConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + String rootLogLevel = sqoopConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO"); log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A"); log4jProperties.setProperty("log4j.logger.org.apache.sqoop", logLevel + ", A"); http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java index 045f174..3b38042 100644 --- a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java +++ b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java @@ -616,7 +616,7 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase { final Map<String, String> actionData = LauncherHelper.getActionData(fs, context.getActionDir(), context.getProtoActionConf()); - Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS)); + Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherAMUtils.ACTION_DATA_ERROR_PROPS)); assertEquals("An Exception occurred while instantiating the action config class", errorProps.getProperty("exception.message")); assertTrue(errorProps.getProperty("exception.stacktrace").startsWith(OozieActionConfiguratorException.class.getName())); @@ -646,7 +646,7 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase { final Map<String, String> actionData = LauncherHelper.getActionData(fs, context.getActionDir(), context.getProtoActionConf()); - Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS)); + Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherAMUtils.ACTION_DATA_ERROR_PROPS)); assertEquals("doh", errorProps.getProperty("exception.message")); assertTrue(errorProps.getProperty("exception.stacktrace").startsWith(OozieActionConfiguratorException.class.getName())); } http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java ---------------------------------------------------------------------- diff --git a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java index a57580e..ef268b2 100644 --- a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java +++ b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java @@ -74,7 +74,7 @@ public class TestStreamingMain extends MainTestCase { StreamingMain.main(null); assertTrue(newId.exists()); - assertNotNull(LauncherMapper.getLocalFileContentStr(newId, "", -1)); + assertNotNull(LauncherAMUtils.getLocalFileContentStr(newId, "", -1)); return null; }
