Repository: oozie Updated Branches: refs/heads/oya 0e9e5e380 -> 5dd08b159
review fixes Change-Id: I262eafa0dc5b83875dc136cd555ea162fe320d54 Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5dd08b15 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5dd08b15 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5dd08b15 Branch: refs/heads/oya Commit: 5dd08b1596efb0293409e6bb4f308aa99c25de21 Parents: 0e9e5e3 Author: Gezapeti Cseh <gezap...@gmail.com> Authored: Wed May 17 09:40:15 2017 +0200 Committer: Gezapeti Cseh <gezap...@gmail.com> Committed: Wed May 17 09:41:24 2017 +0200 ---------------------------------------------------------------------- .../action/hadoop/CredentialsProvider.java | 2 +- .../hadoop/CredentialsProviderFactory.java | 3 ++- .../oozie/action/hadoop/FsELFunctions.java | 1 - .../action/hadoop/HCatCredentialHelper.java | 19 ++++++++-------- .../oozie/action/hadoop/HbaseCredentials.java | 2 -- .../oozie/action/hadoop/JavaActionExecutor.java | 23 +++++++++++--------- .../oozie/action/hadoop/LauncherHelper.java | 1 - .../action/hadoop/MapReduceActionExecutor.java | 1 - .../action/hadoop/SparkActionExecutor.java | 12 ---------- .../oozie/command/wf/SubmitMRXCommand.java | 1 - .../oozie/service/HadoopAccessorService.java | 17 +++------------ .../java/org/apache/oozie/service/Services.java | 1 - .../apache/oozie/servlet/CallbackServlet.java | 1 - .../org/apache/oozie/util/ClasspathUtils.java | 3 --- .../org/apache/oozie/util/Instrumentation.java | 2 -- core/src/main/resources/oozie-default.xml | 8 ------- .../oozie/action/hadoop/TestJavaMain.java | 2 +- .../java/org/apache/oozie/test/XTestCase.java | 4 ++-- docs/src/site/twiki/ENG_Building.twiki | 4 ++-- .../oozie/action/hadoop/HdfsOperations.java | 3 +-- .../apache/oozie/action/hadoop/JavaMain.java | 9 ++++++++ .../apache/oozie/action/hadoop/LauncherAM.java | 5 +++-- .../oozie/action/hadoop/LauncherMain.java | 8 ++----- .../oozie/action/hadoop/LauncherMapper.java | 11 +--------- .../oozie/action/hadoop/MapReduceMain.java | 10 --------- .../action/hadoop/LauncherAMTestMainClass.java | 2 +- 26 files changed, 51 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java index 1801b2d..d49da90 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java @@ -26,7 +26,7 @@ public interface CredentialsProvider { /** * This is the interface for all the Credentials implementation. Any new credential implementation must implement - * this function. This function should modify the jobconf which will be used further to pass the credentials to the + * this function. This function should modify the configuration which will be used further to pass the credentials to the * tasks while running it. Credentials properties and context is also provided by that user can get all the * necessary configuration. * http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java index ddffc79..eb9f79a 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java @@ -21,6 +21,7 @@ package org.apache.oozie.action.hadoop; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.security.UserGroupInformation; @@ -46,7 +47,7 @@ public class CredentialsProviderFactory { } private CredentialsProviderFactory() throws Exception { - providerCache = new HashMap<>(); + providerCache = new ConcurrentHashMap<>(); for (String function : ConfigurationService.getStrings(CRED_KEY)) { function = trim(function); LOG.debug("Creating Credential class for : " + function); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java index 210747a..c16f560 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java @@ -186,7 +186,6 @@ public class FsELFunctions { * ReachingGlobMaxException thrown when globbed file count exceeds the limit */ static class ReachingGlobMaxException extends RuntimeException { - private static final long serialVersionUID = -3569871817672303526L; } } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java index 4d9d3fa..01bd989 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java @@ -33,7 +33,7 @@ import org.apache.oozie.util.XLog; /** * Helper class to handle the HCat credentials * Performs internally the heavy-lifting of fetching delegation tokens from Hive Metastore, abstracted from the user - * Token is added to jobConf + * Token is added to the launcher configuration */ public class HCatCredentialHelper { @@ -45,8 +45,9 @@ public class HCatCredentialHelper { private static final String HADOOP_RPC_PROTECTION = "hadoop.rpc.protection"; /** - * This Function will set the HCat token to jobconf - * @param launcherJobConf - job conf + * This Function will set the HCat token to configuration + * @param credentials - the credentials + * @param launcherConfig - launcher configuration * @param principal - principal for HCat server * @param server - Serevr URI for HCat server * @throws Exception @@ -56,17 +57,17 @@ public class HCatCredentialHelper { try { client = getHCatClient(launcherConfig, principal, server); XLog.getLog(getClass()).debug( - "HCatCredentialHelper: set: User name for which token will be asked from HCat: " - + launcherConfig.get(USER_NAME)); + "HCatCredentialHelper: set: User name for which token will be asked from HCat: {0}", + launcherConfig.get(USER_NAME)); String tokenStrForm = client.getDelegationToken(launcherConfig.get(USER_NAME), UserGroupInformation .getLoginUser().getShortUserName()); Token<DelegationTokenIdentifier> hcatToken = new Token<DelegationTokenIdentifier>(); hcatToken.decodeFromUrlString(tokenStrForm); credentials.addToken(new Text("HCat Token"), hcatToken); - XLog.getLog(getClass()).debug("Added the HCat token in job conf"); + XLog.getLog(getClass()).debug("Added the HCat token to launcher configuration"); } catch (Exception ex) { - XLog.getLog(getClass()).debug("set Exception" + ex.getMessage()); + XLog.getLog(getClass()).debug("set Exception {0}", ex.getMessage()); throw ex; } finally { @@ -89,7 +90,7 @@ public class HCatCredentialHelper { HiveConf hiveConf = null; HCatClient hiveclient = null; hiveConf = new HiveConf(); - XLog.getLog(getClass()).debug("getHCatClient: Principal: " + principal + " Server: " + server); + XLog.getLog(getClass()).debug("getHCatClient: Principal: {0} Server: {1}", principal, server); // specified a thrift url hiveConf.set(HIVE_METASTORE_SASL_ENABLED, "true"); @@ -99,7 +100,7 @@ public class HCatCredentialHelper { String protection = launcherConfig.get(HADOOP_RPC_PROTECTION, SaslRpcServer.QualityOfProtection.AUTHENTICATION.name() .toLowerCase()); - XLog.getLog(getClass()).debug("getHCatClient, setting rpc protection to " + protection); + XLog.getLog(getClass()).debug("getHCatClient, setting rpc protection to {0}", protection); hiveConf.set(HADOOP_RPC_PROTECTION, protection); hiveclient = HCatClient.create(hiveConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java index 327ba83..22b6dc9 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java @@ -102,6 +102,4 @@ public class HbaseCredentials implements CredentialsProvider { } } } - - } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 7a762ab..87deb69 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -153,7 +153,6 @@ public class JavaActionExecutor extends ActionExecutor { protected XLog LOG = XLog.getLog(getClass()); private static final String JAVA_TMP_DIR_SETTINGS = "-Djava.io.tmpdir="; - private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator(); public XConfiguration workflowConf = null; @@ -174,9 +173,6 @@ public class JavaActionExecutor extends ActionExecutor { public static List<Class<?>> getCommonLauncherClasses() { List<Class<?>> classes = new ArrayList<Class<?>>(); classes.add(LauncherMain.class); - classes.add(launcherInputFormatClassLocator.locateOrGet()); - classes.add(OozieLauncherOutputFormat.class); - classes.add(OozieLauncherOutputCommitter.class); classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher()); classes.add(LauncherAM.class); classes.add(LauncherAMCallbackNotifier.class); @@ -970,6 +966,7 @@ public class JavaActionExecutor extends ActionExecutor { // if user-retry is on, always submit new launcher boolean isUserRetry = ((WorkflowActionBean)action).isUserRetry(); + LOG.debug("creating yarnClinet for action {0}", action.getId()); yarnClient = createYarnClient(context, launcherJobConf); if (alreadyRunning && !isUserRetry) { @@ -1133,6 +1130,7 @@ public class JavaActionExecutor extends ActionExecutor { // TODO: OYA: make resources allocated for the AM configurable and choose good defaults (memory MB, vcores) Resource resource = Resource.newInstance(2048, 1); appContext.setResource(resource); + appContext.setCancelTokensWhenComplete(true); return appContext; } @@ -1184,8 +1182,8 @@ public class JavaActionExecutor extends ActionExecutor { String credName = entry.getKey(); CredentialsProperties credProps = entry.getValue(); if (credProps != null) { - CredentialsProviderFactory tokenProviderFactory = CredentialsProviderFactory.getInstance(); - CredentialsProvider tokenProvider = tokenProviderFactory.createCredentialsProvider(credProps.getType()); + CredentialsProvider tokenProvider = CredentialsProviderFactory.getInstance() + .createCredentialsProvider(credProps.getType()); if (tokenProvider != null) { tokenProvider.updateCredentials(credentials, jobconf, credProps, context); LOG.debug("Retrieved Credential '" + credName + "' for action " + action.getId()); @@ -1341,8 +1339,9 @@ public class JavaActionExecutor extends ActionExecutor { /** * If returns true, it means that we have to add Hadoop MR jars to the classpath. - * Subclasses should override this method if necessary. - * + * Subclasses should override this method if necessary. By default we don't add + * MR jars to the classpath. + * @return false by default */ protected boolean needToAddMRJars() { return false; @@ -1379,7 +1378,7 @@ public class JavaActionExecutor extends ActionExecutor { } } catch (Exception ye) { - LOG.info("Exception occurred while checking Launcher AM status; will try checking action data file instead ", ye); + LOG.warn("Exception occurred while checking Launcher AM status; will try checking action data file instead ", ye); // Fallback to action data file if we can't find the Launcher AM (maybe it got purged) fallback = true; } @@ -1511,7 +1510,11 @@ public class JavaActionExecutor extends ActionExecutor { } for(ApplicationId id : LauncherMain.getChildYarnJobs(jobConf, ApplicationsRequestScope.ALL, action.getStartTime().getTime())){ - yarnClient.killApplication(id); + try { + yarnClient.killApplication(id); + } catch (Exception e) { + LOG.warn("Could not kill child of {0}, {1}", action.getExternalId(), id); + } } context.setExternalStatus(KILLED); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java index 442f9a3..f80141c 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java @@ -51,7 +51,6 @@ import org.apache.oozie.service.UserGroupInformationService; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.PropertiesUtils; -// TODO: we're no longer using Launcher Mapper -- give this class a better name public class LauncherHelper { public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag"; http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index bbc0e89..2050591 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -133,7 +133,6 @@ public class MapReduceActionExecutor extends JavaActionExecutor { @Override protected Configuration createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) { Configuration conf = super.createBaseHadoopConf(context, actionXml, loadResources); - injectConfigClass(conf, actionXml); return conf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index b237702..2161af0 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -36,7 +36,6 @@ import java.util.Properties; public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SparkMain"; - public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence"; public static final String SPARK_MASTER = "oozie.spark.master"; public static final String SPARK_MODE = "oozie.spark.mode"; public static final String SPARK_OPTS = "oozie.spark.spark-opts"; @@ -101,17 +100,6 @@ public class SparkActionExecutor extends JavaActionExecutor { } @Override - Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, - Configuration actionConf) throws ActionExecutorException { - - Configuration launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf); - if (launcherJobConf.get("oozie.launcher." + TASK_USER_PRECEDENCE) == null) { - launcherJobConf.set(TASK_USER_PRECEDENCE, "true"); - } - return launcherJobConf; - } - - @Override Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException { super.setupLauncherConf(conf, actionXml, appPath, context); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java index 79f0beb..05e7595 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java @@ -44,7 +44,6 @@ public class SubmitMRXCommand extends SubmitHttpXCommand { SKIPPED_CONFS.add(XOozieClient.RM); SKIPPED_CONFS.add(XOozieClient.NN); - //DEPRECATE_MAP.put(XOozieClient.RM, "yarn.resourcemanager.address"); DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2); DEPRECATE_MAP.put(WorkflowAppService.HADOOP_USER, "mapreduce.job.user.name"); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 45f91a3..9624104 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -25,11 +25,9 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -47,7 +45,6 @@ import org.apache.oozie.workflow.lite.LiteWorkflowAppParser; import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; @@ -523,10 +520,7 @@ public class HadoopAccessorService implements Service { }); return jobClient; } - catch (InterruptedException ex) { - throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex); - } - catch (IOException ex) { + catch (IOException | InterruptedException ex) { throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex); } } @@ -571,9 +565,7 @@ public class HadoopAccessorService implements Service { } }); return yarnClient; - } catch (InterruptedException ex) { - throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex); - } catch (IOException ex) { + } catch (IOException | InterruptedException ex) { throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex); } } @@ -624,10 +616,7 @@ public class HadoopAccessorService implements Service { } }); } - catch (InterruptedException ex) { - throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex); - } - catch (IOException ex) { + catch (IOException | InterruptedException ex) { throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/service/Services.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/Services.java b/core/src/main/java/org/apache/oozie/service/Services.java index 4e1775e..7f47f88 100644 --- a/core/src/main/java/org/apache/oozie/service/Services.java +++ b/core/src/main/java/org/apache/oozie/service/Services.java @@ -304,7 +304,6 @@ public class Services { setService(entry.getValue().getClass()); } } catch (RuntimeException rex) { - rex.printStackTrace(); log.fatal("Runtime Exception during Services Load. Check your list of '{0}' or '{1}'", CONF_SERVICE_CLASSES, CONF_SERVICE_EXT_CLASSES, rex); throw new ServiceException(ErrorCode.E0103, rex.getMessage(), rex); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java b/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java index 86ba467..7d92ffc 100644 --- a/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java @@ -40,7 +40,6 @@ import org.apache.oozie.util.PropertiesUtils; import org.apache.oozie.util.XLog; public class CallbackServlet extends JsonRestServlet { - private static final long serialVersionUID = 6439106936153152786L; private static final String INSTRUMENTATION_NAME = "callback"; http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java index 8533371..5833607 100644 --- a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java +++ b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java @@ -41,9 +41,6 @@ public class ClasspathUtils { private static boolean usingMiniYarnCluster = false; private static final List<String> CLASSPATH_ENTRIES = Arrays.asList( ApplicationConstants.Environment.PWD.$(), - MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR, - MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR, - MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*", ApplicationConstants.Environment.PWD.$() + Path.SEPARATOR + "*" ); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/util/Instrumentation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/Instrumentation.java b/core/src/main/java/org/apache/oozie/util/Instrumentation.java index 87d2eae..45219a9 100644 --- a/core/src/main/java/org/apache/oozie/util/Instrumentation.java +++ b/core/src/main/java/org/apache/oozie/util/Instrumentation.java @@ -32,7 +32,6 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; @@ -696,7 +695,6 @@ public class Instrumentation { public String toString() { String E = System.getProperty("line.separator"); StringBuilder sb = new StringBuilder(4096); - for (String element : all.keySet()) { sb.append(element).append(':').append(E); List<String> groups = new ArrayList<String>(all.get(element).keySet()); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 61b1d88..4ab7512 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1835,14 +1835,6 @@ will be the requeue interval for the actions which are waiting for a long time w </property> <property> - <name>oozie.action.launcher.mapreduce.input.format.class</name> - <value>org.apache.oozie.action.hadoop.OozieLauncherInputFormat</value> - <description> - Make the Launcher Mapper map-only job's InputFormat class pluggable in order to provide alternative implementations. - </description> - </property> - - <property> <name>oozie.action.spark.setup.hadoop.conf.dir</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java index bb2df43..491e745 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java @@ -53,7 +53,7 @@ public class TestJavaMain extends MainTestCase { // Check Exception handling try { JavaMain.main(new String[]{"ex2"}); - } catch(JavaMainException jme) { + } catch(JavaMain.JavaMainException jme) { assertTrue(jme.getCause() instanceof IOException); assertEquals("throwing exception", jme.getCause().getMessage()); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 7ffabd4..72d646d 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -1235,8 +1235,8 @@ public abstract class XTestCase extends TestCase { final ApplicationId appId = ConverterUtils.toApplicationId(externalId); final MutableObject<YarnApplicationState> finalState = new MutableObject<YarnApplicationState>(); - JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(getJobTrackerUri()); - final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), jobConf); + Configuration conf = Services.get().get(HadoopAccessorService.class).getCachedConf(); + final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), conf); try { waitFor(60 * 1000, new Predicate() { http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/docs/src/site/twiki/ENG_Building.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki index 3d23b26..36d77b2 100644 --- a/docs/src/site/twiki/ENG_Building.twiki +++ b/docs/src/site/twiki/ENG_Building.twiki @@ -112,9 +112,9 @@ Except for the options marked with =(*)=, the options can be specified in the =t of the Oozie project. The options marked with =(*)= are used in Maven POMs, thus they don't take effect if specified in the =test.properties= file (which is loaded by the =XTestCase= class at class initialization time). -*hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-2) you wish to build Oozie against specifically. It will +*hadoop.version* =(*)=: indicates the Hadoop version you wish to build Oozie against specifically. It will substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven. -For Hadoop 2.x, the default (and minimum) version is 2.6.0. +The default version is 2.6.0 and that is the minimum supported Hadoop version. *generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated) http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java index 8e7b258..874d371 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java @@ -57,8 +57,7 @@ public class HdfsOperations { public Void run() throws Exception { Path finalPath = new Path(actionDir, LauncherAM.ACTION_DATA_SEQUENCE_FILE); // upload into sequence file - System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: " - + new Path(actionDir, LauncherAM.ACTION_DATA_SEQUENCE_FILE).toUri()); + System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: " + finalPath.toUri()); try (SequenceFile.Writer wr = seqFileWriterFactory.createSequenceFileWriter(launcherJobConf, finalPath, Text.class, Text.class)) { http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java index 3b4e0a6..c3e3d3f 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@ -60,4 +60,13 @@ public class JavaMain extends LauncherMain { } + /** + * Used by JavaMain to wrap a Throwable when an Exception occurs + */ + @SuppressWarnings("serial") + static class JavaMainException extends Exception { + public JavaMainException(Throwable t) { + super(t); + } + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 2b55997..9484804 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -53,6 +53,7 @@ public class LauncherAM { public static final String OOZIE_ACTION_ID = "oozie.action.id"; public static final String OOZIE_JOB_ID = "oozie.job.id"; public static final String ACTION_PREFIX = "oozie.action."; + static final String OOZIE_ACTION_RECOVERY_ID = ACTION_PREFIX + "recovery.id"; public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = ACTION_PREFIX + "max.output.data"; public static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = ACTION_PREFIX + "main.arg."; public static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + "count"; @@ -375,7 +376,7 @@ public class LauncherAM { // Get what actually caused the exception Throwable cause = ex.getCause(); // If we got a JavaMainException from JavaMain, then we need to unwrap it - if (JavaMainException.class.isInstance(cause)) { + if (JavaMain.JavaMainException.class.isInstance(cause)) { cause = cause.getCause(); } if (LauncherMainException.class.isInstance(cause)) { @@ -429,7 +430,7 @@ public class LauncherAM { ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId(); String applicationIdStr = applicationId.toString(); - String recoveryId = Preconditions.checkNotNull(launcherJobConf.get(LauncherMapper.OOZIE_ACTION_RECOVERY_ID), + String recoveryId = Preconditions.checkNotNull(launcherJobConf.get(OOZIE_ACTION_RECOVERY_ID), "RecoveryID should not be null"); Path path = new Path(actionDir, recoveryId); http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 0f14ec0..0236e1b 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -219,10 +219,8 @@ public abstract class LauncherMain { for(ApplicationReport appReport : appsList) { childYarnJobs.add(appReport.getApplicationId()); } - } catch (IOException ioe) { + } catch (YarnException | IOException ioe) { throw new RuntimeException("Exception occurred while finding child jobs", ioe); - } catch (YarnException ye) { - throw new RuntimeException("Exception occurred while finding child jobs", ye); } System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ",")); @@ -257,10 +255,8 @@ public abstract class LauncherMain { } System.out.println(); } - } catch (YarnException ye) { + } catch (IOException | YarnException ye) { throw new RuntimeException("Exception occurred while killing child job(s)", ye); - } catch (IOException ioe) { - throw new RuntimeException("Exception occurred while killing child job(s)", ioe); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java index eda86f4..412807f 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java @@ -240,7 +240,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R // Get what actually caused the exception Throwable cause = ex.getCause(); // If we got a JavaMainException from JavaMain, then we need to unwrap it - if (JavaMainException.class.isInstance(cause)) { + if (JavaMain.JavaMainException.class.isInstance(cause)) { cause = cause.getCause(); } if (LauncherMainException.class.isInstance(cause)) { @@ -702,12 +702,3 @@ class LauncherSecurityManager extends SecurityManager { } } -/** - * Used by JavaMain to wrap a Throwable when an Exception occurs - */ -@SuppressWarnings("serial") -class JavaMainException extends Exception { - public JavaMainException(Throwable t) { - super(t); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java index 02e3fbb..e0974e8 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java @@ -132,16 +132,6 @@ public class MapReduceMain extends LauncherMain { return runJob; } - @Deprecated - public static void setStrings(Configuration conf, String key, String[] values) { - ActionUtils.setStrings(conf, key, values); - } - - @Deprecated - public static String[] getStrings(Configuration conf, String key) { - return ActionUtils.getStrings(conf, key); - } - protected JobClient createJobClient(JobConf jobConf) throws IOException { return new JobClient(jobConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java index 6deca64..718bf64 100644 --- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java @@ -35,7 +35,7 @@ public class LauncherAMTestMainClass { if (args != null && args.length == 1) { switch (args[0]){ case JAVA_EXCEPTION: - throw new JavaMainException(new RuntimeException(JAVA_EXCEPTION_MESSAGE)); + throw new JavaMain.JavaMainException(new RuntimeException(JAVA_EXCEPTION_MESSAGE)); case LAUNCHER_EXCEPTION: throw new LauncherMainException(LAUNCHER_ERROR_CODE); case SECURITY_EXCEPTION: