oozie git commit: OOZIE-2658 --driver-class-path can overwrite the classpath in SparkMain (gezapeti via rkanter)
Repository: oozie Updated Branches: refs/heads/branch-4.3 77f927690 -> 934ffc72e OOZIE-2658 --driver-class-path can overwrite the classpath in SparkMain (gezapeti via rkanter) (cherry picked from commit e0016c93ad903bdee07fa63b9265382f1c6e3a62) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/934ffc72 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/934ffc72 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/934ffc72 Branch: refs/heads/branch-4.3 Commit: 934ffc72e2ed917dff47dc5d6736267a967630d5 Parents: 77f9276 Author: Robert KanterAuthored: Mon Oct 10 21:02:47 2016 -0700 Committer: Robert Kanter Committed: Mon Oct 10 21:05:04 2016 -0700 -- release-log.txt | 1 + .../apache/oozie/action/hadoop/SparkMain.java | 64 +++- 2 files changed, 37 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/934ffc72/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 5f271a2..aab0dc0 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release +OOZIE-2658 --driver-class-path can overwrite the classpath in SparkMain (gezapeti via rkanter) OOZIE-1814 Oozie should mask any passwords in logs and REST interfaces (andras.piros via rkanter) OOZIE-2622 ExtJS 2.2 is no longer available (rkanter) OOZIE-2606 Set spark.yarn.jars to fix Spark 2.0 with Oozie (satishsaley via rohini) http://git-wip-us.apache.org/repos/asf/oozie/blob/934ffc72/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java -- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 539fb5c..19a39a9 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -49,6 +49,7 @@ public class SparkMain extends LauncherMain { private static final String JOB_NAME_OPTION = "--name"; private static final String CLASS_NAME_OPTION = "--class"; private static final String VERBOSE_OPTION = "--verbose"; +private static final String DRIVER_CLASSPATH_OPTION = "--driver-class-path"; private static final String EXECUTOR_CLASSPATH = "spark.executor.extraClassPath="; private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath="; private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled"; @@ -116,29 +117,31 @@ public class SparkMain extends LauncherMain { if(jarPath!=null && jarPath.endsWith(".py")){ isPyspark = true; } -boolean addedExecutorClasspath = false; -boolean addedDriverClasspath = false; boolean addedHiveSecurityToken = false; boolean addedHBaseSecurityToken = false; +StringBuilder driverClassPath = new StringBuilder(); +StringBuilder executorClassPath = new StringBuilder(); String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS); if (StringUtils.isNotEmpty(sparkOpts)) { List sparkOptions = splitSparkOpts(sparkOpts); for (int i = 0; i < sparkOptions.size(); i++) { String opt = sparkOptions.get(i); +boolean addToSparkArgs = true; if (yarnClusterMode || yarnClientMode) { if (opt.startsWith(EXECUTOR_CLASSPATH)) { -// Include the current working directory (of executor -// container) in executor classpath, because it will contain -// localized files -opt = opt + File.pathSeparator + PWD; -addedExecutorClasspath = true; + appendWithPathSeparator(opt.substring(EXECUTOR_CLASSPATH.length()), executorClassPath); +addToSparkArgs = false; } if (opt.startsWith(DRIVER_CLASSPATH)) { -// Include the current working directory (of driver -// container) in executor classpath, because it will contain -// localized files -opt = opt + File.pathSeparator + PWD; -addedDriverClasspath = true; + appendWithPathSeparator(opt.substring(DRIVER_CLASSPATH.length()), driverClassPath); +addToSparkArgs = false; +} +if
[38/50] [abbrv] oozie git commit: OOZIE-2685 [amend] DEBUG: modify oozie-log4j.properties so that logs become more verbose
OOZIE-2685 [amend] DEBUG: modify oozie-log4j.properties so that logs become more verbose Change-Id: I07f11bb3e13b3fcfd311c376492750aa2a93af8a Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ed725708 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ed725708 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ed725708 Branch: refs/heads/oya Commit: ed725708d05c20f0036ea4c06cb9beacb97957ea Parents: 938dcab Author: Peter BacskoAuthored: Tue Sep 27 10:50:58 2016 +0200 Committer: Peter Bacsko Committed: Tue Sep 27 10:50:58 2016 +0200 -- core/src/main/resources/oozie-log4j.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/ed725708/core/src/main/resources/oozie-log4j.properties -- diff --git a/core/src/main/resources/oozie-log4j.properties b/core/src/main/resources/oozie-log4j.properties index e281986..46b5cfc 100644 --- a/core/src/main/resources/oozie-log4j.properties +++ b/core/src/main/resources/oozie-log4j.properties @@ -28,8 +28,8 @@ log4j.appender.test=org.apache.log4j.ConsoleAppender log4j.appender.test.Target=System.out log4j.appender.test.layout=org.apache.log4j.PatternLayout -log4j.appender.test.layout.ConversionPattern=%d{ABSOLUTE} [%t] %5p %c{1}:%L - %m%n# note the [%t] so we can see the threads too - +log4j.appender.test.layout.ConversionPattern=%d{ABSOLUTE} [%t] %5p %c{1}:%L - %m%n + log4j.appender.none=org.apache.log4j.varia.NullAppender log4j.rootLogger=DEBUG, test
[32/50] [abbrv] oozie git commit: OOZIE-2594 Make MapReduce action work, small refactors, remove RunningJob from test cases, test fixes. Follow up: OOZIE-2686
http://git-wip-us.apache.org/repos/asf/oozie/blob/ca7e56fd/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMCallbackNotifier.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMCallbackNotifier.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMCallbackNotifier.java index dbef441..23648b8 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMCallbackNotifier.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMCallbackNotifier.java @@ -26,6 +26,7 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.Proxy; import java.net.URL; +import java.util.EnumSet; // Adapted from org.apache.hadoop.mapreduce.v2.app.JobEndNotifier /** @@ -34,9 +35,11 @@ import java.net.URL; */ public class LauncherAMCallbackNotifier { private static final String OOZIE_LAUNCHER_CALLBACK = "oozie.launcher.callback."; +private static final int OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL_MAX = 5000; +private static final EnumSet FAILED_APPLICATION_STATES = EnumSet.of(FinalApplicationStatus.KILLED, FinalApplicationStatus.FAILED); + public static final String OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS = OOZIE_LAUNCHER_CALLBACK + "retry.attempts"; public static final String OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL = OOZIE_LAUNCHER_CALLBACK + "retry.interval"; -static final int OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL_MAX = 5000; public static final String OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS = OOZIE_LAUNCHER_CALLBACK + "max.attempts"; public static final String OOZIE_LAUNCHER_CALLBACK_TIMEOUT = OOZIE_LAUNCHER_CALLBACK + "timeout"; public static final String OOZIE_LAUNCHER_CALLBACK_URL = OOZIE_LAUNCHER_CALLBACK + "url"; @@ -51,6 +54,7 @@ public class LauncherAMCallbackNotifier { protected URL urlToNotify; //URL to notify read from the config protected Proxy proxyToUse = Proxy.NO_PROXY; //Proxy to use for notification + /** * Parse the URL that needs to be notified of the end of the job, along * with the number of retries in case of failure, the amount of time to @@ -136,7 +140,7 @@ public class LauncherAMCallbackNotifier { * * @throws InterruptedException */ -public void notifyURL(FinalApplicationStatus finalStatus) throws InterruptedException { +public void notifyURL(FinalApplicationStatus finalStatus, boolean backgroundAction) throws InterruptedException { // Do we need job-end notification? if (userUrl == null) { System.out.println("Callback notification URL not set, skipping."); @@ -145,7 +149,12 @@ public class LauncherAMCallbackNotifier { //Do string replacements for final status if (userUrl.contains(OOZIE_LAUNCHER_CALLBACK_JOBSTATUS_TOKEN)) { -userUrl = userUrl.replace(OOZIE_LAUNCHER_CALLBACK_JOBSTATUS_TOKEN, finalStatus.toString()); +// only send back "RUNNING" if the submission was successful +if (backgroundAction && !FAILED_APPLICATION_STATES.contains(finalStatus)) { +userUrl = userUrl.replace(OOZIE_LAUNCHER_CALLBACK_JOBSTATUS_TOKEN, "RUNNING"); +} else { +userUrl = userUrl.replace(OOZIE_LAUNCHER_CALLBACK_JOBSTATUS_TOKEN, finalStatus.toString()); +} } // Create the URL, ensure sanity http://git-wip-us.apache.org/repos/asf/oozie/blob/ca7e56fd/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index fef6523..6955416 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -201,12 +201,12 @@ public abstract class LauncherMain { * @param conf Configuration/Properties object to dump to STDOUT * @throws IOException thrown if an IO error ocurred. */ -@SuppressWarnings("unchecked") -protected static void logMasking(String header, Collection maskSet, Iterable conf) throws IOException { + +protected static void logMasking(String header, Collection maskSet, Iterable> conf) throws IOException { StringWriter writer = new StringWriter(); writer.write(header + "\n"); writer.write("\n"); -for (Map.Entry entry : (Iterable) conf) { +for (Map.Entry entry : conf) { String name = (String) entry.getKey(); String value = (String) entry.getValue(); for (String mask : maskSet) {
[40/50] [abbrv] oozie git commit: OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini)
OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e8a9b24b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e8a9b24b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e8a9b24b Branch: refs/heads/oya Commit: e8a9b24b9916df20cfbc1c48e5e1a755151d8245 Parents: 76b60cc Author: Rohini PalaniswamyAuthored: Tue Sep 27 12:05:19 2016 -0700 Committer: Rohini Palaniswamy Committed: Tue Sep 27 12:05:19 2016 -0700 -- .../action/hadoop/DistcpActionExecutor.java | 27 +-- .../action/hadoop/Hive2ActionExecutor.java | 29 ++- .../oozie/action/hadoop/HiveActionExecutor.java | 29 ++- .../oozie/action/hadoop/JavaActionExecutor.java | 21 +++--- .../action/hadoop/SparkActionExecutor.java | 25 +-- .../action/hadoop/SqoopActionExecutor.java | 41 ++ .../oozie/action/hadoop/TestDistcpMain.java | 2 + examples/src/main/apps/hive/script.q| 1 + release-log.txt | 1 + .../apache/oozie/action/hadoop/DistcpMain.java | 9 ++- .../apache/oozie/action/hadoop/HiveMain.java| 5 +- .../action/hadoop/TestHiveActionExecutor.java | 15 +--- .../oozie/action/hadoop/TestHiveMain.java | 8 +- .../apache/oozie/action/hadoop/Hive2Main.java | 3 +- .../action/hadoop/TestHive2ActionExecutor.java | 57 +++--- .../oozie/action/hadoop/LauncherMain.java | 79 ++-- .../apache/oozie/action/hadoop/ShellMain.java | 2 +- .../org/apache/oozie/action/hadoop/PigMain.java | 8 +- .../oozie/action/hadoop/PigMainWithOldAPI.java | 2 +- .../apache/oozie/action/hadoop/SparkMain.java | 1 + .../apache/oozie/action/hadoop/SqoopMain.java | 16 ++-- .../action/hadoop/TestSqoopActionExecutor.java | 22 +- 22 files changed, 158 insertions(+), 245 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java index 96726da..78cd257 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java @@ -18,24 +18,15 @@ package org.apache.oozie.action.hadoop; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; -import org.apache.oozie.action.ActionExecutor.Context; -import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; -import org.apache.oozie.service.HadoopAccessorException; -import org.apache.oozie.service.Services; import org.apache.oozie.util.XLog; import org.jdom.Element; -import org.jdom.JDOMException; + +import java.util.ArrayList; +import java.util.List; public class DistcpActionExecutor extends JavaActionExecutor{ public static final String CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS = "org.apache.oozie.action.hadoop.DistcpMain"; @@ -124,16 +115,4 @@ public class DistcpActionExecutor extends JavaActionExecutor{ protected String getLauncherMain(Configuration launcherConf, Element actionXml) { return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS); } - -@Override -protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) -throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { -super.getActionData(actionFs, runningJob, action, context); -readExternalChildIDs(action, context); -} - -@Override -protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { -return true; -} } http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java index b5b1bf9..2aed936 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java +++
[43/50] [abbrv] oozie git commit: OOZIE-2673 Include XSD for shell-action:0.3 in documentation (abhishekbafna via rkanter)
OOZIE-2673 Include XSD for shell-action:0.3 in documentation (abhishekbafna via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3eca3c2b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3eca3c2b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3eca3c2b Branch: refs/heads/oya Commit: 3eca3c2bdecb61c995c2cf2180345cbc56ffb8a5 Parents: 376cdb3 Author: Robert KanterAuthored: Thu Sep 29 17:03:21 2016 -0700 Committer: Robert Kanter Committed: Thu Sep 29 17:03:21 2016 -0700 -- .../site/twiki/DG_ShellActionExtension.twiki| 58 release-log.txt | 1 + 2 files changed, 59 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/3eca3c2b/docs/src/site/twiki/DG_ShellActionExtension.twiki -- diff --git a/docs/src/site/twiki/DG_ShellActionExtension.twiki b/docs/src/site/twiki/DG_ShellActionExtension.twiki index 5a5759a..493254a 100644 --- a/docs/src/site/twiki/DG_ShellActionExtension.twiki +++ b/docs/src/site/twiki/DG_ShellActionExtension.twiki @@ -251,6 +251,64 @@ Although Shell action can execute any shell command, there are some limitations. ---+++ AE.A Appendix A, Shell XML-Schema +--- Shell Action Schema Version 0.3 + +http://www.w3.org/2001/XMLSchema; + xmlns:shell="uri:oozie:shell-action:0.3" elementFormDefault="qualified" + targetNamespace="uri:oozie:shell-action:0.3"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + --- Shell Action Schema Version 0.2 http://www.w3.org/2001/XMLSchema; http://git-wip-us.apache.org/repos/asf/oozie/blob/3eca3c2b/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 1ca7e2e..13c11df 100644 --- a/release-log.txt +++ b/release-log.txt @@ -3,6 +3,7 @@ -- Oozie 4.3.0 release +OOZIE-2673 Include XSD for shell-action:0.3 in documentation (abhishekbafna via rkanter) OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini) OOZIE-2501 ZK reentrant lock doesn't work for few cases (puru) OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini)
[28/50] [abbrv] oozie git commit: OOZIE-2487 Temporary workaround for Java 8 Javadoc errors (abhishekbafna via rkanter)
OOZIE-2487 Temporary workaround for Java 8 Javadoc errors (abhishekbafna via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a1d2687d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a1d2687d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a1d2687d Branch: refs/heads/oya Commit: a1d2687d6bc37875c0d3e868d3f4c460c7ccda8e Parents: 8e9b904 Author: Robert KanterAuthored: Wed Sep 21 17:55:04 2016 -0700 Committer: Robert Kanter Committed: Wed Sep 21 17:55:04 2016 -0700 -- pom.xml | 3 ++- release-log.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/a1d2687d/pom.xml -- diff --git a/pom.xml b/pom.xml index 73201f5..7ac9eae 100644 --- a/pom.xml +++ b/pom.xml @@ -1543,7 +1543,8 @@ maven-javadoc-plugin 2.10.3 - ${maven.javadoc.opts} + ${maven.javadoc.opts} +false http://git-wip-us.apache.org/repos/asf/oozie/blob/a1d2687d/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index e772070..dbfbafe 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2487 Temporary workaround for Java 8 Javadoc errors (abhishekbafna via rkanter) OOZIE-1978 Forkjoin validation code is ridiculously slow in some cases (pbacsko via rkanter) OOZIE-2682 Oozie test-patch script is not updating Jira with jenkins test report (abhishekbafna via rkanter) OOZIE-2679 Decrease HttpClient library versions due to Hadoop incompatibility (abhishekbafna via rkanter)
[29/50] [abbrv] oozie git commit: OOZIE-2676 Make hadoop-2 as the default profile (gezapeti via rkanter)
OOZIE-2676 Make hadoop-2 as the default profile (gezapeti via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/9827036d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/9827036d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/9827036d Branch: refs/heads/oya Commit: 9827036d7cecf9726ea31ee513895c81c988e22f Parents: a1d2687 Author: Robert KanterAuthored: Thu Sep 22 13:28:55 2016 -0700 Committer: Robert Kanter Committed: Thu Sep 22 13:28:55 2016 -0700 -- docs/src/site/twiki/DG_QuickStart.twiki | 4 ++-- docs/src/site/twiki/ENG_Building.twiki | 6 +++--- hadooplibs/pom.xml | 4 ++-- pom.xml | 8 release-log.txt | 1 + 5 files changed, 12 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/9827036d/docs/src/site/twiki/DG_QuickStart.twiki -- diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki index 0653f08..b1dea12 100644 --- a/docs/src/site/twiki/DG_QuickStart.twiki +++ b/docs/src/site/twiki/DG_QuickStart.twiki @@ -41,9 +41,9 @@ suitable when same oozie package needs to be used in multiple set-ups with diffe 2. Build with -Puber which will bundle the required libraries in the oozie war. Further, the following options are available to customise the versions of the dependencies: --P - default hadoop-1. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop +-P - default hadoop-2. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop profile depending on the hadoop version used. --Dhadoop.version= - default 1.2.1 for hadoop-1, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 +-Dhadoop.version= - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 -Dhadoop.auth.version= - defaults to hadoop version -Ddistcp.version= - defaults to hadoop version -Dpig.version= - default 0.16.0 http://git-wip-us.apache.org/repos/asf/oozie/blob/9827036d/docs/src/site/twiki/ENG_Building.twiki -- diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki index 9864098..f6c88d6 100644 --- a/docs/src/site/twiki/ENG_Building.twiki +++ b/docs/src/site/twiki/ENG_Building.twiki @@ -114,7 +114,7 @@ specified in the =test.properties= file (which is loaded by the =XTestCase= clas *hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-1 or Hadoop-2) you wish to build Oozie against specifically. It will substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven. Default version is 1.2.1 -for Hadoop-1 (the most common case). For Hadoop-2, the version you can pass is *2.3.0*. +for Hadoop-1 (the most common case). For Hadoop-2, the version you can pass is *2.4.0*. *generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated) @@ -212,9 +212,9 @@ $ bin/mkdistro.sh [-DskipTests] Running =mkdistro.sh= will create the binary distribution of Oozie. The following options are available to customise the versions of the dependencies: -Puber - Bundle required hadoop and hcatalog libraries in oozie war --P - default hadoop-1. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop +-P - default hadoop-2. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop profile depending on the hadoop version used. --Dhadoop.version= - default 1.2.1 for hadoop-1, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 +-Dhadoop.version= - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 -Dhadoop.auth.version= - defaults to hadoop version -Ddistcp.version= - defaults to hadoop version -Dpig.version= - default 0.16.0 http://git-wip-us.apache.org/repos/asf/oozie/blob/9827036d/hadooplibs/pom.xml -- diff --git a/hadooplibs/pom.xml b/hadooplibs/pom.xml index cd0f478..53844cc 100644 --- a/hadooplibs/pom.xml +++ b/hadooplibs/pom.xml @@ -47,7 +47,7 @@ hadoop-1 -true +false hadoop-utils-1 @@ -58,7 +58,7 @@ hadoop-2 -false +true hadoop-utils-2 http://git-wip-us.apache.org/repos/asf/oozie/blob/9827036d/pom.xml -- diff --git a/pom.xml b/pom.xml index 7ac9eae..c710468 100644 ---
[39/50] [abbrv] oozie git commit: OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini)
OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/76b60cc8 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/76b60cc8 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/76b60cc8 Branch: refs/heads/oya Commit: 76b60cc82840c0f66b0b3a75fd01aff2f57d27ec Parents: a7d500e Author: Rohini PalaniswamyAuthored: Tue Sep 27 11:56:32 2016 -0700 Committer: Rohini Palaniswamy Committed: Tue Sep 27 11:56:32 2016 -0700 -- release-log.txt | 1 + .../main/java/org/apache/oozie/action/hadoop/HiveMain.java | 1 + .../main/java/org/apache/oozie/action/hadoop/JavaMain.java | 1 + .../java/org/apache/oozie/action/hadoop/LauncherMain.java| 8 .../main/java/org/apache/oozie/action/hadoop/ShellMain.java | 2 ++ .../main/java/org/apache/oozie/action/hadoop/PigMain.java| 1 + 6 files changed, 14 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/76b60cc8/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 4805f84..ef11bb3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -3,6 +3,7 @@ -- Oozie 4.3.0 release +OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini) OOZIE-2676 Make hadoop-2 as the default profile (gezapeti via rkanter) OOZIE-2487 Temporary workaround for Java 8 Javadoc errors (abhishekbafna via rkanter) OOZIE-1978 Forkjoin validation code is ridiculously slow in some cases (pbacsko via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/76b60cc8/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java -- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index 8de3766..5866117 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -87,6 +87,7 @@ public class HiveMain extends LauncherMain { hiveConf.addResource(new Path("file:///", actionXml)); setYarnTag(hiveConf); +setApplicationTags(hiveConf, TEZ_APPLICATION_TAGS); // Propagate delegation related props from launcher job to Hive job String delegationToken = getFilePathFromEnv("HADOOP_TOKEN_FILE_LOCATION"); http://git-wip-us.apache.org/repos/asf/oozie/blob/76b60cc8/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java index e4f4b43..16bd4e6 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@ -41,6 +41,7 @@ public class JavaMain extends LauncherMain { Configuration actionConf = loadActionConf(); setYarnTag(actionConf); +setApplicationTags(actionConf, TEZ_APPLICATION_TAGS); LauncherMainHadoopUtils.killChildYarnJobs(actionConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/76b60cc8/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 054f8ea..7c5713c 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -44,6 +44,7 @@ public abstract class LauncherMain { public static final String HADOOP_JOBS = "hadoopJobs"; public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags"; +public static final String TEZ_APPLICATION_TAGS = "tez.application.tags"; protected static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"}; @@ -216,6 +217,13 @@ public abstract class LauncherMain { } } +protected static void setApplicationTags(Configuration configName, String tagConfigName) { +if (configName.get(MAPREDUCE_JOB_TAGS) != null) { +System.out.println("Setting [" + tagConfigName + "] tag: " + configName.get(MAPREDUCE_JOB_TAGS)); +
[26/50] [abbrv] oozie git commit: OOZIE-1978 Forkjoin validation code is ridiculously slow in some cases (pbacsko via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/8e9b9042/core/src/test/resources/wf-long.xml -- diff --git a/core/src/test/resources/wf-long.xml b/core/src/test/resources/wf-long.xml new file mode 100644 index 000..ef8315e --- /dev/null +++ b/core/src/test/resources/wf-long.xml @@ -0,0 +1,1456 @@ + + + +${jobTracker} +${nameNode} + + +mapred.job.queue.name +${queueName} + + + + + + + + + + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + + + + + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + + + + + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + + + + + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + + + + + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + + + + + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + +echo +foo + + + + + + + + + + + + + + + + + + + +echo +foo + + + + + + + + +echo +foo +
[48/50] [abbrv] oozie git commit: OOZIE-2697 fix compilation problem
OOZIE-2697 fix compilation problem Change-Id: Ic16acc07ef75ff2a1e429c6b96632ed2e7c21b81 Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b1069b2f Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b1069b2f Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b1069b2f Branch: refs/heads/oya Commit: b1069b2fa0dd861e960900f8673069eedb586b3c Parents: 3a8f00f Author: Peter BacskoAuthored: Fri Sep 30 17:28:56 2016 +0200 Committer: Peter Bacsko Committed: Fri Sep 30 17:28:56 2016 +0200 -- .../src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java| 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/b1069b2f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 0570d16..d51dcf7 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -145,7 +145,6 @@ public class LauncherAM { System.out.println("Login: " + login.getUserName()); System.out.println("SecurityEnabled:" + UserGroupInformation.isSecurityEnabled()); System.out.println("Login keytab based:" + UserGroupInformation.isLoginKeytabBased()); -System.out.println("Login ticket based:" + UserGroupInformation.isLoginTicketBased()); System.out.println("Login from keytab: " + login.isFromKeytab()); System.out.println("Login has kerberos credentials: " + login.hasKerberosCredentials()); System.out.println("Login authMethod: " + login.getAuthenticationMethod());
[30/50] [abbrv] oozie git commit: Changed version to 4.4.0-SNAPSHOT
Changed version to 4.4.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a7d500e8 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a7d500e8 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a7d500e8 Branch: refs/heads/oya Commit: a7d500e88cf50d31eb55b4a5e8e6c95a8875028c Parents: 9827036 Author: Shwetha GSAuthored: Mon Sep 26 10:23:07 2016 +0530 Committer: Shwetha GS Committed: Mon Sep 26 10:23:07 2016 +0530 -- client/pom.xml | 4 ++-- core/pom.xml | 4 ++-- distro/pom.xml | 4 ++-- docs/pom.xml | 4 ++-- examples/pom.xml | 4 ++-- hadooplibs/hadoop-auth-1/pom.xml | 4 ++-- hadooplibs/hadoop-auth-2/pom.xml | 4 ++-- hadooplibs/hadoop-distcp-1/pom.xml | 4 ++-- hadooplibs/hadoop-distcp-2/pom.xml | 4 ++-- hadooplibs/hadoop-distcp-3/pom.xml | 4 ++-- hadooplibs/hadoop-utils-1/pom.xml | 4 ++-- hadooplibs/hadoop-utils-2/pom.xml | 4 ++-- hadooplibs/hadoop-utils-3/pom.xml | 4 ++-- hadooplibs/pom.xml | 4 ++-- login/pom.xml | 4 ++-- minitest/pom.xml | 4 ++-- pom.xml| 2 +- release-log.txt| 5 - sharelib/distcp/pom.xml| 4 ++-- sharelib/hcatalog/pom.xml | 4 ++-- sharelib/hive/pom.xml | 4 ++-- sharelib/hive2/pom.xml | 4 ++-- sharelib/oozie/pom.xml | 4 ++-- sharelib/pig/pom.xml | 4 ++-- sharelib/pom.xml | 4 ++-- sharelib/spark/pom.xml | 4 ++-- sharelib/sqoop/pom.xml | 4 ++-- sharelib/streaming/pom.xml | 4 ++-- tools/pom.xml | 4 ++-- webapp/pom.xml | 4 ++-- zookeeper-security-tests/pom.xml | 4 ++-- 31 files changed, 63 insertions(+), 60 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/a7d500e8/client/pom.xml -- diff --git a/client/pom.xml b/client/pom.xml index 632d256..f99366c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -22,11 +22,11 @@ org.apache.oozie oozie-main -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT org.apache.oozie oozie-client -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT Apache Oozie Client Apache Oozie Client jar http://git-wip-us.apache.org/repos/asf/oozie/blob/a7d500e8/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index 4c63cca..4e6ef84 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,11 +22,11 @@ org.apache.oozie oozie-main -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT org.apache.oozie oozie-core -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT Apache Oozie Core Apache Oozie Core jar http://git-wip-us.apache.org/repos/asf/oozie/blob/a7d500e8/distro/pom.xml -- diff --git a/distro/pom.xml b/distro/pom.xml index 7baaff2..c50572c 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -22,11 +22,11 @@ org.apache.oozie oozie-main -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT org.apache.oozie oozie-distro -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT Apache Oozie Distro Apache Oozie Distro jar http://git-wip-us.apache.org/repos/asf/oozie/blob/a7d500e8/docs/pom.xml -- diff --git a/docs/pom.xml b/docs/pom.xml index 59d1a2d..d7a297d 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -22,11 +22,11 @@ org.apache.oozie oozie-main -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT org.apache.oozie oozie-docs -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT Apache Oozie Docs Apache Oozie Docs war http://git-wip-us.apache.org/repos/asf/oozie/blob/a7d500e8/examples/pom.xml -- diff --git a/examples/pom.xml b/examples/pom.xml index be4e932..8203664 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -22,11 +22,11 @@ org.apache.oozie oozie-main -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT org.apache.oozie oozie-examples -4.3.0-SNAPSHOT +4.4.0-SNAPSHOT Apache Oozie Examples Apache Oozie Examples jar http://git-wip-us.apache.org/repos/asf/oozie/blob/a7d500e8/hadooplibs/hadoop-auth-1/pom.xml -- diff --git
[45/50] [abbrv] oozie git commit: Merge branch 'apache_forked_master' into apache_forked_oya
Merge branch 'apache_forked_master' into apache_forked_oya Conflicts: core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java core/src/main/java/org/apache/oozie/service/ShareLibService.java core/src/main/java/org/apache/oozie/util/JobUtils.java core/src/main/java/org/apache/oozie/util/NamedThreadFactory.java core/src/main/resources/oozie-default.xml core/src/test/java/org/apache/oozie/QueryServlet.java core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java docs/src/site/twiki/DG_QuickStart.twiki docs/src/site/twiki/ENG_Building.twiki hadooplibs/hadoop-auth-1/pom.xml hadooplibs/hadoop-auth-2/pom.xml hadooplibs/hadoop-distcp-1/pom.xml hadooplibs/hadoop-distcp-2/pom.xml hadooplibs/hadoop-distcp-3/pom.xml hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java hadooplibs/hadoop-utils-1/pom.xml hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/hadoop-utils-2/pom.xml hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/hadoop-utils-3/pom.xml hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/pom.xml pom.xml release-log.txt sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java sharelib/spark/pom.xml sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java Change-Id: I6697e098f84051365e3b11d93eb6effe124ed47d Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c49f382b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c49f382b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c49f382b Branch: refs/heads/oya Commit: c49f382bef2d0b101dda9a40cdf36c4cba454f85 Parents: ed72570 3eca3c2 Author: Peter BacskoAuthored: Fri Sep 30 12:33:27 2016 +0200 Committer: Peter Bacsko Committed: Fri Sep 30 12:33:27 2016 +0200 -- bin/test-patch |2 +- bin/test-patch-10-compile |4 +- client/pom.xml |4 +- .../java/org/apache/oozie/cli/OozieCLI.java |5 +- .../java/org/apache/oozie/client/BundleJob.java |4 +- .../apache/oozie/client/CoordinatorAction.java |2 +- .../org/apache/oozie/client/CoordinatorJob.java |4 +- .../main/java/org/apache/oozie/client/Job.java |6 +- .../org/apache/oozie/client/OozieClient.java| 23 +- .../java/org/apache/oozie/client/SLAEvent.java | 46 +- .../org/apache/oozie/client/WorkflowAction.java |2 +- .../org/apache/oozie/client/WorkflowJob.java|2 +- .../org/apache/oozie/client/XOozieClient.java | 31 +- .../org/apache/oozie/client/rest/JsonTags.java | 438 +++--- .../org/apache/oozie/client/rest/JsonUtils.java |9 +- .../apache/oozie/client/rest/RestConstants.java | 192 +-- .../src/main/resources/oozie-workflow-0.5.xsd |1 + core/pom.xml| 45 +- .../org/apache/oozie/CoordinatorEngine.java |6 +- .../main/java/org/apache/oozie/DagEngine.java |1 + .../main/java/org/apache/oozie/ErrorCode.java |4 +- .../org/apache/oozie/LocalOozieClientCoord.java | 25 + .../org/apache/oozie/action/ActionExecutor.java | 46 +- .../action/hadoop/DistcpActionExecutor.java | 24 +- .../oozie/action/hadoop/FsActionExecutor.java |9 +- .../action/hadoop/HCatCredentialHelper.java |8 +- .../oozie/action/hadoop/HCatCredentials.java| 49 +- .../oozie/action/hadoop/HadoopELFunctions.java |5 - .../action/hadoop/Hive2ActionExecutor.java | 30 +- .../oozie/action/hadoop/Hive2Credentials.java | 17 +-
[16/50] [abbrv] oozie git commit: OOZIE-2498 Oozie CallerId configuration for downstream components (abhishekbafna via rohini)
OOZIE-2498 Oozie CallerId configuration for downstream components (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3ad35372 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3ad35372 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3ad35372 Branch: refs/heads/oya Commit: 3ad35372e20d9e995688d52548f7858537f22be3 Parents: 3bd45e9 Author: Rohini PalaniswamyAuthored: Mon Sep 19 10:20:44 2016 -0700 Committer: Rohini Palaniswamy Committed: Mon Sep 19 10:20:44 2016 -0700 -- release-log.txt | 1 + .../src/main/java/org/apache/oozie/action/hadoop/HiveMain.java | 5 + .../src/main/java/org/apache/oozie/action/hadoop/PigMain.java | 4 3 files changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/3ad35372/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index a8afcf4..515521c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2498 Oozie CallerId configuration for downstream components (abhishekbafna via rohini) OOZIE-2491 oozie acl cannot specify group,it does`t work (abhishekbafna via rohini) OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) http://git-wip-us.apache.org/repos/asf/oozie/blob/3ad35372/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java -- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index a2c75dc..8de3766 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -224,6 +224,11 @@ public class HiveMain extends LauncherMain { arguments.add("--hiveconf"); arguments.add("hive.exec.log4j.file=" + new File(HIVE_EXEC_L4J_PROPS).getAbsolutePath()); +//setting oozie workflow id as caller context id for hive +String callerId = "oozie:" + System.getProperty("oozie.job.id"); +arguments.add("--hiveconf"); +arguments.add("hive.log.trace.id=" + callerId); + String scriptPath = hiveConf.get(HiveActionExecutor.HIVE_SCRIPT); String query = hiveConf.get(HiveActionExecutor.HIVE_QUERY); if (scriptPath != null) { http://git-wip-us.apache.org/repos/asf/oozie/blob/3ad35372/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java -- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index eaab1fe..44debbd 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -117,6 +117,10 @@ public class PigMain extends LauncherMain { System.out.println("Non-kerberoes execution"); } +//setting oozie workflow id as caller context id for pig +String callerId = "oozie:" + System.getProperty("oozie.job.id"); +pigProperties.setProperty("pig.log.trace.id", callerId); + OutputStream os = new FileOutputStream("pig.properties"); pigProperties.store(os, ""); os.close();
[12/50] [abbrv] oozie git commit: OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)
OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2fd64fa6 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2fd64fa6 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2fd64fa6 Branch: refs/heads/oya Commit: 2fd64fa646708ae684f2274729a0df6623598709 Parents: 9dc474e Author: Rohini PalaniswamyAuthored: Mon Sep 19 09:53:25 2016 -0700 Committer: Rohini Palaniswamy Committed: Mon Sep 19 09:53:25 2016 -0700 -- .../oozie/action/hadoop/HadoopELFunctions.java | 5 -- .../oozie/command/wf/SubmitMRXCommand.java | 3 -- .../java/org/apache/oozie/util/JobUtils.java| 2 +- .../java/org/apache/oozie/test/XTestCase.java | 4 +- docs/src/site/twiki/DG_QuickStart.twiki | 5 +- .../site/twiki/DG_SqoopActionExtension.twiki| 2 +- docs/src/site/twiki/ENG_Building.twiki | 5 +- hadooplibs/hadoop-auth-0.23/pom.xml | 45 - hadooplibs/hadoop-distcp-0.23/pom.xml | 43 - hadooplibs/hadoop-utils-0.23/pom.xml| 42 .../action/hadoop/LauncherMainHadoopUtils.java | 35 -- .../apache/oozie/hadoop/utils/HadoopShims.java | 51 hadooplibs/pom.xml | 11 - pom.xml | 12 - release-log.txt | 1 + 15 files changed, 8 insertions(+), 258 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java index c322887..ad2a71d 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java @@ -52,11 +52,6 @@ public class HadoopELFunctions { Map > counters = (Map >) obj; if (counters == null) { counters = getCounters(nodeName); -// In Hadoop 0.23 they deprecated 'org.apache.hadoop.mapred.Task$Counter' and they REMOVED IT -// Here we are getting the new Name and inserting it using the old name if the old name is not found -if (counters.get(RECORDS) == null) { -counters.put(RECORDS, counters.get(RECORDS_023)); -} instance.setTransientVar(nodeName + WorkflowInstance.NODE_VAR_SEPARATOR + HADOOP_COUNTERS, counters); } return counters; http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java -- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java index 9124a45..cc61d3d 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java @@ -43,9 +43,6 @@ public class SubmitMRXCommand extends SubmitHttpXCommand { SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER); SKIPPED_CONFS.add(XOozieClient.JT); SKIPPED_CONFS.add(XOozieClient.NN); -// a brillant mind made a change in Configuration that 'fs.default.name' key gets converted to 'fs.defaultFS' -// in Hadoop 0.23, we need skip that one too, keeping the old one because of Hadoop 1 -SKIPPED_CONFS.add(XOozieClient.NN_2); DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2); DEPRECATE_MAP.put(XOozieClient.JT, XOozieClient.JT_2); http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/util/JobUtils.java -- diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java index a4d7272..a7a53b3 100644 --- a/core/src/main/java/org/apache/oozie/util/JobUtils.java +++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java @@ -162,7 +162,7 @@ public class JobUtils { URI uri = fs.makeQualified(file).toUri(); DistributedCache.addCacheFile(uri, conf); } -else { // Hadoop 0.23/2.x +else { // Hadoop 2.x DistributedCache.addFileToClassPath(file, conf, fs); } }
[34/50] [abbrv] oozie git commit: OOZIE-2596 Pyspark temporary fix
OOZIE-2596 Pyspark temporary fix Change-Id: I25f3619a4b1df73a73e6833a45c27e79dfa3751e Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b584481a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b584481a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b584481a Branch: refs/heads/oya Commit: b584481a70e80379d80806ac06b9e244e78d3149 Parents: 11a8429 Author: Peter CsehAuthored: Mon Sep 26 15:21:47 2016 +0200 Committer: Peter Cseh Committed: Mon Sep 26 15:25:51 2016 +0200 -- .../java/org/apache/oozie/action/hadoop/JavaActionExecutor.java | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/b584481a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 8b5f2b0..dbe7bc1 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -1173,6 +1173,9 @@ public class JavaActionExecutor extends ActionExecutor { // FIXME: move this to specific places where it's actually needed - keeping it here for now ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf); +// FIXME: Pyspark fix +// FIXME: Do we want to support mapred.child.env? +env.put("SPARK_HOME", "."); amContainer.setEnvironment(env);
[21/50] [abbrv] oozie git commit: OOZIE-2667 Optimize queries for DB export (gezapeti via rkanter)
OOZIE-2667 Optimize queries for DB export (gezapeti via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a16de898 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a16de898 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a16de898 Branch: refs/heads/oya Commit: a16de8980c4439768bfaa055a048f437b95fcee9 Parents: ee4d8f3 Author: Robert KanterAuthored: Tue Sep 20 14:22:47 2016 -0700 Committer: Robert Kanter Committed: Tue Sep 20 14:22:47 2016 -0700 -- release-log.txt | 1 + .../apache/oozie/tools/OozieDBExportCLI.java| 34 +++- 2 files changed, 20 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/a16de898/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 8978c37..0dd42d0 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2667 Optimize queries for DB export (gezapeti via rkanter) OOZIE-1793 Improve find bugs reporting for Oozie (rkanter) OOZIE-2572 SLA DURATION miss not shown when job is running for longer than expected time (satishsaley via puru) OOZIE-2525 SchemaChecker fails with NPE (rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/a16de898/tools/src/main/java/org/apache/oozie/tools/OozieDBExportCLI.java -- diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieDBExportCLI.java b/tools/src/main/java/org/apache/oozie/tools/OozieDBExportCLI.java index 740a48b..ba74ef1 100644 --- a/tools/src/main/java/org/apache/oozie/tools/OozieDBExportCLI.java +++ b/tools/src/main/java/org/apache/oozie/tools/OozieDBExportCLI.java @@ -35,10 +35,6 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.Iterator; import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -65,6 +61,16 @@ public class OozieDBExportCLI { public static final String OOZIEDB_SLASUM_JSON = "ooziedb_slasum.json"; public static final String OOZIEDB_SYS_INFO_JSON = "ooziedb_sysinfo.json"; +private static final String GET_DB_VERSION = "select name, data from OOZIE_SYS where name = 'db.version'"; +private static final String GET_WORKFLOW_JOBS = "select OBJECT(w) from WorkflowJobBean w"; +private static final String GET_WORKFLOW_ACTIONS = "select OBJECT(a) from WorkflowActionBean a"; +private static final String GET_COORD_JOBS = "select OBJECT(w) from CoordinatorJobBean w"; +private static final String GET_COORD_ACTIONS = "select OBJECT(w) from CoordinatorActionBean w"; +private static final String GET_BUNDLE_JOBS = "select OBJECT(w) from BundleJobBean w"; +private static final String GET_BUNDLE_ACIONS = "select OBJECT(w) from BundleActionBean w"; +private static final String GET_SLA_REGISTRATIONS = "select OBJECT(w) from SLARegistrationBean w"; +private static final String GET_SLA_SUMMARYS = "select OBJECT(w) from SLASummaryBean w"; + private static final int LIMIT = 1000; private static final String[] HELP_INFO = { "", @@ -126,33 +132,31 @@ public class OozieDBExportCLI { manager = Services.get().get(JPAService.class).getEntityManager(); manager.setFlushMode(FlushModeType.COMMIT); -int infoSize = exportTableToJSON(manager.createNativeQuery("select name, data from OOZIE_SYS where name = 'db.version'"), zos, OOZIEDB_SYS_INFO_JSON); +int infoSize = exportTableToJSON(manager.createNativeQuery(GET_DB_VERSION), zos, OOZIEDB_SYS_INFO_JSON); System.out.println(infoSize + " rows exported from OOZIE_SYS"); -int wfjSize = exportTableToJSON(manager.createNamedQuery("GET_WORKFLOWS"), zos, OOZIEDB_WF_JSON); +int wfjSize = exportTableToJSON(manager.createQuery(GET_WORKFLOW_JOBS), zos, OOZIEDB_WF_JSON); System.out.println(wfjSize + " rows exported from WF_JOBS"); -int wfaSize = exportTableToJSON(manager.createNamedQuery("GET_ACTIONS"), zos, OOZIEDB_AC_JSON); +int wfaSize = exportTableToJSON(manager.createQuery(GET_WORKFLOW_ACTIONS), zos, OOZIEDB_AC_JSON); System.out.println(wfaSize + " rows exported from WF_ACTIONS"); -int cojSize = exportTableToJSON(manager.createNamedQuery("GET_COORD_JOBS"), zos, OOZIEDB_CJ_JSON); +int cojSize = exportTableToJSON(manager.createQuery(GET_COORD_JOBS), zos,
[36/50] [abbrv] oozie git commit: OOZIE-2598 make Shell action work, fix tests
OOZIE-2598 make Shell action work, fix tests Change-Id: Ibaad392c9beb7ad9dec7f59b2b49436080572257 Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3d339c26 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3d339c26 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3d339c26 Branch: refs/heads/oya Commit: 3d339c26daa29cbde377005b2cf0870f9b3e8186 Parents: b584481 Author: Peter BacskoAuthored: Wed Sep 21 17:03:06 2016 +0200 Committer: Peter Cseh Committed: Mon Sep 26 15:25:51 2016 +0200 -- .../action/hadoop/TestShellActionExecutor.java | 5 ++-- .../java/org/apache/oozie/test/XTestCase.java | 1 + .../oozie/action/hadoop/LauncherMain.java | 25 .../oozie/action/hadoop/MapReduceMain.java | 24 ++- 4 files changed, 26 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/3d339c26/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java -- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java index 3354b3a..a48f211 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java @@ -48,7 +48,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { private static final String SHELL_EXEC = Shell.WINDOWS ? "cmd.exe" : "sh"; private static final String SHELL_PARAM = Shell.WINDOWS ? "/c" : "-c"; -private static final String SHELL_SCRIPTNAME = Shell.WINDOWS ? "script.cmd" : "script.sh"; +private static final String SHELL_SCRIPTNAME = Shell.WINDOWS ? "script.cmd" : "./script.sh"; private static final String SHELL_SCRIPT_CONTENT = Shell.WINDOWS ? "dir /s /b\necho %1 %2\necho %PATH%\ntype %0" : "ls -ltr\necho $1 $2\necho $PATH\npwd\ntype sh"; @@ -252,8 +252,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { private WorkflowAction _testSubmit(String actionXml, boolean checkForSuccess, String capture_output) throws Exception { Context context = createContext(actionXml); -final String launcherId = submitAction(context);// Submit the -// action +final String launcherId = submitAction(context);// Submit the action waitUntilYarnAppDoneAndAssertSuccess(launcherId); Configuration conf = new XConfiguration(); http://git-wip-us.apache.org/repos/asf/oozie/blob/3d339c26/core/src/test/java/org/apache/oozie/test/XTestCase.java -- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 1299fa3..da5c5cf 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -1224,6 +1224,7 @@ public abstract class XTestCase extends TestCase { } } +log.info("Final state is: {0}", finalState.getValue()); assertTrue(endStateOK.isTrue()); return finalState.getValue(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/3d339c26/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 6955416..9b5d57d 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -41,7 +41,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; @@ -241,30 +240,6 @@ public abstract class LauncherMain { } /** - * Will run the user specified OozieActionConfigurator subclass (if one is provided) to update the action configuration. - * - * @param actionConf The action configuration to update - * @throws OozieActionConfiguratorException - */ -protected static void runConfigClass(JobConf actionConf)
[42/50] [abbrv] oozie git commit: OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini)
OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/376cdb3d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/376cdb3d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/376cdb3d Branch: refs/heads/oya Commit: 376cdb3d8cd4f3a5c2c3f3e771df19c50e57b8f4 Parents: d330d40 Author: Rohini PalaniswamyAuthored: Tue Sep 27 16:22:39 2016 -0700 Committer: Rohini Palaniswamy Committed: Tue Sep 27 16:22:39 2016 -0700 -- release-log.txt | 1 + .../src/main/java/org/apache/oozie/action/hadoop/JavaMain.java | 1 + .../main/java/org/apache/oozie/action/hadoop/LauncherMain.java | 2 +- .../src/main/java/org/apache/oozie/action/hadoop/ShellMain.java | 1 + .../src/main/java/org/apache/oozie/action/hadoop/SparkMain.java | 5 + 5 files changed, 9 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index b03a61a..1ca7e2e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -3,6 +3,7 @@ -- Oozie 4.3.0 release +OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini) OOZIE-2501 ZK reentrant lock doesn't work for few cases (puru) OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini) OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini) http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java index 16bd4e6..30d68e2 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@ -42,6 +42,7 @@ public class JavaMain extends LauncherMain { setYarnTag(actionConf); setApplicationTags(actionConf, TEZ_APPLICATION_TAGS); +setApplicationTags(actionConf, SPARK_YARN_TAGS); LauncherMainHadoopUtils.killChildYarnJobs(actionConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 815f60b..785ca5e 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -31,7 +31,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashSet; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,6 +51,7 @@ public abstract class LauncherMain { public static final String HADOOP_JOBS = "hadoopJobs"; public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags"; public static final String TEZ_APPLICATION_TAGS = "tez.application.tags"; +public static final String SPARK_YARN_TAGS = "spark.yarn.tags"; protected static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"}; http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java index 1e9d8af..f109318 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java @@ -65,6 +65,7 @@ public class ShellMain extends LauncherMain { Configuration actionConf = loadActionConf(); setYarnTag(actionConf); setApplicationTags(actionConf, TEZ_APPLICATION_TAGS); +setApplicationTags(actionConf, SPARK_YARN_TAGS); int exitCode = execute(actionConf); if (exitCode != 0) {
[20/50] [abbrv] oozie git commit: OOZIE-1793 Improve find bugs reporting for Oozie (rkanter)
OOZIE-1793 Improve find bugs reporting for Oozie (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ee4d8f32 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ee4d8f32 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ee4d8f32 Branch: refs/heads/oya Commit: ee4d8f324cd5fbb1bcc16c513b40a42b428d8b61 Parents: f45e1eb Author: Robert KanterAuthored: Tue Sep 20 14:15:22 2016 -0700 Committer: Robert Kanter Committed: Tue Sep 20 14:15:22 2016 -0700 -- pom.xml | 65 +++- release-log.txt | 1 + 2 files changed, 65 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/ee4d8f32/pom.xml -- diff --git a/pom.xml b/pom.xml index 2cbc91f..b91dcbc 100644 --- a/pom.xml +++ b/pom.xml @@ -1603,6 +1603,16 @@ maven-bundle-plugin 2.4.0 + +org.codehaus.mojo +findbugs-maven-plugin +3.0.0 + + +org.codehaus.mojo +xml-maven-plugin +1.0 + @@ -1620,17 +1630,70 @@ + + org.codehaus.mojo findbugs-maven-plugin +false true true true +Max +false +Low +true + ${project.build.directory}/findbugs + + + +findbug +verify + +check + + + + + + +org.codehaus.mojo +xml-maven-plugin + +false + + +${project.build.directory}/findbugs + ${project.build.directory}/findbugs +fancy-hist.xsl + + +.html + + + + + + +verify + +transform + + + + + +com.google.code.findbugs +findbugs +2.0.3 + + - + org.apache.maven.plugins maven-checkstyle-plugin http://git-wip-us.apache.org/repos/asf/oozie/blob/ee4d8f32/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index e2e5ec6..8978c37 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-1793 Improve find bugs reporting for Oozie (rkanter) OOZIE-2572 SLA DURATION miss not shown when job is running for longer than expected time (satishsaley via puru) OOZIE-2525 SchemaChecker fails with NPE (rkanter) OOZIE-2672 SLA periodic update does not remove job from map if job is removed from database (satishsaley via puru)
[22/50] [abbrv] oozie git commit: OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter)
OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/00275b71 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/00275b71 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/00275b71 Branch: refs/heads/oya Commit: 00275b71a626a5035b9802309aadf7df378c5e59 Parents: a16de89 Author: Robert KanterAuthored: Tue Sep 20 15:16:29 2016 -0700 Committer: Robert Kanter Committed: Tue Sep 20 15:16:29 2016 -0700 -- .../command/coord/TestCoordRerunXCommand.java | 2 +- ...TestHAPartitionDependencyManagerService.java | 1 + .../org/apache/oozie/test/XHCatTestCase.java| 1 + .../java/org/apache/oozie/test/XTestCase.java | 82 +++- .../oozie/example/TestLocalOozieExample.java| 1 - .../org/apache/oozie/test/WorkflowTest.java | 1 - release-log.txt | 1 + .../action/hadoop/TestPigActionExecutor.java| 1 - .../apache/oozie/action/hadoop/TestPigMain.java | 1 - 9 files changed, 67 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java -- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java index cdfc9d6..891e6ec 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java @@ -569,7 +569,7 @@ public class TestCoordRerunXCommand extends XDataTestCase { * @throws Exception */ public void testCoordRerunCleanupForHCat() throws Exception { - +super.setupHCatalogServer(); services = super.setupServicesForHCatalog(); services.init(); http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java -- diff --git a/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java b/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java index d681d42..3e1df07 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java @@ -47,6 +47,7 @@ public class TestHAPartitionDependencyManagerService extends ZKXTestCase { protected void setUp() throws Exception { super.setUp(); services = super.setupServicesForHCatalog(Services.get()); +super.setupHCatalogServer(); // disable recovery service services.getConf().setInt(RecoveryService.CONF_SERVICE_INTERVAL, 100); // disable regular cache purge http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java -- diff --git a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java index 85ee1f2..2adbee7 100644 --- a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java @@ -40,6 +40,7 @@ public abstract class XHCatTestCase extends XFsTestCase { @Override protected void setUp() throws Exception { super.setUp(); +super.setupHCatalogServer(); hcatServer = super.getHCatalogServer(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/test/XTestCase.java -- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 3dd99d7..70f77be 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -130,12 +130,12 @@ public abstract class XTestCase extends TestCase { OOZIE_SRC_DIR = new File(OOZIE_SRC_DIR, "core"); } if (!OOZIE_SRC_DIR.exists()) { -System.err.println(); -System.err.println("Could not determine project root directory"); -System.err.println(); -System.exit(-1); +// We're probably being run from outside of Oozie (e.g. MiniOozie), so just use a dummy location here. +
[15/50] [abbrv] oozie git commit: OOZIE-2491 oozie acl cannot specify group, it does not work (abhishekbafna via rohini)
OOZIE-2491 oozie acl cannot specify group,it does not work (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3bd45e92 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3bd45e92 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3bd45e92 Branch: refs/heads/oya Commit: 3bd45e92475ef7b983176b1c52c1bceb20de6e49 Parents: 0cdce5a Author: Rohini PalaniswamyAuthored: Mon Sep 19 10:13:25 2016 -0700 Committer: Rohini Palaniswamy Committed: Mon Sep 19 10:13:25 2016 -0700 -- .../java/org/apache/oozie/service/AuthorizationService.java| 6 +++--- .../org/apache/oozie/service/TestAuthorizationService.java | 5 - release-log.txt| 1 + 3 files changed, 8 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/3bd45e92/core/src/main/java/org/apache/oozie/service/AuthorizationService.java -- diff --git a/core/src/main/java/org/apache/oozie/service/AuthorizationService.java b/core/src/main/java/org/apache/oozie/service/AuthorizationService.java index f281873..4e4ab17 100644 --- a/core/src/main/java/org/apache/oozie/service/AuthorizationService.java +++ b/core/src/main/java/org/apache/oozie/service/AuthorizationService.java @@ -295,7 +295,7 @@ public class AuthorizationService implements Service { * * @param user user name. * @param write indicates if the check is for read or write admin tasks (in this implementation this is ignored) - * @throws AuthorizationException thrown if user does not have admin priviledges. + * @throws AuthorizationException thrown if user does not have admin privileges. */ public void authorizeForAdmin(String user, boolean write) throws AuthorizationException { if (authorizationEnabled && write && !isAdmin(user)) { @@ -417,7 +417,7 @@ public class AuthorizationService implements Service { String[] acl = aclStr.split(","); for (int i = 0; !userInAcl && i < acl.length; i++) { String aclItem = acl[i].trim(); -userInAcl = aclItem.equals(user) || groupsService.getGroups(user).equals(aclItem); +userInAcl = aclItem.equals(user) || groupsService.getGroups(user).contains(aclItem); } } return userInAcl; @@ -515,7 +515,7 @@ public class AuthorizationService implements Service { * @param user user name. * @param filter filter used to select jobs * @param start starting index of the jobs in DB - * @param len maximum amount of jbos to select + * @param len maximum amount of jobs to select * @param write indicates if the check is for read or write job tasks. * @throws AuthorizationException thrown if the user is not authorized for the job. */ http://git-wip-us.apache.org/repos/asf/oozie/blob/3bd45e92/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java -- diff --git a/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java b/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java index ce461c1..b5e527e 100644 --- a/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java +++ b/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java @@ -64,6 +64,9 @@ public class TestAuthorizationService extends XDataTestCase { if (getTestUser().equals(user)) { return Arrays.asList("users", getTestGroup()); } +if ("foo".equals(user)) { +return Arrays.asList("users", "foogrp"); +} else { return Arrays.asList("users"); } @@ -129,7 +132,7 @@ public class TestAuthorizationService extends XDataTestCase { jobConf.set(OozieClient.GROUP_NAME, getTestGroup()); } else { -jobConf.set(OozieClient.GROUP_NAME, getTestGroup() + ",foo"); +jobConf.set(OozieClient.GROUP_NAME, getTestGroup() + ",foogrp"); } jobConf.set(OozieClient.LOG_TOKEN, "t"); http://git-wip-us.apache.org/repos/asf/oozie/blob/3bd45e92/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 7851e81..a8afcf4 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2491 oozie acl cannot specify group,it does`t work (abhishekbafna via rohini) OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher
[24/50] [abbrv] oozie git commit: OOZIE-2679 Decrease HttpClient library versions due to Hadoop incompatibility (abhishekbafna via rkanter)
OOZIE-2679 Decrease HttpClient library versions due to Hadoop incompatibility (abhishekbafna via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/85e79ec7 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/85e79ec7 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/85e79ec7 Branch: refs/heads/oya Commit: 85e79ec77cfd8787caf468b533a926f832508612 Parents: 24289ed Author: Robert KanterAuthored: Wed Sep 21 11:11:31 2016 -0700 Committer: Robert Kanter Committed: Wed Sep 21 11:11:31 2016 -0700 -- pom.xml | 4 ++-- release-log.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/85e79ec7/pom.xml -- diff --git a/pom.xml b/pom.xml index b91dcbc..73201f5 100644 --- a/pom.xml +++ b/pom.xml @@ -111,8 +111,8 @@ 1.8.8 1.2.17 5.13.3 - 4.4 - 4.4 + 4.3.3 + 4.3.6 http://git-wip-us.apache.org/repos/asf/oozie/blob/85e79ec7/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 568fc3f..b503f21 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2679 Decrease HttpClient library versions due to Hadoop incompatibility (abhishekbafna via rkanter) OOZIE-2661 Coordinator Action status not updated when workflow job SUSPENDED (satishsaley via puru) OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter) OOZIE-2667 Optimize queries for DB export (gezapeti via rkanter)
[08/50] [abbrv] oozie git commit: OOZIE-2037 Add TLSv1.1, TLSv1.2 (rkanter)
OOZIE-2037 Add TLSv1.1,TLSv1.2 (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/d4d35bd1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/d4d35bd1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/d4d35bd1 Branch: refs/heads/oya Commit: d4d35bd1b973072b1583c7ee92a2bbffedc2006e Parents: 65f59ff Author: Robert KanterAuthored: Fri Sep 16 14:00:40 2016 -0700 Committer: Robert Kanter Committed: Fri Sep 16 14:00:40 2016 -0700 -- distro/src/main/tomcat/ssl-server.xml | 2 +- release-log.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/d4d35bd1/distro/src/main/tomcat/ssl-server.xml -- diff --git a/distro/src/main/tomcat/ssl-server.xml b/distro/src/main/tomcat/ssl-server.xml index 7ad3d16..9a44560 100644 --- a/distro/src/main/tomcat/ssl-server.xml +++ b/distro/src/main/tomcat/ssl-server.xml @@ -86,7 +86,7 @@ http://git-wip-us.apache.org/repos/asf/oozie/blob/d4d35bd1/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index ab8ac8f..bd4ce5c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2037 Add TLSv1.1,TLSv1.2 (rkanter) OOZIE-2500 -DtestJarSimple option mentioned in minioozie doc does not work (abhishekbafna via rkanter) OOZIE-2552 Update ActiveMQ version for security and other fixes (asasvari via rkanter) OOZIE-2571 Add spark.scala.binary.version Maven property so that Scala 2.11 can be used (jonathak via rkanter)
[01/50] [abbrv] oozie git commit: OOZIE-2488 Upgrade to latest OpenJPA version (abhishekbafna via jaydeepvishwakarma)
Repository: oozie Updated Branches: refs/heads/oya fea512cf6 -> aa1dd9613 OOZIE-2488 Upgrade to latest OpenJPA version (abhishekbafna via jaydeepvishwakarma) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/35db5b31 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/35db5b31 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/35db5b31 Branch: refs/heads/oya Commit: 35db5b31fa0c69680de4fe12eabc6fde818b2c54 Parents: 88fc6e5 Author: jvishwakarmaAuthored: Wed Sep 14 14:01:21 2016 +0530 Committer: jvishwakarma Committed: Wed Sep 14 14:01:21 2016 +0530 -- pom.xml | 2 +- release-log.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/35db5b31/pom.xml -- diff --git a/pom.xml b/pom.xml index 6571a1b..f0d7c9d 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ 6.0.44 - 2.2.2 + 2.4.1 2.10.0 2.5.0 1.8.8 http://git-wip-us.apache.org/repos/asf/oozie/blob/35db5b31/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 6f0b7b7..666e202 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2488 Upgrade to latest OpenJPA version (abhishekbafna via jaydeepvishwakarma) OOZIE-2648 Child jobs shouldn't send callbacks to Oozie (abhishekbafna via rkanter) OOZIE-2584 Eliminate Thread.sleep() calls in TestMemoryLocks (pbacsko via rkanter) OOZIE-2635 TimeZone.getTimeZone has performance issue. (satishsaley via rkanter)
[18/50] [abbrv] oozie git commit: OOZIE-2525 SchemaChecker fails with NPE (rkanter)
OOZIE-2525 SchemaChecker fails with NPE (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5759397a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5759397a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5759397a Branch: refs/heads/oya Commit: 5759397a939375d235fa75165ff26bd6fcca8f75 Parents: bcd23e1 Author: Robert KanterAuthored: Tue Sep 20 12:05:43 2016 -0700 Committer: Robert Kanter Committed: Tue Sep 20 12:05:43 2016 -0700 -- .../java/org/apache/oozie/service/SchemaCheckerService.java | 8 release-log.txt | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/5759397a/core/src/main/java/org/apache/oozie/service/SchemaCheckerService.java -- diff --git a/core/src/main/java/org/apache/oozie/service/SchemaCheckerService.java b/core/src/main/java/org/apache/oozie/service/SchemaCheckerService.java index 7fda9e2..1101f99 100644 --- a/core/src/main/java/org/apache/oozie/service/SchemaCheckerService.java +++ b/core/src/main/java/org/apache/oozie/service/SchemaCheckerService.java @@ -45,7 +45,7 @@ public class SchemaCheckerService implements Service, Instrumentable { int interval = ConfigurationService.getInt(CONF_INTERVAL); if (dbType.equals("derby") || dbType.equals("hsqldb") || dbType.equals("sqlserver") || interval <= 0) { LOG.debug("SchemaCheckerService is disabled: not supported for {0}", dbType); -status = "DISABLED (" + dbType + " no supported)"; +status = "DISABLED (" + dbType + " not supported)"; } else { String driver = ConfigurationService.get(JPAService.CONF_DRIVER); String user = ConfigurationService.get(JPAService.CONF_USERNAME); @@ -96,7 +96,7 @@ public class SchemaCheckerService implements Service, Instrumentable { lastCheck = time.toString(); } -static class SchemaCheckerRunnable implements Runnable { +private class SchemaCheckerRunnable implements Runnable { private String dbType; private String url; private String user; @@ -117,8 +117,8 @@ public class SchemaCheckerService implements Service, Instrumentable { Services.get().get(CallableQueueService.class).queue( new SchemaCheckXCommand(dbType, url, user, pass, ignoreExtras)); } else { -Services.get().get(SchemaCheckerService.class).status = "DISABLED (not leader in HA)"; -Services.get().get(SchemaCheckerService.class).lastCheck = "N/A"; +status = "DISABLED (not leader in HA)"; +lastCheck = "N/A"; } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/5759397a/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index cef8876..5f939e8 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2525 SchemaChecker fails with NPE (rkanter) OOZIE-2672 SLA periodic update does not remove job from map if job is removed from database (satishsaley via puru) OOZIE-2498 Oozie CallerId configuration for downstream components (abhishekbafna via rohini) OOZIE-2491 oozie acl cannot specify group,it does`t work (abhishekbafna via rohini)
[47/50] [abbrv] oozie git commit: OOZIE-2697 UGI calls for secure/non-secure clusters (temporary - needs review)
OOZIE-2697 UGI calls for secure/non-secure clusters (temporary - needs review) Change-Id: I0109d1e0d9ef7c17b43810f6f345a612e996591e Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3a8f00fa Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3a8f00fa Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3a8f00fa Branch: refs/heads/oya Commit: 3a8f00fa48862c393d12f1506b7cd69a4ad30d42 Parents: 8d2b49d Author: Peter BacskoAuthored: Fri Sep 30 17:26:27 2016 +0200 Committer: Peter Bacsko Committed: Fri Sep 30 17:26:27 2016 +0200 -- .../apache/oozie/action/hadoop/LauncherAM.java | 31 +++- 1 file changed, 30 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/3a8f00fa/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index c923dda..0570d16 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -40,7 +40,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.SaslRpcServer.AuthMethod; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -131,7 +137,30 @@ public class LauncherAM { String submitterUser = System.getProperty("submitter.user", "").trim(); Preconditions.checkArgument(!submitterUser.isEmpty(), "Submitter user is undefined"); System.out.println("Submitter user is: " + submitterUser); -UserGroupInformation ugi = UserGroupInformation.createRemoteUser(submitterUser); + +String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); + +// DEBUG - will be removed +UserGroupInformation login = UserGroupInformation.getLoginUser(); +System.out.println("Login: " + login.getUserName()); +System.out.println("SecurityEnabled:" + UserGroupInformation.isSecurityEnabled()); +System.out.println("Login keytab based:" + UserGroupInformation.isLoginKeytabBased()); +System.out.println("Login ticket based:" + UserGroupInformation.isLoginTicketBased()); +System.out.println("Login from keytab: " + login.isFromKeytab()); +System.out.println("Login has kerberos credentials: " + login.hasKerberosCredentials()); +System.out.println("Login authMethod: " + login.getAuthenticationMethod()); +System.out.println("JobUserName:" + jobUserName); + +UserGroupInformation ugi = null; + +if (UserGroupInformation.getLoginUser().getShortUserName().equals(submitterUser)) { +System.out.println("Using login user for UGI"); +ugi = UserGroupInformation.getLoginUser(); +} else { +ugi = UserGroupInformation.createRemoteUser(submitterUser); + ugi.addCredentials(UserGroupInformation.getLoginUser().getCredentials()); +} + boolean backgroundAction = false; try {
[10/50] [abbrv] oozie git commit: OOZIE-2657 Clean up redundant access modifiers from oozie interfaces
OOZIE-2657 Clean up redundant access modifiers from oozie interfaces Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/dd053ebf Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/dd053ebf Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/dd053ebf Branch: refs/heads/oya Commit: dd053ebf061c2b68249dcfaaf291b8411f0e2459 Parents: f132c9b Author: jvishwakarmaAuthored: Sun Sep 18 22:36:19 2016 +0530 Committer: jvishwakarma Committed: Sun Sep 18 22:36:19 2016 +0530 -- .../org/apache/oozie/action/ActionExecutor.java | 20 ++-- .../input/dependency/CoordInputDependency.java | 34 ++-- .../apache/oozie/workflow/lite/NodeHandler.java | 28 3 files changed, 41 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/dd053ebf/core/src/main/java/org/apache/oozie/action/ActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/ActionExecutor.java index 3f978fd..1d6456b 100644 --- a/core/src/main/java/org/apache/oozie/action/ActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/ActionExecutor.java @@ -94,28 +94,28 @@ public abstract class ActionExecutor { * @param externalStatusVar variable for the caller to inject the external status. * @return the callback URL. */ -public String getCallbackUrl(String externalStatusVar); +String getCallbackUrl(String externalStatusVar); /** * Return a proto configuration for actions with auth properties already set. * * @return a proto configuration for actions with auth properties already set. */ -public Configuration getProtoActionConf(); +Configuration getProtoActionConf(); /** * Return the workflow job. * * @return the workflow job. */ -public WorkflowJob getWorkflow(); +WorkflowJob getWorkflow(); /** * Return an ELEvaluator with the context injected. * * @return configured ELEvaluator. */ -public ELEvaluator getELEvaluator(); +ELEvaluator getELEvaluator(); /** * Set a workflow action variable. Convenience method that prefixes the variable name with the action name @@ -124,7 +124,7 @@ public abstract class ActionExecutor { * @param name variable name. * @param value variable value, null removes the variable. */ -public void setVar(String name, String value); +void setVar(String name, String value); /** * Get a workflow action variable. Convenience method that prefixes the variable name with the action name @@ -133,7 +133,7 @@ public abstract class ActionExecutor { * @param name variable name. * @return the variable value, null if not set. */ -public String getVar(String name); +String getVar(String name); /** * Set the action tracking information for an successfully started action. @@ -203,7 +203,7 @@ public abstract class ActionExecutor { * @return the path that will be used to store action specific data * @throws IOException @throws URISyntaxException @throws HadoopAccessorException */ -public Path getActionDir() throws HadoopAccessorException, IOException, URISyntaxException; +Path getActionDir() throws HadoopAccessorException, IOException, URISyntaxException; /** * @return filesystem handle for the application deployment fs. @@ -211,9 +211,9 @@ public abstract class ActionExecutor { * @throws URISyntaxException * @throws HadoopAccessorException */ -public FileSystem getAppFileSystem() throws HadoopAccessorException, IOException, URISyntaxException; +FileSystem getAppFileSystem() throws HadoopAccessorException, IOException, URISyntaxException; -public void setErrorInfo(String str, String exMsg); +void setErrorInfo(String str, String exMsg); } @@ -329,7 +329,7 @@ public abstract class ActionExecutor { /** * Register error handling information for an exception. * - * @param exClass excpetion class name (to work in case of a particular exception not being in the classpath, needed + * @param exClass exception class name (to work in case of a particular exception not being in the classpath, needed * to be able to handle multiple version of Hadoop or other JARs used by executors with the same
[44/50] [abbrv] oozie git commit: Merge branch 'apache_forked_master' into apache_forked_oya
http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java -- diff --cc sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 9b5d57d,785ca5e..338bce8 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@@ -27,12 -27,10 +27,14 @@@ import java.io.IOException import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; + import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; + import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@@ -53,11 -43,18 +55,22 @@@ import org.apache.hadoop.yarn.exception public abstract class LauncherMain { + public static final String ACTION_PREFIX = "oozie.action."; + public static final String EXTERNAL_CHILD_IDS = ACTION_PREFIX + "externalChildIDs"; + public static final String EXTERNAL_ACTION_STATS = ACTION_PREFIX + "stats.properties"; + public static final String EXTERNAL_STATS_WRITE = ACTION_PREFIX + "external.stats.write"; + public static final String OUTPUT_PROPERTIES = ACTION_PREFIX + "output.properties"; public static final String HADOOP_JOBS = "hadoopJobs"; public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags"; ++ +public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags"; +public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time"; + + public static final String TEZ_APPLICATION_TAGS = "tez.application.tags"; + public static final String SPARK_YARN_TAGS = "spark.yarn.tags"; + protected static String[] HADOOP_SITE_FILES = new String[] + {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"}; + protected static void run(Class klass, String[] args) throws Exception { LauncherMain main = klass.newInstance(); main.run(args); http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/pig/pom.xml -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/spark/pom.xml -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java -- diff --cc sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 9dbd12f,407ba4b..65daef0 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@@ -64,8 -64,10 +64,10 @@@ public class SparkMain extends Launcher protected void run(String[] args) throws Exception { boolean isPyspark = false; Configuration actionConf = loadActionConf(); + prepareHadoopConfig(actionConf); + setYarnTag(actionConf); -LauncherMainHadoopUtils.killChildYarnJobs(actionConf); +LauncherMain.killChildYarnJobs(actionConf); String logFile = setUpSparkLog4J(actionConf); List sparkArgs = new ArrayList(); http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/sqoop/pom.xml -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java -- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/streaming/pom.xml --
[33/50] [abbrv] oozie git commit: OOZIE-2594 Make MapReduce action work, small refactors, remove RunningJob from test cases, test fixes. Follow up: OOZIE-2686
OOZIE-2594 Make MapReduce action work, small refactors, remove RunningJob from test cases, test fixes. Follow up: OOZIE-2686 Change-Id: I797963d65bc248c81c9e9d0b2a48a68dd2bab5cf Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ca7e56fd Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ca7e56fd Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ca7e56fd Branch: refs/heads/oya Commit: ca7e56fdccbca80ce2f9b87812c15305ca9d09d0 Parents: 2fddebb Author: Peter BacskoAuthored: Mon Sep 12 11:29:14 2016 +0200 Committer: Peter Bacsko Committed: Mon Sep 26 14:11:28 2016 +0200 -- .../oozie/action/hadoop/JavaActionExecutor.java | 117 .../action/hadoop/MapReduceActionExecutor.java | 79 +- .../apache/oozie/servlet/CallbackServlet.java | 4 +- .../action/hadoop/TestDistCpActionExecutor.java | 29 +- .../action/hadoop/TestJavaActionExecutor.java | 12 +- .../hadoop/TestLauncherAMCallbackNotifier.java | 148 ++ .../action/hadoop/TestMapReduceActionError.java | 173 .../oozie/action/hadoop/TestOozieJobInfo.java | 19 +- .../action/hadoop/TestShellActionExecutor.java | 46 +--- .../command/wf/TestActionCheckXCommand.java | 7 +- .../command/wf/TestActionStartXCommand.java | 26 +- .../java/org/apache/oozie/test/XFsTestCase.java | 27 ++ .../java/org/apache/oozie/test/XTestCase.java | 10 +- .../action/hadoop/TestHiveActionExecutor.java | 39 +-- .../action/hadoop/TestHive2ActionExecutor.java | 40 +-- .../apache/oozie/action/hadoop/LauncherAM.java | 274 +++ .../hadoop/LauncherAMCallbackNotifier.java | 15 +- .../oozie/action/hadoop/LauncherMain.java | 8 +- .../oozie/action/hadoop/MapReduceMain.java | 1 - .../org/apache/oozie/action/hadoop/PigMain.java | 2 +- .../action/hadoop/TestPigActionExecutor.java| 58 +--- .../apache/oozie/action/hadoop/TestPyspark.java | 19 +- .../action/hadoop/TestSparkActionExecutor.java | 21 +- .../action/hadoop/TestSqoopActionExecutor.java | 51 +--- .../hadoop/TestMapReduceActionExecutor.java | 267 +- 25 files changed, 645 insertions(+), 847 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/ca7e56fd/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 6a28406..8637f64 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -18,6 +18,28 @@ package org.apache.oozie.action.hadoop; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; +import java.net.ConnectException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; @@ -31,6 +53,7 @@ import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; +import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.token.Token; @@ -48,6 +71,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.oozie.WorkflowActionBean; @@ -78,38 +102,21 @@ import org.jdom.Element; import org.jdom.JDOMException; import org.jdom.Namespace; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.StringReader; -import java.net.ConnectException; -import java.net.URI; -import java.net.URISyntaxException; -import
[19/50] [abbrv] oozie git commit: OOZIE-2572 SLA DURATION miss not shown when job is running for longer than expected time
OOZIE-2572 SLA DURATION miss not shown when job is running for longer than expected time Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f45e1eb7 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f45e1eb7 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f45e1eb7 Branch: refs/heads/oya Commit: f45e1eb7701e9f9112bc7a295700eb9949ae6ac2 Parents: 5759397 Author: Purshotam ShahAuthored: Tue Sep 20 14:05:21 2016 -0700 Committer: Purshotam Shah Committed: Tue Sep 20 14:05:21 2016 -0700 -- .../org/apache/oozie/sla/SLASummaryBean.java| 8 +++--- .../apache/oozie/servlet/TestV2SLAServlet.java | 6 ++--- docs/src/site/twiki/DG_SLAMonitoring.twiki | 26 ++-- release-log.txt | 1 + .../webapp/console/sla/js/oozie-sla-table.js| 21 +--- 5 files changed, 22 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/f45e1eb7/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java -- diff --git a/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java b/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java index 3b2cebd..cfe1522 100644 --- a/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java +++ b/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java @@ -423,7 +423,7 @@ public class SLASummaryBean implements JsonBean { } json.put(JsonTags.SLA_SUMMARY_EXPECTED_DURATION, expectedDuration); if (actualDuration == -1 && expectedDuration != -1 && actualStartTS != null) { -long currentDur = (new Date().getTime() - actualStartTS.getTime()) / (1000 * 60); +long currentDur = new Date().getTime() - actualStartTS.getTime(); json.put(JsonTags.SLA_SUMMARY_ACTUAL_DURATION, currentDur); } else { @@ -432,7 +432,7 @@ public class SLASummaryBean implements JsonBean { Long durationDelay = eventMap.get(EventStatus.DURATION_MET) != null ? eventMap.get(EventStatus.DURATION_MET) : eventMap.get(EventStatus.DURATION_MISS); if (durationDelay != null) { -json.put(JsonTags.SLA_SUMMARY_DURATION_DELAY, durationDelay); +json.put(JsonTags.SLA_SUMMARY_DURATION_DELAY, durationDelay / (1000 * 60)); } json.put(JsonTags.SLA_SUMMARY_JOB_STATUS, jobStatus); json.put(JsonTags.SLA_SUMMARY_SLA_STATUS, slaStatus); @@ -482,9 +482,9 @@ public class SLASummaryBean implements JsonBean { } else { if (actualStartTS != null) { -long currentDur = (new Date().getTime() - actualStartTS.getTime()) / (1000 * 60); +long currentDur = new Date().getTime() - actualStartTS.getTime(); if (expectedDuration < currentDur) { -events.put(EventStatus.DURATION_MISS, (currentDur - expectedDuration)); +events.put(EventStatus.DURATION_MISS, currentDur - expectedDuration); } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/f45e1eb7/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java -- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java index db509ac..03f2323 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java @@ -193,7 +193,7 @@ public class TestV2SLAServlet extends DagServletTestCase { String id = (String)json.get(JsonTags.SLA_SUMMARY_ID); if(id.equals(cjBean1.getId() + "@1")) { assertEquals(-2L, json.get(JsonTags.SLA_SUMMARY_START_DELAY)); -assertEquals(-1L, json.get(JsonTags.SLA_SUMMARY_DURATION_DELAY)); +assertEquals(0L, json.get(JsonTags.SLA_SUMMARY_DURATION_DELAY)); assertEquals(-1L, json.get(JsonTags.SLA_SUMMARY_END_DELAY)); } } @@ -232,7 +232,7 @@ public class TestV2SLAServlet extends DagServletTestCase { parentId = (String) json.get(JsonTags.SLA_SUMMARY_PARENT_ID); assertTrue(parentId.equals(cjBean1.getId())); assertEquals(1L, json.get(JsonTags.SLA_SUMMARY_START_DELAY)); -assertEquals(1L, json.get(JsonTags.SLA_SUMMARY_DURATION_DELAY)); +assertEquals(0L, json.get(JsonTags.SLA_SUMMARY_DURATION_DELAY));
[09/50] [abbrv] oozie git commit: OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter)
OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f132c9ba Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f132c9ba Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f132c9ba Branch: refs/heads/oya Commit: f132c9ba41f23d44c920d4ebe6e6c52c378ba818 Parents: d4d35bd Author: Robert KanterAuthored: Sat Sep 17 11:40:19 2016 -0700 Committer: Robert Kanter Committed: Sat Sep 17 11:40:19 2016 -0700 -- core/pom.xml | 4 ++-- examples/pom.xml | 4 ++-- pom.xml| 26 +- release-log.txt| 1 + sharelib/sqoop/pom.xml | 4 ++-- webapp/pom.xml | 2 -- 6 files changed, 24 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/f132c9ba/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index 88821c1..62ace06 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -224,8 +224,8 @@ -commons-httpclient -commons-httpclient +org.apache.httpcomponents +httpclient compile http://git-wip-us.apache.org/repos/asf/oozie/blob/f132c9ba/examples/pom.xml -- diff --git a/examples/pom.xml b/examples/pom.xml index 0452836..be4e932 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -57,8 +57,8 @@ -commons-httpclient -commons-httpclient +org.apache.httpcomponents +httpclient test http://git-wip-us.apache.org/repos/asf/oozie/blob/f132c9ba/pom.xml -- diff --git a/pom.xml b/pom.xml index de0ce82..0712d4c 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,8 @@ 1.8.8 1.2.17 5.13.3 + 4.4 + 4.4 @@ -370,8 +372,8 @@ commons-io -commons-httpclient -commons-httpclient +org.apache.httpcomponents +httpclient org.apache.commons @@ -542,8 +544,8 @@ commons-cli -commons-httpclient -commons-httpclient +org.apache.httpcomponents +httpclient commons-lang @@ -813,8 +815,8 @@ hadoop-core -commons-httpclient -commons-httpclient +org.apache.httpcomponents +httpclient org.mortbay.jetty @@ -1353,9 +1355,15 @@ -commons-httpclient -commons-httpclient -3.1 +org.apache.httpcomponents +httpclient +${httpclient.version} + + + +org.apache.httpcomponents +httpcore +${httpcore.version} http://git-wip-us.apache.org/repos/asf/oozie/blob/f132c9ba/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index bd4ce5c..247886c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter) OOZIE-2037 Add TLSv1.1,TLSv1.2 (rkanter) OOZIE-2500 -DtestJarSimple option mentioned in minioozie doc does not work (abhishekbafna via rkanter) OOZIE-2552 Update ActiveMQ version for security and other fixes (asasvari via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/f132c9ba/sharelib/sqoop/pom.xml -- diff --git a/sharelib/sqoop/pom.xml b/sharelib/sqoop/pom.xml index 5b80a4f..517a37b 100644 --- a/sharelib/sqoop/pom.xml +++ b/sharelib/sqoop/pom.xml @@ -73,8 +73,8 @@ thrift -commons-httpclient -commons-httpclient +org.apache.httpcomponents +
[46/50] [abbrv] oozie git commit: amend OOZIE-1978 testForkJoinMismatch should pass on Java8
amend OOZIE-1978 testForkJoinMismatch should pass on Java8 Change-Id: I6847a876a5be876510aca916dc8b29b75850d5b3 Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/8d2b49d4 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/8d2b49d4 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/8d2b49d4 Branch: refs/heads/oya Commit: 8d2b49d424ca5dbe9c6c3c58e0a073e1f1336671 Parents: c49f382 Author: Peter BacskoAuthored: Fri Sep 30 14:31:43 2016 +0200 Committer: Peter Bacsko Committed: Fri Sep 30 14:31:43 2016 +0200 -- .../org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/8d2b49d4/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java -- diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java index 6a9633b..0557166 100644 --- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java +++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java @@ -1378,7 +1378,7 @@ public class TestLiteWorkflowAppParser extends XTestCase { } catch (WorkflowException we) { assertEquals(ErrorCode.E0757, we.getErrorCode()); assertTrue(we.getMessage().contains("Fork node [f]")); -assertTrue(we.getMessage().contains("[j2,j1]")); +assertTrue(we.getMessage().contains("[j2,j1]") || we.getMessage().contains("[j1,j2]")); } }
[25/50] [abbrv] oozie git commit: OOZIE-2682 Oozie test-patch script is not updating Jira with jenkins test report (abhishekbafna via rkanter)
OOZIE-2682 Oozie test-patch script is not updating Jira with jenkins test report (abhishekbafna via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5c89163a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5c89163a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5c89163a Branch: refs/heads/oya Commit: 5c89163a4310fc6308d007f678d5f6935399540a Parents: 85e79ec Author: Robert KanterAuthored: Wed Sep 21 15:18:04 2016 -0700 Committer: Robert Kanter Committed: Wed Sep 21 15:18:04 2016 -0700 -- bin/test-patch | 2 +- release-log.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/5c89163a/bin/test-patch -- diff --git a/bin/test-patch b/bin/test-patch index 97f759c..d69cc59 100755 --- a/bin/test-patch +++ b/bin/test-patch @@ -108,7 +108,7 @@ updateJira() { if [[ "$JIRAPASSWORD" != "" ]] ; then JIRACLI=${TOOLSDIR}/jira-cli/jira.sh if [ ! -e "${JIRACLI}" ] ; then -curl https://bobswift.atlassian.net/wiki/download/attachments/16285777/jira-cli-2.6.0-distribution.zip > ${TEMPDIR}/jira-cli.zip +curl -L https://bobswift.atlassian.net/wiki/download/attachments/16285777/jira-cli-2.6.0-distribution.zip > ${TEMPDIR}/jira-cli.zip if [ $? != 0 ] ; then echo echo "Could not download jira-cli tool, thus no JIRA updating" http://git-wip-us.apache.org/repos/asf/oozie/blob/5c89163a/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index b503f21..c2036de 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2682 Oozie test-patch script is not updating Jira with jenkins test report (abhishekbafna via rkanter) OOZIE-2679 Decrease HttpClient library versions due to Hadoop incompatibility (abhishekbafna via rkanter) OOZIE-2661 Coordinator Action status not updated when workflow job SUSPENDED (satishsaley via puru) OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter)
[04/50] [abbrv] oozie git commit: OOZIE-2530 Allow Hive to use a different jline version (poeppt via rkanter)
OOZIE-2530 Allow Hive to use a different jline version (poeppt via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5a689c62 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5a689c62 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5a689c62 Branch: refs/heads/oya Commit: 5a689c62e64d227320e832eef4c2d96e880198aa Parents: 34c469d Author: Robert KanterAuthored: Thu Sep 15 16:47:16 2016 -0700 Committer: Robert Kanter Committed: Thu Sep 15 16:47:16 2016 -0700 -- pom.xml| 1 + release-log.txt| 1 + sharelib/hive/pom.xml | 5 + sharelib/hive2/pom.xml | 5 + 4 files changed, 12 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/5a689c62/pom.xml -- diff --git a/pom.xml b/pom.xml index f0d7c9d..52ce311 100644 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,7 @@ 0.13.1 + 0.9.94 0.16.0 1.4.3 http://git-wip-us.apache.org/repos/asf/oozie/blob/5a689c62/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 8833d35..000a9a6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2530 Allow Hive to use a different jline version (poeppt via rkanter) OOZIE-2660 Create documentation for DB Dump/Load functionality (gezapeti via rkanter) OOZIE-2659 TestPauseTransitService is flaky (gezapeti via rkanter) OOZIE-2488 Upgrade to latest OpenJPA version (abhishekbafna via jaydeepvishwakarma) http://git-wip-us.apache.org/repos/asf/oozie/blob/5a689c62/sharelib/hive/pom.xml -- diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml index 73802da..e02b447 100644 --- a/sharelib/hive/pom.xml +++ b/sharelib/hive/pom.xml @@ -154,6 +154,11 @@ oozie-hadoop-utils provided + +jline +jline +${hive.jline.version} + http://git-wip-us.apache.org/repos/asf/oozie/blob/5a689c62/sharelib/hive2/pom.xml -- diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml index d660b90..6ab72c0 100644 --- a/sharelib/hive2/pom.xml +++ b/sharelib/hive2/pom.xml @@ -131,6 +131,11 @@ commons-io compile + +jline +jline +${hive.jline.version} +
[50/50] [abbrv] oozie git commit: OOZIE-2703 adding extra warning message if MR action check fails
OOZIE-2703 adding extra warning message if MR action check fails Change-Id: I9e7941fc0acdb1cb0bc6fe4f9828202b7da26d98 Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/aa1dd961 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/aa1dd961 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/aa1dd961 Branch: refs/heads/oya Commit: aa1dd96133164a78ef8103154b50a2e1635733e4 Parents: 165847c Author: Peter BacskoAuthored: Fri Oct 7 15:15:27 2016 +0200 Committer: Peter Bacsko Committed: Fri Oct 7 15:15:27 2016 +0200 -- .../org/apache/oozie/action/hadoop/MapReduceActionExecutor.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/aa1dd961/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index f7d230f..1b975ab 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -363,6 +363,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { jobCompleted = runningJob.isComplete(); } catch (Exception e) { LOG.warn("Exception in check(). Message[{0}]", e.getMessage(), e); +LOG.warn("Unable to check the state of a running MapReduce job - please check the health of the Job History Server!"); exception = true; throw convertException(e); } finally {
[14/50] [abbrv] oozie git commit: OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)
OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1 Branch: refs/heads/oya Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73 Parents: f861071 Author: Rohini PalaniswamyAuthored: Mon Sep 19 10:07:57 2016 -0700 Committer: Rohini Palaniswamy Committed: Mon Sep 19 10:07:57 2016 -0700 -- .../oozie/action/hadoop/SparkActionExecutor.java | 6 ++ core/src/main/resources/oozie-default.xml | 11 +++ release-log.txt | 1 + .../org/apache/oozie/action/hadoop/LauncherMain.java | 14 ++ .../org/apache/oozie/action/hadoop/ShellMain.java | 13 ++--- .../org/apache/oozie/action/hadoop/SparkMain.java | 13 + 6 files changed, 47 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 97355fd..15a641b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.Services; import org.apache.oozie.service.SparkConfigurationService; @@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_CLASS = "oozie.spark.class"; public static final String SPARK_JAR = "oozie.spark.jar"; public static final String MAPRED_CHILD_ENV = "mapred.child.env"; +private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir"; public SparkActionExecutor() { super("spark"); @@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor { actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim()); } +// Setting if SparkMain should setup hadoop config *-site.xml +boolean setupHadoopConf = actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, + ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR)); +actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, setupHadoopConf); return actionConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml -- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 40a5fe5..e71ebe3 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are waiting for a long time w +oozie.action.spark.setup.hadoop.conf.dir +false + +Oozie action.xml (oozie.action.conf.xml) contains all the hadoop configuration and user provided configurations. +This property will allow users to copy Oozie action.xml as hadoop *-site configurations files. The advantage is, +user need not to manage these files into spark sharelib. If user wants to manage the hadoop configurations +themselves, it should should disable it. + + + + oozie.action.shell.setup.hadoop.conf.dir false http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 681e5ee..7851e81 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) OOZIE-2588 Support getting credentials for same cluster hcat when credentials config is
[06/50] [abbrv] oozie git commit: OOZIE-2552 Update ActiveMQ version for security and other fixes (asasvari via rkanter)
OOZIE-2552 Update ActiveMQ version for security and other fixes (asasvari via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/d431b8c1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/d431b8c1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/d431b8c1 Branch: refs/heads/oya Commit: d431b8c1afdb2d05e3bf4a16085b4d9ef5773647 Parents: e6b5c95 Author: Robert KanterAuthored: Thu Sep 15 17:19:09 2016 -0700 Committer: Robert Kanter Committed: Thu Sep 15 17:19:09 2016 -0700 -- pom.xml | 7 --- release-log.txt | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/d431b8c1/pom.xml -- diff --git a/pom.xml b/pom.xml index 759d76e..de0ce82 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,7 @@ 2.5.0 1.8.8 1.2.17 + 5.13.3 @@ -1134,19 +1135,19 @@ org.apache.activemq activemq-client -5.10.2 + ${activemq.version} org.apache.activemq activemq-broker -5.10.2 + ${activemq.version} org.apache.activemq activemq-kahadb-store -5.10.2 + ${activemq.version} http://git-wip-us.apache.org/repos/asf/oozie/blob/d431b8c1/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index a71ea12..2203634 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2552 Update ActiveMQ version for security and other fixes (asasvari via rkanter) OOZIE-2571 Add spark.scala.binary.version Maven property so that Scala 2.11 can be used (jonathak via rkanter) OOZIE-2530 Allow Hive to use a different jline version (poeppt via rkanter) OOZIE-2660 Create documentation for DB Dump/Load functionality (gezapeti via rkanter)
[13/50] [abbrv] oozie git commit: New testcase of OOZIE-2588 which was missed in the first commit
New testcase of OOZIE-2588 which was missed in the first commit Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f8610715 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f8610715 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f8610715 Branch: refs/heads/oya Commit: f86107155453d08d7e21cdfe41bc5bb3916d991b Parents: 2fd64fa Author: Rohini PalaniswamyAuthored: Mon Sep 19 09:55:13 2016 -0700 Committer: Rohini Palaniswamy Committed: Mon Sep 19 09:55:13 2016 -0700 -- .../action/hadoop/TestHCatCredentials.java | 258 +++ 1 file changed, 258 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/f8610715/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java -- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java new file mode 100644 index 000..e6d43ca --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.HashMap; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobConf; +import org.apache.oozie.service.HCatAccessorService; +import org.apache.oozie.service.ServiceException; +import org.apache.oozie.service.Services; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ HCatCredentialHelper.class, HCatCredentials.class }) +public class TestHCatCredentials { +private Services services; +private static File OOZIE_HOME_DIR = null; +private static final String TEST_HIVE_METASTORE_PRINCIPAL = "hcat/test-hcat1@oozie.example.com"; +private static final String TEST_HIVE_METASTORE_URI = "thrift://test-hcat1.com:9898"; +private static final String TEST_HIVE_METASTORE_PRINCIPAL2 = "hcat/test-hcat2@oozie.example.com"; +private static final String TEST_HIVE_METASTORE_URI2 = "thrift://test-hcat2.com:9898"; +final String HIVE_METASTORE_PRINCIPAL = "hive.principal"; +final String HIVE_METASTORE_URI = "hive.uri"; +final String HCAT_METASTORE_PRINCIPAL = "hcat.principal"; +final String HCAT_METASTORE_URI = "hcat.uri"; +private static File hiveSiteXml = null; +private static ClassLoader prevClassloader = null; + +@BeforeClass +public static void initialize() throws Exception { +OOZIE_HOME_DIR = new File(new File("").getAbsolutePath(), "test-oozie-home"); +if (!OOZIE_HOME_DIR.exists()) { +OOZIE_HOME_DIR.mkdirs(); +} +System.setProperty(Services.OOZIE_HOME_DIR, OOZIE_HOME_DIR.getAbsolutePath()); +Services.setOozieHome(); +File oozieConfDir = new File(OOZIE_HOME_DIR.getAbsolutePath(), "conf"); +oozieConfDir.mkdir(); +File hadoopConfDir = new File(oozieConfDir, "hadoop-conf"); +hadoopConfDir.mkdir(); +File actionConfDir = new File(oozieConfDir, "action-conf"); +actionConfDir.mkdir(); +
[03/50] [abbrv] oozie git commit: OOZIE-2659 TestPauseTransitService is flaky (gezapeti via rkanter)
OOZIE-2659 TestPauseTransitService is flaky (gezapeti via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bb52d657 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bb52d657 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bb52d657 Branch: refs/heads/oya Commit: bb52d657a8006edf713f56da80cf49082f196ddd Parents: 35db5b3 Author: Robert KanterAuthored: Wed Sep 14 15:36:52 2016 -0700 Committer: Robert Kanter Committed: Wed Sep 14 15:47:18 2016 -0700 -- .../java/org/apache/oozie/service/TestPauseTransitService.java | 6 -- release-log.txt| 1 + 2 files changed, 5 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/bb52d657/core/src/test/java/org/apache/oozie/service/TestPauseTransitService.java -- diff --git a/core/src/test/java/org/apache/oozie/service/TestPauseTransitService.java b/core/src/test/java/org/apache/oozie/service/TestPauseTransitService.java index 677a360..6b04291 100644 --- a/core/src/test/java/org/apache/oozie/service/TestPauseTransitService.java +++ b/core/src/test/java/org/apache/oozie/service/TestPauseTransitService.java @@ -202,7 +202,8 @@ public class TestPauseTransitService extends XDataTestCase { waitFor(10 * 1000, new Predicate() { public boolean evaluate() throws Exception { CoordinatorJobBean cJob1 = jpaService.execute(new CoordJobGetJPAExecutor(coordJobId1)); -return cJob1.getStatus() == Job.Status.PAUSED; +CoordinatorJobBean cJob2 = jpaService.execute(new CoordJobGetJPAExecutor(coordJobId2)); +return cJob1.getStatus() == Job.Status.PAUSED && cJob2.getStatus() == Job.Status.PAUSED; } }); @@ -275,7 +276,8 @@ public class TestPauseTransitService extends XDataTestCase { waitFor(10 * 1000, new Predicate() { public boolean evaluate() throws Exception { CoordinatorJobBean cJob1 = jpaService.execute(new CoordJobGetJPAExecutor(coordJobId1)); -return cJob1.getStatus() == Job.Status.RUNNING; +CoordinatorJobBean cJob2 = jpaService.execute(new CoordJobGetJPAExecutor(coordJobId2)); +return cJob1.getStatus() == Job.Status.RUNNING && cJob2.getStatus() == Job.Status.RUNNING; } }); http://git-wip-us.apache.org/repos/asf/oozie/blob/bb52d657/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 666e202..82cee6d 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2659 TestPauseTransitService is flaky (gezapeti via rkanter) OOZIE-2488 Upgrade to latest OpenJPA version (abhishekbafna via jaydeepvishwakarma) OOZIE-2648 Child jobs shouldn't send callbacks to Oozie (abhishekbafna via rkanter) OOZIE-2584 Eliminate Thread.sleep() calls in TestMemoryLocks (pbacsko via rkanter)
[17/50] [abbrv] oozie git commit: OOZIE-2672 SLA periodic update does not remove job from map if job is removed from database
OOZIE-2672 SLA periodic update does not remove job from map if job is removed from database Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bcd23e18 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bcd23e18 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bcd23e18 Branch: refs/heads/oya Commit: bcd23e18e5cf5757a8d5e365ef95f4a80bda Parents: 3ad3537 Author: Purshotam ShahAuthored: Tue Sep 20 09:49:35 2016 -0700 Committer: Purshotam Shah Committed: Tue Sep 20 09:49:35 2016 -0700 -- .../java/org/apache/oozie/sla/SLACalculatorMemory.java | 13 - release-log.txt| 1 + 2 files changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/bcd23e18/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java -- diff --git a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java index e8638a9..3522ffe 100644 --- a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java +++ b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java @@ -237,8 +237,19 @@ public class SLACalculatorMemory implements SLACalculator { } synchronized (slaCalc) { // get eventProcessed on DB for validation in HA -SLASummaryBean summaryBean = ((SLASummaryQueryExecutor) SLASummaryQueryExecutor.getInstance()).get( +SLASummaryBean summaryBean = null; +try { +summaryBean = ((SLASummaryQueryExecutor) SLASummaryQueryExecutor.getInstance()).get( SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED_LAST_MODIFIED, jobId); +} +catch (JPAExecutorException e) { +if (e.getErrorCode().equals(ErrorCode.E0604) || e.getErrorCode().equals(ErrorCode.E0605)) { +LOG.debug("job [{0}] is is not in DB, removing from Memory", jobId); +slaMap.remove(jobId); +return; +} +throw e; +} byte eventProc = summaryBean.getEventProcessed(); slaCalc.setEventProcessed(eventProc); if (eventProc >= 7) { http://git-wip-us.apache.org/repos/asf/oozie/blob/bcd23e18/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 515521c..cef8876 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2672 SLA periodic update does not remove job from map if job is removed from database (satishsaley via puru) OOZIE-2498 Oozie CallerId configuration for downstream components (abhishekbafna via rohini) OOZIE-2491 oozie acl cannot specify group,it does`t work (abhishekbafna via rohini) OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)
[02/50] [abbrv] oozie git commit: OOZIE-2660 Create documentation for DB Dump/Load functionality (gezapeti via rkanter)
OOZIE-2660 Create documentation for DB Dump/Load functionality (gezapeti via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/34c469dd Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/34c469dd Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/34c469dd Branch: refs/heads/oya Commit: 34c469dd2f7023c92bc0f5605274059c30beea0c Parents: bb52d65 Author: Robert KanterAuthored: Wed Sep 14 15:46:44 2016 -0700 Committer: Robert Kanter Committed: Wed Sep 14 15:47:18 2016 -0700 -- distro/src/main/bin/oozie-setup.sh | 13 ++--- docs/src/site/twiki/AG_Install.twiki | 92 +++ release-log.txt | 1 + 3 files changed, 77 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/34c469dd/distro/src/main/bin/oozie-setup.sh -- diff --git a/distro/src/main/bin/oozie-setup.sh b/distro/src/main/bin/oozie-setup.sh index d3a46a9..79b049b 100644 --- a/distro/src/main/bin/oozie-setup.sh +++ b/distro/src/main/bin/oozie-setup.sh @@ -27,21 +27,22 @@ function printUsage() { echo "FS_URI is the fs.default.name" echo "for hdfs uri; SHARED_LIBRARY, path to the" echo "Oozie sharelib to install, it can be a tarball" - echo "or an expanded version of it. If ommited," + echo "or an expanded version of it. If omitted," echo "the Oozie sharelib tarball from the Oozie" echo " installation directory will be used." echo " CONCURRENCY is a number of threads to be used" echo "for copy operations." echo "By default 1 thread will be used)" - echo " (action failes if sharelib is already installed" + echo " (action fails if sharelib is already installed" echo "in HDFS)" - echo " sharelib upgrade -fs FS_URI [-locallib SHARED_LIBRARY] (upgrade existing sharelib, fails if there" + echo " sharelib upgrade -fs FS_URI [-locallib SHARED_LIBRARY] ([deprecated][use create command to create new version]" + echo " upgrade existing sharelib, fails if there" echo " is no existing sharelib installed in HDFS)" echo " db create|upgrade|postupgrade -run [-sqlfile ] (create, upgrade or postupgrade oozie db with an" echo " optional sql File)" - echo " export file exports the oozie database to the specified" + echo " export exports the oozie database to the specified" echo "file in zip format" - echo " import file imports the oozie database from JSON file" + echo " import imports the oozie database from the zip file" echo " created by export" echo " (without options prints this usage information)" echo @@ -166,10 +167,8 @@ do shift ${JAVA_BIN} ${OOZIE_OPTS} -cp ${OOZIECPPATH} org.apache.oozie.tools.OozieDBCLI "${@}" elif [ "$1" = "export" ]; then - shift ${JAVA_BIN} ${OOZIE_OPTS} -cp ${OOZIECPPATH} org.apache.oozie.tools.OozieDBExportCLI "${@}" elif [ "$1" = "import" ]; then - shift ${JAVA_BIN} ${OOZIE_OPTS} -cp ${OOZIECPPATH} org.apache.oozie.tools.OozieDBImportCLI "${@}" fi exit $? http://git-wip-us.apache.org/repos/asf/oozie/blob/34c469dd/docs/src/site/twiki/AG_Install.twiki -- diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki index
[23/50] [abbrv] oozie git commit: OOZIE-2661 Coordinator Action status not updated when workflow job SUSPENDED
OOZIE-2661 Coordinator Action status not updated when workflow job SUSPENDED Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/24289edc Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/24289edc Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/24289edc Branch: refs/heads/oya Commit: 24289edc620c635c56cc75a43c614f98e63f1102 Parents: 00275b7 Author: Purshotam ShahAuthored: Wed Sep 21 09:25:09 2016 -0700 Committer: Purshotam Shah Committed: Wed Sep 21 09:25:09 2016 -0700 -- .../command/coord/CoordActionCheckXCommand.java | 8 +++- .../apache/oozie/event/TestEventGeneration.java | 19 +++ release-log.txt | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/24289edc/core/src/main/java/org/apache/oozie/command/coord/CoordActionCheckXCommand.java -- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordActionCheckXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordActionCheckXCommand.java index bdbbd24..cdad69e 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordActionCheckXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordActionCheckXCommand.java @@ -107,6 +107,12 @@ public class CoordActionCheckXCommand extends CoordinatorXCommand { // set pending to false as the status is KILLED coordAction.setPending(0); } +else if (workflowJob.getStatus() == WorkflowJob.Status.SUSPENDED) { + coordAction.setStatus(CoordinatorAction.Status.SUSPENDED); +slaStatus = Status.FAILED; +// set pending to false as the status is SUSPENDED +coordAction.setPending(0); +} else { LOG.warn("Unexpected workflow " + workflowJob.getId() + " STATUS " + workflowJob.getStatus()); coordAction.setLastModifiedTime(new Date()); @@ -150,7 +156,7 @@ public class CoordActionCheckXCommand extends CoordinatorXCommand { */ @Override public String getEntityKey() { -return actionId; +return actionId.substring(0, actionId.indexOf("@")); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/24289edc/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java -- diff --git a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java index afd3f8f..59d0420 100644 --- a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java +++ b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java @@ -314,6 +314,25 @@ public class TestEventGeneration extends XDataTestCase { assertEquals(coord.getUser(), event.getUser()); assertEquals(coord.getAppName(), event.getAppName()); +// Action Suspended +wfJob.setStatus(WorkflowJob.Status.SUSPENDED); +action.setStatus(CoordinatorAction.Status.RUNNING); + CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, +action); + WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfJob); +new CoordActionCheckXCommand(action.getId(), 0).call(); +action = jpaService.execute(coordGetCmd); +assertEquals(CoordinatorAction.Status.SUSPENDED, action.getStatus()); +event = (JobEvent) queue.poll(); +assertEquals(EventStatus.SUSPEND, event.getEventStatus()); +assertEquals(AppType.COORDINATOR_ACTION, event.getAppType()); +assertEquals(action.getId(), event.getId()); +assertEquals(action.getJobId(), event.getParentId()); +assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime()); +assertEquals(wfJob.getStartTime(), event.getStartTime()); +assertEquals(coord.getUser(), event.getUser()); +assertEquals(coord.getAppName(), event.getAppName()); + // Action start on Coord Resume coord.setStatus(CoordinatorJobBean.Status.SUSPENDED); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS, coord); http://git-wip-us.apache.org/repos/asf/oozie/blob/24289edc/release-log.txt -- diff --git a/release-log.txt
[07/50] [abbrv] oozie git commit: OOZIE-2500 -DtestJarSimple option mentioned in minioozie doc does not work (abhishekbafna via rkanter)
OOZIE-2500 -DtestJarSimple option mentioned in minioozie doc does not work (abhishekbafna via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/65f59ff0 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/65f59ff0 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/65f59ff0 Branch: refs/heads/oya Commit: 65f59ff0ecbc13e84b88d2debd45227f8296cd4a Parents: d431b8c Author: Robert KanterAuthored: Thu Sep 15 17:29:10 2016 -0700 Committer: Robert Kanter Committed: Thu Sep 15 17:29:10 2016 -0700 -- core/pom.xml| 11 +++ release-log.txt | 1 + 2 files changed, 12 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/65f59ff0/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index f19d83b..88821c1 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -662,6 +662,17 @@ **/XFsTestCase.class **/MiniOozieTestCase.class **/XTestCase$*.class + **/MainTestCase.class + **/ActionExecutorTestCase*.class + **/XHCatTestCase.class + **/MapperReducerForTest.class + **/MapperReducerCredentialsForTest.class + **/MapperReducerUberJarForTest.class + **/SharelibUtils.class + **/MiniHCatServer.class + **/LauncherMainTester.class + **/ZKXTestCase*.class + **/XDataTestCase.class http://git-wip-us.apache.org/repos/asf/oozie/blob/65f59ff0/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index 2203634..ab8ac8f 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2500 -DtestJarSimple option mentioned in minioozie doc does not work (abhishekbafna via rkanter) OOZIE-2552 Update ActiveMQ version for security and other fixes (asasvari via rkanter) OOZIE-2571 Add spark.scala.binary.version Maven property so that Scala 2.11 can be used (jonathak via rkanter) OOZIE-2530 Allow Hive to use a different jline version (poeppt via rkanter)
[oozie] Git Push Summary [forced push!] [Forced Update!]
Repository: oozie Updated Branches: refs/heads/oya f921bb7ae -> fea512cf6 (forced update)
oozie git commit: OOZIE-2591 OYA: Java Action (pbacsko and gezapeti)
Repository: oozie Updated Branches: refs/heads/oya fea512cf6 -> f921bb7ae OOZIE-2591 OYA: Java Action (pbacsko and gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f921bb7a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f921bb7a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f921bb7a Branch: refs/heads/oya Commit: f921bb7ae4d523d392d2c69d3d80b6be75121e4c Parents: fea512c Author: Peter CsehAuthored: Mon Oct 10 13:17:42 2016 +0200 Committer: Peter Cseh Committed: Mon Oct 10 13:17:42 2016 +0200 -- .../java/org/apache/oozie/WorkflowJobBean.java | 3 +- .../apache/oozie/action/hadoop/Credentials.java | 1 - .../action/hadoop/CredentialsProvider.java | 6 +- .../action/hadoop/DistcpActionExecutor.java | 9 +- .../oozie/action/hadoop/FsActionExecutor.java | 6 +- .../oozie/action/hadoop/FsELFunctions.java | 6 +- .../action/hadoop/HCatCredentialHelper.java | 3 +- .../oozie/action/hadoop/HadoopELFunctions.java | 2 +- .../oozie/action/hadoop/HbaseCredentials.java | 7 +- .../action/hadoop/Hive2ActionExecutor.java | 4 +- .../oozie/action/hadoop/HiveActionExecutor.java | 4 +- .../oozie/action/hadoop/JavaActionExecutor.java | 16 +- .../action/hadoop/LauncherMapperHelper.java | 4 +- .../action/hadoop/MapReduceActionExecutor.java | 26 +-- .../oozie/action/hadoop/OozieJobInfo.java | 2 - .../oozie/action/hadoop/PigActionExecutor.java | 23 +-- .../hadoop/ScriptLanguageActionExecutor.java| 3 +- .../action/hadoop/ShellActionExecutor.java | 5 +- .../action/hadoop/SparkActionExecutor.java | 4 +- .../action/hadoop/SqoopActionExecutor.java | 6 +- .../org/apache/oozie/service/JPAService.java| 6 + .../java/org/apache/oozie/service/Services.java | 9 +- .../apache/oozie/service/ShareLibService.java | 18 +- .../action/hadoop/TestJavaActionExecutor.java | 187 +-- .../oozie/action/hadoop/TestLauncher.java | 22 ++- .../command/wf/TestActionCheckXCommand.java | 17 +- .../oozie/service/TestShareLibService.java | 12 +- .../java/org/apache/oozie/test/XTestCase.java | 74 +++- .../apache/oozie/action/hadoop/JavaMain.java| 4 +- .../apache/oozie/action/hadoop/LauncherAM.java | 72 --- 30 files changed, 264 insertions(+), 297 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/f921bb7a/core/src/main/java/org/apache/oozie/WorkflowJobBean.java -- diff --git a/core/src/main/java/org/apache/oozie/WorkflowJobBean.java b/core/src/main/java/org/apache/oozie/WorkflowJobBean.java index 55d79a5..f2f79dc 100644 --- a/core/src/main/java/org/apache/oozie/WorkflowJobBean.java +++ b/core/src/main/java/org/apache/oozie/WorkflowJobBean.java @@ -462,7 +462,6 @@ public class WorkflowJobBean implements Writable, WorkflowJob, JsonBean { return pInstance; } -@SuppressWarnings("unchecked") public JSONObject toJSONObject() { return toJSONObject("GMT"); } @@ -605,7 +604,7 @@ public class WorkflowJobBean implements Writable, WorkflowJob, JsonBean { @SuppressWarnings("unchecked") public List getActions() { -return (List) actions; +return (List)(List) actions; } public void setActions(List nodes) { http://git-wip-us.apache.org/repos/asf/oozie/blob/f921bb7a/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java index eadb47b..728d626 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java @@ -21,7 +21,6 @@ package org.apache.oozie.action.hadoop; import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.action.ActionExecutor.Context; -@SuppressWarnings("deprecation") public abstract class Credentials { /** http://git-wip-us.apache.org/repos/asf/oozie/blob/f921bb7a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java index 6fe22fb..9cc1c28 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java @@ -18,15 +18,13 @@ package