[2/2] oozie git commit: OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti)
OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/9cb4bd05
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/9cb4bd05
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/9cb4bd05
Branch: refs/heads/master
Commit: 9cb4bd05aa4a86d2193efe5f7bb9ef53f8fc33d2
Parents: d135b88
Author: Gezapeti Cseh
Authored: Wed Sep 20 12:17:27 2017 +0200
Committer: Gezapeti Cseh
Committed: Wed Sep 20 12:17:27 2017 +0200
--
.../main/java/org/apache/oozie/ErrorCode.java | 3 +
.../action/hadoop/CredentialsProperties.java| 5 +
.../hadoop/CredentialsProviderFactory.java | 17 +-
.../action/hadoop/HCatCredentialHelper.java | 5 +-
.../oozie/action/hadoop/HCatCredentials.java| 2 +-
.../oozie/action/hadoop/HDFSCredentials.java| 61 +++
.../oozie/action/hadoop/HadoopTokenHelper.java | 85 +
.../oozie/action/hadoop/HbaseCredentials.java | 4 +-
.../oozie/action/hadoop/Hive2Credentials.java | 14 +-
.../oozie/action/hadoop/JHSCredentials.java | 119 ++
.../oozie/action/hadoop/JavaActionExecutor.java | 203 +++---
.../oozie/action/hadoop/YarnRMCredentials.java | 67
.../oozie/service/HadoopAccessorService.java| 91 +
.../action/hadoop/TestHadoopTokenHelper.java| 44 +++
.../action/hadoop/TestJavaActionExecutor.java | 1 -
.../oozie/action/hadoop/TestShellMain.java | 2 +-
.../wf/TestWorkflowActionKillXCommand.java | 2 +-
.../service/TestHadoopAccessorService.java | 31 --
release-log.txt | 1 +
.../action/hadoop/AMRMClientAsyncFactory.java | 4 +-
.../oozie/action/hadoop/HdfsOperations.java | 84 ++---
.../apache/oozie/action/hadoop/LauncherAM.java | 367 ++-
.../apache/oozie/action/hadoop/ShellMain.java | 1 -
.../oozie/action/hadoop/TestHdfsOperations.java | 11 -
.../oozie/action/hadoop/TestLauncherAM.java | 36 +-
25 files changed, 728 insertions(+), 532 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/core/src/main/java/org/apache/oozie/ErrorCode.java
--
diff --git a/core/src/main/java/org/apache/oozie/ErrorCode.java
b/core/src/main/java/org/apache/oozie/ErrorCode.java
index 662e1ed..168c4fa 100644
--- a/core/src/main/java/org/apache/oozie/ErrorCode.java
+++ b/core/src/main/java/org/apache/oozie/ErrorCode.java
@@ -89,6 +89,9 @@ public enum ErrorCode {
E0508(XLog.OPS, "User [{0}] not authorized for WF job [{1}]"),
E0509(XLog.OPS, "User [{0}] not authorized for Coord job [{1}]"),
E0510(XLog.OPS, "Unable to get Credential [{0}]"),
+E0511(XLog.STD, "No HDFS delegation token present, can''t set credentials.
[serverPrincipal={0}]"),
+E0512(XLog.STD, "Could not get RM delegation token: {0}"),
+E0513(XLog.STD, "No YARN renewer present, can''t get token.
[servicePrincipal={0}]"),
E0550(XLog.OPS, "Could not normalize host name [{0}], {1}"),
E0551(XLog.OPS, "Missing [{0}] property"),
http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java
--
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java
b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java
index 20f93ce..3dea787 100644
---
a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java
+++
b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java
@@ -84,4 +84,9 @@ public class CredentialsProperties {
public void setProperties(HashMap properties) {
this.properties = properties;
}
+
+@Override
+public String toString() {
+return String.format("name=%s, type=%s", name, type);
+}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
--
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
index 5ca8d3e..a353e15 100644
---
a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
+++
b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
@@ -23,15 +23,20 @@ import java.util.HashMap;
import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.T
[1/2] oozie git commit: OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti)
Repository: oozie
Updated Branches:
refs/heads/master d135b88ce -> 9cb4bd05a
http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
--
diff --git
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
index 3e6ffc9..ee1a32a 100644
---
a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
+++
b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
@@ -26,15 +26,22 @@ import java.lang.reflect.Method;
import java.security.Permission;
import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -44,6 +51,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
public class LauncherAM {
public static final String OOZIE_ACTION_CONF_XML = "oozie.action.conf.xml";
@@ -80,10 +88,9 @@ public class LauncherAM {
public static final String LAUNCHER_JOB_CONF_XML = "launcher.xml";
public static final String ACTION_CONF_XML = "action.xml";
public static final String ACTION_DATA_FINAL_STATUS = "final.status";
+public static final String OOZIE_SUBMITTER_USER = "oozie.submitter.user";
-private final UserGroupInformation ugi;
-private final AMRMCallBackHandler callbackHandler;
-private final AMRMClientAsyncFactory amRmClientAsyncFactory;
+private final AMRMClientAsyncFactory amrmClientAsyncFactory;
private final HdfsOperations hdfsOperations;
private final LocalFsOperations localFsOperations;
private final PrepareActionsHandler prepareHandler;
@@ -91,24 +98,25 @@ public class LauncherAM {
private final LauncherSecurityManager launcherSecurityManager;
private final ContainerId containerId;
-private Configuration launcherJobConf;
+private final Configuration launcherConf;
+private final AMRMCallBackHandler amrmCallBackHandler;
private AMRMClientAsync amRmClientAsync;
private Path actionDir;
private Map actionData = new HashMap();
-public LauncherAM(UserGroupInformation ugi,
-AMRMClientAsyncFactory amRmClientAsyncFactory,
-AMRMCallBackHandler callbackHandler,
+public LauncherAM(
+AMRMClientAsyncFactory amrmClientAsyncFactory,
+AMRMCallBackHandler amrmCallBackHandler,
HdfsOperations hdfsOperations,
LocalFsOperations localFsOperations,
PrepareActionsHandler prepareHandler,
LauncherAMCallbackNotifierFactory callbackNotifierFactory,
LauncherSecurityManager launcherSecurityManager,
-String containerId) {
-this.ugi = Preconditions.checkNotNull(ugi, "ugi should not be null");
-this.amRmClientAsyncFactory =
Preconditions.checkNotNull(amRmClientAsyncFactory,
-"amRmClientAsyncFactory should not be null");
-this.callbackHandler = Preconditions.checkNotNull(callbackHandler,
"callbackHandler should not be null");
+String containerId,
+Configuration launcherConf) {
+this.amrmClientAsyncFactory =
Preconditions.checkNotNull(amrmClientAsyncFactory,
+"amrmClientAsyncFactory should not be null");
+this.amrmCallBackHandler =
Preconditions.checkNotNull(amrmCallBackHandler, "amrmCallBackHandler should not
be null");
this.hdfsOperations = Preconditions.checkNotNull(hdfsOperations,
"hdfsOperations should not be null");
this.localFsOperations = Preconditions.checkNotNull(localFsOperations,
"localFsOperations should not be null");
this.prepareHandler = Preconditions.checkNotNull(prepareHandler,
"prepareHandler should not be null");
@@ -117,70 +125,107 @@ public class LauncherAM {
this.launcherSecurityManager =
Preconditions.checkNotNull(
oozie git commit: OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley)
Repository: oozie Updated Branches: refs/heads/master 9cb4bd05a -> 5e4d28843 OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5e4d2884 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5e4d2884 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5e4d2884 Branch: refs/heads/master Commit: 5e4d28843c64671a1308a4484f8d68c3e2afccbb Parents: 9cb4bd0 Author: satishsaley Authored: Wed Sep 20 09:47:46 2017 -0700 Committer: satishsaley Committed: Wed Sep 20 09:47:46 2017 -0700 -- client/src/main/resources/oozie-coordinator-0.5.xsd | 1 + docs/src/site/twiki/CoordinatorFunctionalSpec.twiki | 2 ++ release-log.txt | 1 + 3 files changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/5e4d2884/client/src/main/resources/oozie-coordinator-0.5.xsd -- diff --git a/client/src/main/resources/oozie-coordinator-0.5.xsd b/client/src/main/resources/oozie-coordinator-0.5.xsd index 2b63629..80909ad 100644 --- a/client/src/main/resources/oozie-coordinator-0.5.xsd +++ b/client/src/main/resources/oozie-coordinator-0.5.xsd @@ -163,6 +163,7 @@ + http://git-wip-us.apache.org/repos/asf/oozie/blob/5e4d2884/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki -- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 4e6ba81..69e1b33 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -4240,6 +4240,7 @@ the notification: + @@ -4365,6 +4366,7 @@ the notification: + http://git-wip-us.apache.org/repos/asf/oozie/blob/5e4d2884/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index bacd686..0d6deb4 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley) OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti) OOZIE-2687 Create XML schema for launcher configurations (asasvari) OOZIE-3041 TestWorkflowActionRetryInfoXCommand fails in oozie core module (andras.piros via gezapeti)
oozie git commit: OOZIE-3048 Check El Functions for the coordinator action (satishsaley)
Repository: oozie
Updated Branches:
refs/heads/master 5e4d28843 -> a5f9aa54a
OOZIE-3048 Check El Functions for the coordinator action (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a5f9aa54
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a5f9aa54
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a5f9aa54
Branch: refs/heads/master
Commit: a5f9aa54a553bb5d981538c321ace2cefe0e5e28
Parents: 5e4d288
Author: satishsaley
Authored: Wed Sep 20 09:53:50 2017 -0700
Committer: satishsaley
Committed: Wed Sep 20 09:53:50 2017 -0700
--
.../coord/CoordActionInputCheckXCommand.java| 38 +---
.../coord/CoordPushDependencyCheckXCommand.java | 6 ++
.../apache/oozie/coord/CoordELFunctions.java| 2 +-
.../org/apache/oozie/coord/ElException.java | 32 ++
.../TestCoordActionInputCheckXCommand.java | 35 +++
.../TestCoordPushDependencyCheckXCommand.java | 38
.../coord-hcatinput-invalid-elfunction.xml | 62 +++
.../coord-hdfsinput-invalid-elfunction.xml | 64
release-log.txt | 1 +
9 files changed, 270 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/oozie/blob/a5f9aa54/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
--
diff --git
a/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
b/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
index 650e7f8..401b2c7 100644
---
a/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
+++
b/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
@@ -34,6 +34,7 @@ import org.apache.oozie.command.CommandException;
import org.apache.oozie.command.PreconditionException;
import org.apache.oozie.coord.CoordELEvaluator;
import org.apache.oozie.coord.CoordELFunctions;
+import org.apache.oozie.coord.ElException;
import org.apache.oozie.coord.input.dependency.CoordInputDependency;
import org.apache.oozie.executor.jpa.CoordActionGetForInputCheckJPAExecutor;
import org.apache.oozie.executor.jpa.CoordActionQueryExecutor;
@@ -182,13 +183,7 @@ public class CoordActionInputCheckXCommand extends
CoordinatorXCommand {
isChangeInDependency = isChangeInDependency(nonExistList,
missingDependencies, nonResolvedList, status);
if (status && isPushDependenciesMet) {
-String newActionXml = resolveCoordConfiguration(actionXml,
actionConf, actionId,
-coordPullInputDependency, coordPushInputDependency);
-actionXml.replace(0, actionXml.length(), newActionXml);
-coordAction.setActionXml(actionXml.toString());
-coordAction.setStatus(CoordinatorAction.Status.READY);
-updateCoordAction(coordAction, true);
-new CoordActionReadyXCommand(coordAction.getJobId()).call();
+moveCoordActionToReady(actionXml, actionConf,
coordPullInputDependency, coordPushInputDependency);
}
else if (!isTimeout(currentTime)) {
if (!status) {
@@ -429,6 +424,35 @@ public class CoordActionInputCheckXCommand extends
CoordinatorXCommand {
}
/**
+ * Resolves coordinator configuration and moves CoordAction to READY state
+ *
+ * @param actionXml
+ * @param actionConf
+ * @param coordPullInputDependency
+ * @param coordPushInputDependency
+ * @throws Exception
+ */
+private void moveCoordActionToReady(StringBuilder actionXml, Configuration
actionConf,
+CoordInputDependency coordPullInputDependency,
CoordInputDependency coordPushInputDependency)
+throws Exception {
+String newActionXml = null;
+try {
+newActionXml = resolveCoordConfiguration(actionXml, actionConf,
actionId, coordPullInputDependency,
+coordPushInputDependency);
+}
+catch (ElException e) {
+coordAction.setStatus(CoordinatorAction.Status.FAILED);
+updateCoordAction(coordAction, true);
+throw e;
+}
+actionXml.replace(0, actionXml.length(), newActionXml);
+coordAction.setActionXml(actionXml.toString());
+coordAction.setStatus(CoordinatorAction.Status.READY);
+updateCoordAction(coordAction, true);
+new CoordActionReadyXCommand(coordAction.getJobId()).call();
+}
+
+/**
* getting the error code of the coord action. (used mainly for unit
testing)
*/
protected String getCoordActionErrorCode() {
http://git-wi
