[2/2] oozie git commit: OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti)
OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/9cb4bd05 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/9cb4bd05 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/9cb4bd05 Branch: refs/heads/master Commit: 9cb4bd05aa4a86d2193efe5f7bb9ef53f8fc33d2 Parents: d135b88 Author: Gezapeti Cseh Authored: Wed Sep 20 12:17:27 2017 +0200 Committer: Gezapeti Cseh Committed: Wed Sep 20 12:17:27 2017 +0200 -- .../main/java/org/apache/oozie/ErrorCode.java | 3 + .../action/hadoop/CredentialsProperties.java| 5 + .../hadoop/CredentialsProviderFactory.java | 17 +- .../action/hadoop/HCatCredentialHelper.java | 5 +- .../oozie/action/hadoop/HCatCredentials.java| 2 +- .../oozie/action/hadoop/HDFSCredentials.java| 61 +++ .../oozie/action/hadoop/HadoopTokenHelper.java | 85 + .../oozie/action/hadoop/HbaseCredentials.java | 4 +- .../oozie/action/hadoop/Hive2Credentials.java | 14 +- .../oozie/action/hadoop/JHSCredentials.java | 119 ++ .../oozie/action/hadoop/JavaActionExecutor.java | 203 +++--- .../oozie/action/hadoop/YarnRMCredentials.java | 67 .../oozie/service/HadoopAccessorService.java| 91 + .../action/hadoop/TestHadoopTokenHelper.java| 44 +++ .../action/hadoop/TestJavaActionExecutor.java | 1 - .../oozie/action/hadoop/TestShellMain.java | 2 +- .../wf/TestWorkflowActionKillXCommand.java | 2 +- .../service/TestHadoopAccessorService.java | 31 -- release-log.txt | 1 + .../action/hadoop/AMRMClientAsyncFactory.java | 4 +- .../oozie/action/hadoop/HdfsOperations.java | 84 ++--- .../apache/oozie/action/hadoop/LauncherAM.java | 367 ++- .../apache/oozie/action/hadoop/ShellMain.java | 1 - .../oozie/action/hadoop/TestHdfsOperations.java | 11 - .../oozie/action/hadoop/TestLauncherAM.java | 36 +- 25 files changed, 728 insertions(+), 532 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/core/src/main/java/org/apache/oozie/ErrorCode.java -- diff --git a/core/src/main/java/org/apache/oozie/ErrorCode.java b/core/src/main/java/org/apache/oozie/ErrorCode.java index 662e1ed..168c4fa 100644 --- a/core/src/main/java/org/apache/oozie/ErrorCode.java +++ b/core/src/main/java/org/apache/oozie/ErrorCode.java @@ -89,6 +89,9 @@ public enum ErrorCode { E0508(XLog.OPS, "User [{0}] not authorized for WF job [{1}]"), E0509(XLog.OPS, "User [{0}] not authorized for Coord job [{1}]"), E0510(XLog.OPS, "Unable to get Credential [{0}]"), +E0511(XLog.STD, "No HDFS delegation token present, can''t set credentials. [serverPrincipal={0}]"), +E0512(XLog.STD, "Could not get RM delegation token: {0}"), +E0513(XLog.STD, "No YARN renewer present, can''t get token. [servicePrincipal={0}]"), E0550(XLog.OPS, "Could not normalize host name [{0}], {1}"), E0551(XLog.OPS, "Missing [{0}] property"), http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java index 20f93ce..3dea787 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProperties.java @@ -84,4 +84,9 @@ public class CredentialsProperties { public void setProperties(HashMap properties) { this.properties = properties; } + +@Override +public String toString() { +return String.format("name=%s, type=%s", name, type); +} } http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java -- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java index 5ca8d3e..a353e15 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java @@ -23,15 +23,20 @@ import java.util.HashMap; import java.util.Map; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.T
[1/2] oozie git commit: OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti)
Repository: oozie Updated Branches: refs/heads/master d135b88ce -> 9cb4bd05a http://git-wip-us.apache.org/repos/asf/oozie/blob/9cb4bd05/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java -- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 3e6ffc9..ee1a32a 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -26,15 +26,22 @@ import java.lang.reflect.Method; import java.security.Permission; import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; +import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.StringTokenizer; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -44,6 +51,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; public class LauncherAM { public static final String OOZIE_ACTION_CONF_XML = "oozie.action.conf.xml"; @@ -80,10 +88,9 @@ public class LauncherAM { public static final String LAUNCHER_JOB_CONF_XML = "launcher.xml"; public static final String ACTION_CONF_XML = "action.xml"; public static final String ACTION_DATA_FINAL_STATUS = "final.status"; +public static final String OOZIE_SUBMITTER_USER = "oozie.submitter.user"; -private final UserGroupInformation ugi; -private final AMRMCallBackHandler callbackHandler; -private final AMRMClientAsyncFactory amRmClientAsyncFactory; +private final AMRMClientAsyncFactory amrmClientAsyncFactory; private final HdfsOperations hdfsOperations; private final LocalFsOperations localFsOperations; private final PrepareActionsHandler prepareHandler; @@ -91,24 +98,25 @@ public class LauncherAM { private final LauncherSecurityManager launcherSecurityManager; private final ContainerId containerId; -private Configuration launcherJobConf; +private final Configuration launcherConf; +private final AMRMCallBackHandler amrmCallBackHandler; private AMRMClientAsync amRmClientAsync; private Path actionDir; private Map actionData = new HashMap(); -public LauncherAM(UserGroupInformation ugi, -AMRMClientAsyncFactory amRmClientAsyncFactory, -AMRMCallBackHandler callbackHandler, +public LauncherAM( +AMRMClientAsyncFactory amrmClientAsyncFactory, +AMRMCallBackHandler amrmCallBackHandler, HdfsOperations hdfsOperations, LocalFsOperations localFsOperations, PrepareActionsHandler prepareHandler, LauncherAMCallbackNotifierFactory callbackNotifierFactory, LauncherSecurityManager launcherSecurityManager, -String containerId) { -this.ugi = Preconditions.checkNotNull(ugi, "ugi should not be null"); -this.amRmClientAsyncFactory = Preconditions.checkNotNull(amRmClientAsyncFactory, -"amRmClientAsyncFactory should not be null"); -this.callbackHandler = Preconditions.checkNotNull(callbackHandler, "callbackHandler should not be null"); +String containerId, +Configuration launcherConf) { +this.amrmClientAsyncFactory = Preconditions.checkNotNull(amrmClientAsyncFactory, +"amrmClientAsyncFactory should not be null"); +this.amrmCallBackHandler = Preconditions.checkNotNull(amrmCallBackHandler, "amrmCallBackHandler should not be null"); this.hdfsOperations = Preconditions.checkNotNull(hdfsOperations, "hdfsOperations should not be null"); this.localFsOperations = Preconditions.checkNotNull(localFsOperations, "localFsOperations should not be null"); this.prepareHandler = Preconditions.checkNotNull(prepareHandler, "prepareHandler should not be null"); @@ -117,70 +125,107 @@ public class LauncherAM { this.launcherSecurityManager = Preconditions.checkNotNull(
oozie git commit: OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley)
Repository: oozie Updated Branches: refs/heads/master 9cb4bd05a -> 5e4d28843 OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5e4d2884 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5e4d2884 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5e4d2884 Branch: refs/heads/master Commit: 5e4d28843c64671a1308a4484f8d68c3e2afccbb Parents: 9cb4bd0 Author: satishsaley Authored: Wed Sep 20 09:47:46 2017 -0700 Committer: satishsaley Committed: Wed Sep 20 09:47:46 2017 -0700 -- client/src/main/resources/oozie-coordinator-0.5.xsd | 1 + docs/src/site/twiki/CoordinatorFunctionalSpec.twiki | 2 ++ release-log.txt | 1 + 3 files changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/5e4d2884/client/src/main/resources/oozie-coordinator-0.5.xsd -- diff --git a/client/src/main/resources/oozie-coordinator-0.5.xsd b/client/src/main/resources/oozie-coordinator-0.5.xsd index 2b63629..80909ad 100644 --- a/client/src/main/resources/oozie-coordinator-0.5.xsd +++ b/client/src/main/resources/oozie-coordinator-0.5.xsd @@ -163,6 +163,7 @@ + http://git-wip-us.apache.org/repos/asf/oozie/blob/5e4d2884/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki -- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 4e6ba81..69e1b33 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -4240,6 +4240,7 @@ the notification: + @@ -4365,6 +4366,7 @@ the notification: + http://git-wip-us.apache.org/repos/asf/oozie/blob/5e4d2884/release-log.txt -- diff --git a/release-log.txt b/release-log.txt index bacd686..0d6deb4 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley) OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti) OOZIE-2687 Create XML schema for launcher configurations (asasvari) OOZIE-3041 TestWorkflowActionRetryInfoXCommand fails in oozie core module (andras.piros via gezapeti)
oozie git commit: OOZIE-3048 Check El Functions for the coordinator action (satishsaley)
Repository: oozie Updated Branches: refs/heads/master 5e4d28843 -> a5f9aa54a OOZIE-3048 Check El Functions for the coordinator action (satishsaley) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a5f9aa54 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a5f9aa54 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a5f9aa54 Branch: refs/heads/master Commit: a5f9aa54a553bb5d981538c321ace2cefe0e5e28 Parents: 5e4d288 Author: satishsaley Authored: Wed Sep 20 09:53:50 2017 -0700 Committer: satishsaley Committed: Wed Sep 20 09:53:50 2017 -0700 -- .../coord/CoordActionInputCheckXCommand.java| 38 +--- .../coord/CoordPushDependencyCheckXCommand.java | 6 ++ .../apache/oozie/coord/CoordELFunctions.java| 2 +- .../org/apache/oozie/coord/ElException.java | 32 ++ .../TestCoordActionInputCheckXCommand.java | 35 +++ .../TestCoordPushDependencyCheckXCommand.java | 38 .../coord-hcatinput-invalid-elfunction.xml | 62 +++ .../coord-hdfsinput-invalid-elfunction.xml | 64 release-log.txt | 1 + 9 files changed, 270 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/oozie/blob/a5f9aa54/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java -- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java index 650e7f8..401b2c7 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java @@ -34,6 +34,7 @@ import org.apache.oozie.command.CommandException; import org.apache.oozie.command.PreconditionException; import org.apache.oozie.coord.CoordELEvaluator; import org.apache.oozie.coord.CoordELFunctions; +import org.apache.oozie.coord.ElException; import org.apache.oozie.coord.input.dependency.CoordInputDependency; import org.apache.oozie.executor.jpa.CoordActionGetForInputCheckJPAExecutor; import org.apache.oozie.executor.jpa.CoordActionQueryExecutor; @@ -182,13 +183,7 @@ public class CoordActionInputCheckXCommand extends CoordinatorXCommand { isChangeInDependency = isChangeInDependency(nonExistList, missingDependencies, nonResolvedList, status); if (status && isPushDependenciesMet) { -String newActionXml = resolveCoordConfiguration(actionXml, actionConf, actionId, -coordPullInputDependency, coordPushInputDependency); -actionXml.replace(0, actionXml.length(), newActionXml); -coordAction.setActionXml(actionXml.toString()); -coordAction.setStatus(CoordinatorAction.Status.READY); -updateCoordAction(coordAction, true); -new CoordActionReadyXCommand(coordAction.getJobId()).call(); +moveCoordActionToReady(actionXml, actionConf, coordPullInputDependency, coordPushInputDependency); } else if (!isTimeout(currentTime)) { if (!status) { @@ -429,6 +424,35 @@ public class CoordActionInputCheckXCommand extends CoordinatorXCommand { } /** + * Resolves coordinator configuration and moves CoordAction to READY state + * + * @param actionXml + * @param actionConf + * @param coordPullInputDependency + * @param coordPushInputDependency + * @throws Exception + */ +private void moveCoordActionToReady(StringBuilder actionXml, Configuration actionConf, +CoordInputDependency coordPullInputDependency, CoordInputDependency coordPushInputDependency) +throws Exception { +String newActionXml = null; +try { +newActionXml = resolveCoordConfiguration(actionXml, actionConf, actionId, coordPullInputDependency, +coordPushInputDependency); +} +catch (ElException e) { +coordAction.setStatus(CoordinatorAction.Status.FAILED); +updateCoordAction(coordAction, true); +throw e; +} +actionXml.replace(0, actionXml.length(), newActionXml); +coordAction.setActionXml(actionXml.toString()); +coordAction.setStatus(CoordinatorAction.Status.READY); +updateCoordAction(coordAction, true); +new CoordActionReadyXCommand(coordAction.getJobId()).call(); +} + +/** * getting the error code of the coord action. (used mainly for unit testing) */ protected String getCoordActionErrorCode() { http://git-wi