http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index a110f10..2304501c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -37,6 +37,8 @@ import java.io.FileReader; import java.io.IOException; import java.io.LineNumberReader; import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -88,12 +90,11 @@ public class TestLinuxContainerExecutorWithMocks { private static final Logger LOG = LoggerFactory.getLogger(TestLinuxContainerExecutorWithMocks.class); - private static final String MOCK_EXECUTOR = - "./src/test/resources/mock-container-executor"; + private static final String MOCK_EXECUTOR = "mock-container-executor"; private static final String MOCK_EXECUTOR_WITH_ERROR = - "./src/test/resources/mock-container-executer-with-error"; + "mock-container-executer-with-error"; private static final String MOCK_EXECUTOR_WITH_CONFIG_ERROR = - "./src/test/resources/mock-container-executer-with-configuration-error"; + "mock-container-executer-with-configuration-error"; private String tmpMockExecutor; private LinuxContainerExecutor mockExec = null; @@ -121,11 +122,13 @@ public class TestLinuxContainerExecutorWithMocks { return ret; } - private void setupMockExecutor(String executorPath, Configuration conf) - throws IOException { + private void setupMockExecutor(String executorName, Configuration conf) + throws IOException, URISyntaxException { //we'll always use the tmpMockExecutor - since // PrivilegedOperationExecutor can only be initialized once. + URI executorPath = getClass().getClassLoader().getResource(executorName) + .toURI(); Files.copy(Paths.get(executorPath), Paths.get(tmpMockExecutor), REPLACE_EXISTING); @@ -140,7 +143,8 @@ public class TestLinuxContainerExecutorWithMocks { } @Before - public void setup() throws IOException, ContainerExecutionException { + public void setup() throws IOException, ContainerExecutionException, + URISyntaxException { assumeNotWindows(); tmpMockExecutor = System.getProperty("test.build.data") + @@ -172,7 +176,18 @@ public class TestLinuxContainerExecutorWithMocks { } @Test - public void testContainerLaunch() + public void testContainerLaunchWithoutHTTPS() + throws IOException, ConfigurationException { + testContainerLaunch(false); + } + + @Test + public void testContainerLaunchWithHTTPS() + throws IOException, ConfigurationException { + testContainerLaunch(true); + } + + private void testContainerLaunch(boolean https) throws IOException, ConfigurationException { String appSubmitter = "nobody"; String cmd = String.valueOf( @@ -193,41 +208,64 @@ public class TestLinuxContainerExecutorWithMocks { Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); + Path keystorePath = new Path("file:///dev/null"); + Path truststorePath = new Path("file:///dev/null"); Path workDir = new Path("/tmp"); Path pidFile = new Path(workDir, "pid.txt"); mockExec.activateContainer(cId, pidFile); - int ret = mockExec.launchContainer(new ContainerStartContext.Builder() - .setContainer(container) - .setNmPrivateContainerScriptPath(scriptPath) - .setNmPrivateTokensPath(tokensPath) - .setUser(appSubmitter) - .setAppId(appId) - .setContainerWorkDir(workDir) - .setLocalDirs(dirsHandler.getLocalDirs()) - .setLogDirs(dirsHandler.getLogDirs()) - .setFilecacheDirs(new ArrayList<>()) - .setUserLocalDirs(new ArrayList<>()) - .setContainerLocalDirs(new ArrayList<>()) - .setContainerLogDirs(new ArrayList<>()) - .setUserFilecacheDirs(new ArrayList<>()) - .setApplicationLocalDirs(new ArrayList<>()) - .build()); + ContainerStartContext.Builder ctxBuilder = + new ContainerStartContext.Builder() + .setContainer(container) + .setNmPrivateContainerScriptPath(scriptPath) + .setNmPrivateTokensPath(tokensPath) + .setUser(appSubmitter) + .setAppId(appId) + .setContainerWorkDir(workDir) + .setLocalDirs(dirsHandler.getLocalDirs()) + .setLogDirs(dirsHandler.getLogDirs()) + .setFilecacheDirs(new ArrayList<>()) + .setUserLocalDirs(new ArrayList<>()) + .setContainerLocalDirs(new ArrayList<>()) + .setContainerLogDirs(new ArrayList<>()) + .setUserFilecacheDirs(new ArrayList<>()) + .setApplicationLocalDirs(new ArrayList<>()); + if (https) { + ctxBuilder.setNmPrivateKeystorePath(keystorePath); + ctxBuilder.setNmPrivateTruststorePath(truststorePath); + } + int ret = mockExec.launchContainer(ctxBuilder.build()); assertEquals(0, ret); - assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, - appSubmitter, cmd, appId, containerId, - workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), - StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, - dirsHandler.getLocalDirs()), - StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, - dirsHandler.getLogDirs()), "cgroups=none"), - readMockParams()); - + if (https) { + assertEquals(Arrays.asList( + YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, appId, containerId, + workDir.toString(), scriptPath.toUri().getPath(), + tokensPath.toUri().getPath(), "--https", + keystorePath.toUri().getPath(), truststorePath.toUri().getPath(), + pidFile.toString(), + StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLocalDirs()), + StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLogDirs()), "cgroups=none"), + readMockParams()); + } else { + assertEquals(Arrays.asList( + YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, appId, containerId, + workDir.toString(), scriptPath.toUri().getPath(), + tokensPath.toUri().getPath(), "--http", pidFile.toString(), + StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLocalDirs()), + StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLogDirs()), "cgroups=none"), + readMockParams()); + } } @Test (timeout = 5000) public void testContainerLaunchWithPriority() - throws IOException, ConfigurationException { + throws IOException, ConfigurationException, URISyntaxException { // set the scheduler priority to make sure still works with nice -n prio Configuration conf = new Configuration(); @@ -242,7 +280,7 @@ public class TestLinuxContainerExecutorWithMocks { assertEquals("third should be the priority", Integer.toString(2), command.get(2)); - testContainerLaunch(); + testContainerLaunchWithoutHTTPS(); } @Test (timeout = 5000) @@ -306,7 +344,7 @@ public class TestLinuxContainerExecutorWithMocks { @Test public void testContainerLaunchError() - throws IOException, ContainerExecutionException { + throws IOException, ContainerExecutionException, URISyntaxException { final String[] expecetedMessage = {"badcommand", "Exit code: 24"}; final String[] executor = { @@ -410,7 +448,8 @@ public class TestLinuxContainerExecutorWithMocks { assertEquals(Arrays.asList(YarnConfiguration. DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, - workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), + workDir.toString(), "/bin/echo", "/dev/null", "--http", + pidFile.toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLocalDirs()), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, @@ -462,7 +501,7 @@ public class TestLinuxContainerExecutorWithMocks { } @Test - public void testDeleteAsUser() throws IOException { + public void testDeleteAsUser() throws IOException, URISyntaxException { String appSubmitter = "nobody"; String cmd = String.valueOf( PrivilegedOperation.RunAsUserCommand.DELETE_AS_USER.getValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 5714a1c..1f218d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -25,6 +25,7 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; import java.io.BufferedReader; +import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; @@ -55,9 +56,12 @@ import com.google.common.base.Supplier; import com.google.common.collect.Lists; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.test.GenericTestUtils; @@ -108,11 +112,13 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.server.security.AMSecretKeys; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper; @@ -124,6 +130,8 @@ import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class TestContainerLaunch extends BaseContainerManagerTest { @@ -2443,4 +2451,145 @@ public class TestContainerLaunch extends BaseContainerManagerTest { launch.getUserFilecacheDirs(localDirsForRead)), StringUtils.join(",", ctx.getUserFilecacheDirs())); } + + @Test(timeout = 20000) + public void testFilesAndEnvWithoutHTTPS() throws Exception { + testFilesAndEnv(false); + } + + @Test(timeout = 20000) + public void testFilesAndEnvWithHTTPS() throws Exception { + testFilesAndEnv(true); + } + + private void testFilesAndEnv(boolean https) throws Exception { + // setup mocks + Dispatcher dispatcher = mock(Dispatcher.class); + EventHandler handler = mock(EventHandler.class); + when(dispatcher.getEventHandler()).thenReturn(handler); + ContainerExecutor containerExecutor = mock(ContainerExecutor.class); + doAnswer(new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + Object[] args = invocation.getArguments(); + DataOutputStream dos = (DataOutputStream) args[0]; + dos.writeBytes("script"); + return null; + } + }).when(containerExecutor).writeLaunchEnv( + any(), any(), any(), any(), any(), any(), any()); + Application app = mock(Application.class); + ApplicationId appId = mock(ApplicationId.class); + when(appId.toString()).thenReturn("1"); + when(app.getAppId()).thenReturn(appId); + Container container = mock(Container.class); + ContainerId id = mock(ContainerId.class); + when(id.toString()).thenReturn("1"); + when(container.getContainerId()).thenReturn(id); + when(container.getUser()).thenReturn("user"); + ContainerLaunchContext clc = mock(ContainerLaunchContext.class); + when(clc.getCommands()).thenReturn(Lists.newArrayList()); + when(container.getLaunchContext()).thenReturn(clc); + Credentials credentials = mock(Credentials.class); + when(container.getCredentials()).thenReturn(credentials); + doAnswer(new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + Object[] args = invocation.getArguments(); + DataOutputStream dos = (DataOutputStream) args[0]; + dos.writeBytes("credentials"); + return null; + } + }).when(credentials).writeTokenStorageToStream(any(DataOutputStream.class)); + if (https) { + when(credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE)) + .thenReturn("keystore".getBytes()); + when(credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD)) + .thenReturn("keystore_password".getBytes()); + when(credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE)) + .thenReturn("truststore".getBytes()); + when(credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD)) + .thenReturn("truststore_password".getBytes()); + } + + // call containerLaunch + ContainerLaunch containerLaunch = new ContainerLaunch( + distContext, conf, dispatcher, + containerExecutor, app, container, dirsHandler, containerManager); + containerLaunch.call(); + + // verify the nmPrivate paths and files + ArgumentCaptor<ContainerStartContext> cscArgument = + ArgumentCaptor.forClass(ContainerStartContext.class); + verify(containerExecutor, times(1)).launchContainer(cscArgument.capture()); + ContainerStartContext csc = cscArgument.getValue(); + Path nmPrivate = dirsHandler.getLocalPathForWrite( + ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR + + appId.toString() + Path.SEPARATOR + id.toString()); + Assert.assertEquals(new Path(nmPrivate, ContainerLaunch.CONTAINER_SCRIPT), + csc.getNmPrivateContainerScriptPath()); + Assert.assertEquals(new Path(nmPrivate, + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, + id.toString())), csc.getNmPrivateTokensPath()); + Assert.assertEquals("script", + readStringFromPath(csc.getNmPrivateContainerScriptPath())); + Assert.assertEquals("credentials", + readStringFromPath(csc.getNmPrivateTokensPath())); + if (https) { + Assert.assertEquals(new Path(nmPrivate, ContainerLaunch.KEYSTORE_FILE), + csc.getNmPrivateKeystorePath()); + Assert.assertEquals(new Path(nmPrivate, ContainerLaunch.TRUSTSTORE_FILE), + csc.getNmPrivateTruststorePath()); + Assert.assertEquals("keystore", + readStringFromPath(csc.getNmPrivateKeystorePath())); + Assert.assertEquals("truststore", + readStringFromPath(csc.getNmPrivateTruststorePath())); + } else { + Assert.assertNull(csc.getNmPrivateKeystorePath()); + Assert.assertNull(csc.getNmPrivateTruststorePath()); + } + + // verify env + ArgumentCaptor<Map> envArgument = ArgumentCaptor.forClass(Map.class); + verify(containerExecutor, times(1)).writeLaunchEnv(any(), + envArgument.capture(), any(), any(), any(), any(), any()); + Map env = envArgument.getValue(); + Path workDir = dirsHandler.getLocalPathForWrite( + ContainerLocalizer.USERCACHE + Path.SEPARATOR + container.getUser() + + Path.SEPARATOR + ContainerLocalizer.APPCACHE + Path.SEPARATOR + + app.getAppId().toString() + Path.SEPARATOR + + container.getContainerId().toString()); + Assert.assertEquals(new Path(workDir, + ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE).toUri().getPath(), + env.get(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME)); + if (https) { + Assert.assertEquals(new Path(workDir, + ContainerLaunch.KEYSTORE_FILE).toUri().getPath(), + env.get(ApplicationConstants.KEYSTORE_FILE_LOCATION_ENV_NAME)); + Assert.assertEquals("keystore_password", + env.get(ApplicationConstants.KEYSTORE_PASSWORD_ENV_NAME)); + Assert.assertEquals(new Path(workDir, + ContainerLaunch.TRUSTSTORE_FILE).toUri().getPath(), + env.get(ApplicationConstants.TRUSTSTORE_FILE_LOCATION_ENV_NAME)); + Assert.assertEquals("truststore_password", + env.get(ApplicationConstants.TRUSTSTORE_PASSWORD_ENV_NAME)); + } else { + Assert.assertNull(env.get("KEYSTORE_FILE_LOCATION")); + Assert.assertNull(env.get("KEYSTORE_PASSWORD")); + Assert.assertNull(env.get("TRUSTSTORE_FILE_LOCATION")); + Assert.assertNull(env.get("TRUSTSTORE_PASSWORD")); + } + } + + private String readStringFromPath(Path p) throws IOException { + FileSystem fs = FileSystem.get(conf); + try (FSDataInputStream is = fs.open(p)) { + byte[] bytes = IOUtils.readFullyToByteArray(is); + return new String(bytes); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java index 4374e66..3b57938 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -32,22 +33,34 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; +import org.apache.hadoop.yarn.server.security.AMSecretKeys; import org.junit.Test; import org.mockito.ArgumentCaptor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Unit tests for relaunching containers. */ public class TestContainerRelaunch { @Test - public void testRelaunchContext() throws Exception { + public void testRelaunchContextWithoutHTTPS() throws Exception { + testRelaunchContext(false); + } + + @Test + public void testRelaunchContextWithHTTPS() throws Exception { + testRelaunchContext(true); + } + + private void testRelaunchContext(boolean https) throws Exception { Configuration conf = new Configuration(); Context mockContext = mock(Context.class); @@ -63,6 +76,16 @@ public class TestContainerRelaunch { doReturn(cid).when(mockContainer).getContainerId(); doReturn("/foo").when(mockContainer).getWorkDir(); doReturn("/bar").when(mockContainer).getLogDir(); + Credentials mockCredentials = mock(Credentials.class); + when(mockContainer.getCredentials()).thenReturn(mockCredentials); + if (https) { + when(mockCredentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE)) + .thenReturn("keystore".getBytes()); + when(mockCredentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE)) + .thenReturn("truststore".getBytes()); + } LocalDirsHandlerService mockDirsHandler = mock(LocalDirsHandlerService.class); doReturn(true).when(mockDirsHandler).isGoodLocalDir(any(String.class)); @@ -91,6 +114,13 @@ public class TestContainerRelaunch { assertNotNull("log dirs null", csc.getLogDirs()); assertNotNull("script path null", csc.getNmPrivateContainerScriptPath()); assertNotNull("tokens path null", csc.getNmPrivateTokensPath()); + if (https) { + assertNotNull("keystore path null", csc.getNmPrivateKeystorePath()); + assertNotNull("truststore path null", csc.getNmPrivateTruststorePath()); + } else { + assertNull("keystore path not null", csc.getNmPrivateKeystorePath()); + assertNull("truststore path not null", csc.getNmPrivateTruststorePath()); + } assertNotNull("user null", csc.getUser()); assertNotNull("user local dirs null", csc.getUserLocalDirs()); assertNotNull("user filecache dirs null", csc.getUserFilecacheDirs()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index a880ba9..934e16f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -57,6 +57,8 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.slf4j.Logger; @@ -75,6 +77,8 @@ import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -98,7 +102,9 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCAL_DIRS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOG_DIRS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.NM_PRIVATE_CONTAINER_SCRIPT_PATH; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.NM_PRIVATE_KEYSTORE_PATH; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.NM_PRIVATE_TOKENS_PATH; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.NM_PRIVATE_TRUSTSTORE_PATH; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.PID; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.PID_FILE_PATH; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.PROCFS; @@ -118,6 +124,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +@RunWith(Parameterized.class) public class TestDockerContainerRuntime { private static final Logger LOG = LoggerFactory.getLogger(TestDockerContainerRuntime.class); @@ -141,6 +148,8 @@ public class TestDockerContainerRuntime { private Path containerWorkDir; private Path nmPrivateContainerScriptPath; private Path nmPrivateTokensPath; + private Path nmPrivateKeystorePath; + private Path nmPrivateTruststorePath; private Path pidFilePath; private List<String> localDirs; private List<String> logDirs; @@ -159,6 +168,16 @@ public class TestDockerContainerRuntime { @Rule public TemporaryFolder tempDir = new TemporaryFolder(); + @Parameterized.Parameters(name = "https={0}") + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + {true}, {false} + }); + } + + @Parameterized.Parameter + public boolean https; + @Before public void setup() { String tmpPath = new StringBuffer(System.getProperty("test.build.data")) @@ -227,6 +246,13 @@ public class TestDockerContainerRuntime { containerWorkDir = new Path("/test_container_work_dir"); nmPrivateContainerScriptPath = new Path("/test_script_path"); nmPrivateTokensPath = new Path("/test_private_tokens_path"); + if (https) { + nmPrivateKeystorePath = new Path("/test_private_keystore_path"); + nmPrivateTruststorePath = new Path("/test_private_truststore_path"); + } else { + nmPrivateKeystorePath = null; + nmPrivateTruststorePath = null; + } pidFilePath = new Path("/test_pid_file_path"); localDirs = new ArrayList<>(); logDirs = new ArrayList<>(); @@ -261,6 +287,9 @@ public class TestDockerContainerRuntime { .setExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH, nmPrivateContainerScriptPath) .setExecutionAttribute(NM_PRIVATE_TOKENS_PATH, nmPrivateTokensPath) + .setExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH, nmPrivateKeystorePath) + .setExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH, + nmPrivateTruststorePath) .setExecutionAttribute(PID_FILE_PATH, pidFilePath) .setExecutionAttribute(LOCAL_DIRS, localDirs) .setExecutionAttribute(LOG_DIRS, logDirs) @@ -385,9 +414,9 @@ public class TestDockerContainerRuntime { List<String> args = op.getArguments(); - //This invocation of container-executor should use 12 arguments in a + //This invocation of container-executor should use 15 or 13 arguments in a // specific order - int expected = 12; + int expected = (https) ? 15 : 13; int counter = 1; Assert.assertEquals(expected, args.size()); Assert.assertEquals(user, args.get(counter++)); @@ -396,10 +425,19 @@ public class TestDockerContainerRuntime { Assert.assertEquals(appId, args.get(counter++)); Assert.assertEquals(containerId, args.get(counter++)); Assert.assertEquals(containerWorkDir.toString(), args.get(counter++)); - Assert.assertEquals(nmPrivateContainerScriptPath.toUri() - .toString(), args.get(counter++)); + Assert.assertEquals(nmPrivateContainerScriptPath.toUri().toString(), + args.get(counter++)); Assert.assertEquals(nmPrivateTokensPath.toUri().getPath(), args.get(counter++)); + if (https) { + Assert.assertEquals("--https", args.get(counter++)); + Assert.assertEquals(nmPrivateKeystorePath.toUri().toString(), + args.get(counter++)); + Assert.assertEquals(nmPrivateTruststorePath.toUri().toString(), + args.get(counter++)); + } else { + Assert.assertEquals("--http", args.get(counter++)); + } Assert.assertEquals(pidFilePath.toString(), args.get(counter++)); Assert.assertEquals(localDirs.get(0), args.get(counter++)); Assert.assertEquals(logDirs.get(0), args.get(counter++)); @@ -415,13 +453,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); runtime.launchContainer(builder.build()); - - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -467,12 +499,7 @@ public class TestDockerContainerRuntime { runtime.initialize(conf, nmContext); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines(Paths.get( - dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -516,13 +543,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); runtime.launchContainer(builder.build()); - - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); Assert.assertEquals(13, dockerCommands.size()); int counter = 0; @@ -631,13 +652,9 @@ public class TestDockerContainerRuntime { //this should cause no failures. runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); + List<String> dockerCommands = readDockerCommands(); //This is the expected docker invocation for this case - List<String> dockerCommands = Files - .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); int expected = 14; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); @@ -690,13 +707,9 @@ public class TestDockerContainerRuntime { expectedHostname); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); + List<String> dockerCommands = readDockerCommands(); //This is the expected docker invocation for this case - List<String> dockerCommands = Files - .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); int expected = 14; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); @@ -757,15 +770,10 @@ public class TestDockerContainerRuntime { //this should cause no failures. runtime.initialize(conf, nmContext); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); + List<String> dockerCommands = readDockerCommands(); //This is the expected docker invocation for this case. customNetwork1 // ("sdn1") is the expected network to be used in this case - List<String> dockerCommands = Files - .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); - int expected = 14; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); @@ -807,15 +815,10 @@ public class TestDockerContainerRuntime { env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, customNetwork2); runtime.launchContainer(builder.build()); - - op = capturePrivilegedOperationAndVerifyArgs(); - args = op.getArguments(); - dockerCommandFile = args.get(11); + dockerCommands = readDockerCommands(); //This is the expected docker invocation for this case. customNetwork2 // ("sdn2") is the expected network to be used in this case - dockerCommands = Files - .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); counter = 0; Assert.assertEquals(expected, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", @@ -874,13 +877,7 @@ public class TestDockerContainerRuntime { env.put(DockerLinuxContainerRuntime .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "invalid-value"); runtime.launchContainer(builder.build()); - - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; Assert.assertEquals(expected, dockerCommands.size()); @@ -926,12 +923,7 @@ public class TestDockerContainerRuntime { .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "host"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 14; int counter = 0; @@ -977,13 +969,7 @@ public class TestDockerContainerRuntime { env.put(DockerLinuxContainerRuntime .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value"); runtime.launchContainer(builder.build()); - - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; Assert.assertEquals(expected, dockerCommands.size()); @@ -1087,12 +1073,7 @@ public class TestDockerContainerRuntime { .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -1203,12 +1184,7 @@ public class TestDockerContainerRuntime { "test_dir/test_resource_file:test_mount:ro"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -1257,12 +1233,7 @@ public class TestDockerContainerRuntime { "test_dir/test_resource_file:test_mount2:ro"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -1312,12 +1283,7 @@ public class TestDockerContainerRuntime { "/a:/a:shared,/b:/b:ro+shared,/c:/c:rw+rshared,/d:/d:private"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -1421,12 +1387,7 @@ public class TestDockerContainerRuntime { "/run"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); Assert.assertTrue(dockerCommands.contains(" tmpfs=/run")); } @@ -1444,12 +1405,7 @@ public class TestDockerContainerRuntime { "/run,/tmp"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); Assert.assertTrue(dockerCommands.contains(" tmpfs=/run,/tmp")); } @@ -1468,12 +1424,7 @@ public class TestDockerContainerRuntime { "/tmpfs"); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); Assert.assertTrue(dockerCommands.contains(" tmpfs=/tmpfs,/run,/var/run")); } @@ -1567,12 +1518,7 @@ public class TestDockerContainerRuntime { runtime.initialize(conf, nmContext); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -1634,12 +1580,7 @@ public class TestDockerContainerRuntime { runtime.initialize(conf, nmContext); runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 13; int counter = 0; @@ -2224,12 +2165,7 @@ public class TestDockerContainerRuntime { checkVolumeCreateCommand(); runtime.launchContainer(containerRuntimeContext); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(); int expected = 14; int counter = 0; @@ -2348,7 +2284,7 @@ public class TestDockerContainerRuntime { List<String> args = op.getArguments(); - int expectedArgs = 12; + int expectedArgs = (https) ? 15 : 13; int argsCounter = 0; Assert.assertEquals(expectedArgs, args.size()); Assert.assertEquals(runAsUser, args.get(argsCounter++)); @@ -2362,6 +2298,15 @@ public class TestDockerContainerRuntime { Assert.assertEquals(outDir.toUri().getPath(), args.get(argsCounter++)); Assert.assertEquals(nmPrivateTokensPath.toUri().getPath(), args.get(argsCounter++)); + if (https) { + Assert.assertEquals("--https", args.get(argsCounter++)); + Assert.assertEquals(nmPrivateKeystorePath.toUri().toString(), + args.get(argsCounter++)); + Assert.assertEquals(nmPrivateTruststorePath.toUri().toString(), + args.get(argsCounter++)); + } else { + Assert.assertEquals("--http", args.get(argsCounter++)); + } Assert.assertEquals(pidFilePath.toString(), args.get(argsCounter++)); Assert.assertEquals(localDirs.get(0), args.get(argsCounter++)); Assert.assertEquals(logDirs.get(0), args.get(argsCounter++)); @@ -2416,13 +2361,7 @@ public class TestDockerContainerRuntime { DockerCommandExecutor.DockerContainerStatus.STOPPED.getName()); runtime.initialize(conf, nmContext); runtime.relaunchContainer(builder.build()); - - PrivilegedOperation op = capturePrivilegedOperation(2); - List<String> args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List<String> dockerCommands = Files.readAllLines( - Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + List<String> dockerCommands = readDockerCommands(2); int expected = 3; int counter = 0; @@ -2445,4 +2384,22 @@ public class TestDockerContainerRuntime { dockerCommands.get(2)); Assert.assertEquals(" signal=" + signal, dockerCommands.get(3)); } + + private List<String> readDockerCommands() throws IOException, + PrivilegedOperationException { + return readDockerCommands(1); + } + + private List<String> readDockerCommands(int invocations) throws IOException, + PrivilegedOperationException { + PrivilegedOperation op = (invocations == 1) + ? capturePrivilegedOperationAndVerifyArgs() + : capturePrivilegedOperation(invocations); + List<String> args = op.getArguments(); + String dockerCommandFile = args.get((https) ? 14 : 12); + + List<String> dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + return dockerCommands; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 1596598..3562078 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretMan import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.util.Clock; @@ -119,6 +120,8 @@ public class RMActiveServiceContext { private ResourceProfilesManager resourceProfilesManager; private MultiNodeSortingManager<SchedulerNode> multiNodeSortingManager; + private ProxyCAManager proxyCAManager; + public RMActiveServiceContext() { queuePlacementManager = new PlacementManager(); } @@ -554,4 +557,16 @@ public class RMActiveServiceContext { ResourceProfilesManager resourceProfilesManager) { this.resourceProfilesManager = resourceProfilesManager; } + + @Private + @Unstable + public ProxyCAManager getProxyCAManager() { + return proxyCAManager; + } + + @Private + @Unstable + public void setProxyCAManager(ProxyCAManager proxyCAManager) { + this.proxyCAManager = proxyCAManager; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index d3daa05..f06befe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretMan import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; @@ -188,4 +189,8 @@ public interface RMContext extends ApplicationMasterServiceContext { void setMultiNodeSortingManager( MultiNodeSortingManager<SchedulerNode> multiNodeSortingManager); + + ProxyCAManager getProxyCAManager(); + + void setProxyCAManager(ProxyCAManager proxyCAManager); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 5b295f6..48f74d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretMan import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; @@ -637,6 +638,16 @@ public class RMContextImpl implements RMContext { public void setResourceProfilesManager(ResourceProfilesManager mgr) { this.activeServiceContext.setResourceProfilesManager(mgr); } + + @Override + public ProxyCAManager getProxyCAManager() { + return this.activeServiceContext.getProxyCAManager(); + } + + @Override + public void setProxyCAManager(ProxyCAManager proxyCAManager) { + this.activeServiceContext.setProxyCAManager(proxyCAManager); + } // Note: Read java doc before adding any services over here. @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 16f019f..5f4ae6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -106,6 +106,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; +import org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp; @@ -113,6 +114,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.service.SystemServiceManager; import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher; +import org.apache.hadoop.yarn.server.webproxy.ProxyCA; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; @@ -197,6 +199,7 @@ public class ResourceManager extends CompositeService protected ApplicationACLsManager applicationACLsManager; protected QueueACLsManager queueACLsManager; private FederationStateStoreService federationStateStoreService; + private ProxyCAManager proxyCAManager; private WebApp webApp; private AppReportFetcher fetcher = null; protected ResourceTrackerService resourceTracker; @@ -830,6 +833,10 @@ public class ResourceManager extends CompositeService LOG.info("Initialized Federation membership."); } + proxyCAManager = new ProxyCAManager(new ProxyCA(), rmContext); + addService(proxyCAManager); + rmContext.setProxyCAManager(proxyCAManager); + new RMNMInfo(rmContext, scheduler); if (conf.getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, @@ -1180,6 +1187,8 @@ public class ResourceManager extends CompositeService } builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); + builder.withAttribute(WebAppProxy.PROXY_CA, + rmContext.getProxyCAManager().getProxyCA()); builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); String[] proxyParts = proxyHostAndPort.split(":"); builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 0bedb52..a734cb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.amlauncher; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -62,6 +63,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; +import org.apache.hadoop.yarn.server.security.AMSecretKeys; +import org.apache.hadoop.yarn.server.webproxy.ProxyCA; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @@ -233,6 +236,32 @@ public class AMLauncher implements Runnable { if (amrmToken != null) { credentials.addToken(amrmToken.getService(), amrmToken); } + + // Setup Keystore and Truststore + String httpsPolicy = conf.get(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY, + YarnConfiguration.DEFAULT_RM_APPLICATION_HTTPS_POLICY); + if (httpsPolicy.equals("LENIENT") || httpsPolicy.equals("STRICT")) { + ProxyCA proxyCA = rmContext.getProxyCAManager().getProxyCA(); + try { + String kPass = proxyCA.generateKeyStorePassword(); + byte[] keyStore = proxyCA.createChildKeyStore(applicationId, kPass); + credentials.addSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE, keyStore); + credentials.addSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD, + kPass.getBytes(StandardCharsets.UTF_8)); + String tPass = proxyCA.generateKeyStorePassword(); + byte[] trustStore = proxyCA.getChildTrustStore(tPass); + credentials.addSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE, trustStore); + credentials.addSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD, + tPass.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + throw new IOException(e); + } + } + DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ProxyCAManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ProxyCAManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ProxyCAManager.java new file mode 100644 index 0000000..12b677e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ProxyCAManager.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; +import org.apache.hadoop.yarn.server.webproxy.ProxyCA; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manager for {@link ProxyCA}, which contains the Certificate Authority for + * AMs to have certificates for HTTPS communication with the RM Proxy. + */ +@Private +@InterfaceStability.Unstable +public class ProxyCAManager extends AbstractService implements Recoverable { + private static final Logger LOG = + LoggerFactory.getLogger(ProxyCAManager.class); + + private ProxyCA proxyCA; + private RMContext rmContext; + + public ProxyCAManager(ProxyCA proxyCA, RMContext rmContext) { + super(ProxyCAManager.class.getName()); + this.proxyCA = proxyCA; + this.rmContext = rmContext; + } + + @Override + protected void serviceStart() throws Exception { + proxyCA.init(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + super.serviceStop(); + } + + public ProxyCA getProxyCA() { + return proxyCA; + } + + public void recover(RMState state) { + // TODO: RM HA YARN-8449 + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 38181e2..03ccd76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -29,8 +29,11 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; @@ -79,7 +82,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager; +import org.apache.hadoop.yarn.server.security.AMSecretKeys; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.server.webproxy.ProxyCA; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -91,6 +97,7 @@ import com.google.common.base.Supplier; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; public class TestApplicationMasterLauncher { @@ -424,16 +431,47 @@ public class TestApplicationMasterLauncher { } @Test - public void testSetupTokens() throws Exception { - MockRM rm = new MockRM(); + public void testSetupTokensWithoutHTTPS() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + // default conf + testSetupTokens(false, conf); + conf.set(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY, "NONE"); + testSetupTokens(false, conf); + } + + @Test + public void testSetupTokensWithHTTPS() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY, "LENIENT"); + testSetupTokens(true, conf); + conf.set(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY, "STRICT"); + testSetupTokens(true, conf); + } + + private void testSetupTokens(boolean https, YarnConfiguration conf) + throws Exception { + MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5000); RMApp app = rm.submitApp(2000); /// kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); - MyAMLauncher launcher = new MyAMLauncher(rm.getRMContext(), - attempt, AMLauncherEventType.LAUNCH, rm.getConfig()); + AMRMTokenIdentifier tokenIdentifier = + new AMRMTokenIdentifier(attempt.getAppAttemptId(), 1); + ProxyCA proxyCA = mock(ProxyCA.class); + when(proxyCA.generateKeyStorePassword()) + .thenReturn("kPassword").thenReturn("tPassword"); + when(proxyCA.createChildKeyStore(any(), any())) + .thenReturn("keystore".getBytes()); + when(proxyCA.getChildTrustStore(any())) + .thenReturn("truststore".getBytes()); + RMContext rmContext = spy(rm.getRMContext()); + ProxyCAManager proxyCAManager = mock(ProxyCAManager.class); + when(proxyCAManager.getProxyCA()).thenReturn(proxyCA); + when(rmContext.getProxyCAManager()).thenReturn(proxyCAManager); + MyAMLauncher launcher = new MyAMLauncher(rmContext, + attempt, AMLauncherEventType.LAUNCH, rm.getConfig(), tokenIdentifier); DataOutputBuffer dob = new DataOutputBuffer(); Credentials ts = new Credentials(); ts.writeTokenStorageToStream(dob); @@ -455,14 +493,48 @@ public class TestApplicationMasterLauncher { } catch (java.io.EOFException e) { Assert.fail("EOFException should not happen."); } + + // verify token + DataInputByteBuffer dibb = new DataInputByteBuffer(); + dibb.reset(amContainer.getTokens()); + Credentials credentials = new Credentials(); + credentials.readTokenStorageStream(dibb); + Assert.assertEquals(1, credentials.numberOfTokens()); + org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token = + credentials.getAllTokens().iterator().next(); + Assert.assertEquals(tokenIdentifier.getKind(), token.getKind()); + Assert.assertArrayEquals(tokenIdentifier.getBytes(), token.getIdentifier()); + Assert.assertArrayEquals("password".getBytes(), token.getPassword()); + + // verify keystore and truststore + if (https) { + Assert.assertEquals(4, credentials.numberOfSecretKeys()); + Assert.assertArrayEquals("keystore".getBytes(), + credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE)); + Assert.assertArrayEquals("kPassword".getBytes(), + credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD)); + Assert.assertArrayEquals("truststore".getBytes(), + credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE)); + Assert.assertArrayEquals("tPassword".getBytes(), + credentials.getSecretKey( + AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD)); + } else { + Assert.assertEquals(0, credentials.numberOfSecretKeys()); + } } static class MyAMLauncher extends AMLauncher { int count; + AMRMTokenIdentifier tokenIdentifier; public MyAMLauncher(RMContext rmContext, RMAppAttempt application, - AMLauncherEventType eventType, Configuration conf) { + AMLauncherEventType eventType, Configuration conf, + AMRMTokenIdentifier tokenIdentifier) { super(rmContext, application, eventType, conf); count = 0; + this.tokenIdentifier = tokenIdentifier; } protected org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> @@ -471,7 +543,9 @@ public class TestApplicationMasterLauncher { if (count == 1) { throw new RuntimeException("createAndSetAMRMToken failure"); } - return null; + return new org.apache.hadoop.security.token.Token<>( + tokenIdentifier.getBytes(), "password".getBytes(), + tokenIdentifier.getKind(), new Text()); } protected void setupTokens(ContainerLaunchContext container, http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestProxyCAManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestProxyCAManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestProxyCAManager.java new file mode 100644 index 0000000..8633519 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestProxyCAManager.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.security; + + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.webproxy.ProxyCA; +import org.junit.Assert; +import org.junit.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class TestProxyCAManager { + + @Test + public void testBasics() throws Exception { + ProxyCA proxyCA = spy(new ProxyCA()); + RMContext rmContext = mock(RMContext.class); + ProxyCAManager proxyCAManager = new ProxyCAManager(proxyCA, rmContext); + proxyCAManager.init(new YarnConfiguration()); + Assert.assertEquals(proxyCA, proxyCAManager.getProxyCA()); + verify(proxyCA, times(0)).init(); + Assert.assertNull(proxyCA.getCaCert()); + Assert.assertNull(proxyCA.getCaKeyPair()); + + proxyCAManager.start(); + verify(proxyCA, times(1)).init(); + Assert.assertNotNull(proxyCA.getCaCert()); + Assert.assertNotNull(proxyCA.getCaKeyPair()); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
