[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop Updated Branches: refs/heads/trunk e81397545 -> 583fa6ed4 http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java -- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 1517b04..c171143 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -31,26 +32,35 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.crypto.key.kms.KMSDelegationToken; +import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer; import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider; +import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider; import org.apache.hadoop.crypto.key.kms.ValueQueue; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MultipleIOException; +import org.apache.hadoop.io.Text; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.KMSUtil; +import org.apache.hadoop.util.KMSUtilFaultInjector; import org.apache.hadoop.util.Time; import org.apache.http.client.utils.URIBuilder; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -71,7 +81,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.Writer; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketTimeoutException; import java.net.URI; @@ -96,6 +105,10 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -113,6 +126,20 @@ public class TestKMS { private SSLFactory sslFactory; + private final KMSUtilFaultInjector oldInjector = + KMSUtilFaultInjector.get(); + + // Injector to create providers with different ports. Can only happen in tests + private final KMSUtilFaultInjector testInjector = + new KMSUtilFaultInjector() { +@Override +public KeyProvider createKeyProviderForTests(String value, +Configuration conf) throws IOException { + return TestLoadBalancingKMSClientProvider + .createKeyProviderForTests(value, conf); +} + }; + // Keep track of all key providers created during a test case, so they can be // closed at test tearDown. private List providersCreated = new LinkedList<>(); @@ -122,7 +149,12 @@ public class TestKMS { @Before public void setUp() throws Exception { -setUpMiniKdc(); +GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE); +GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE); // r
[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop Updated Branches: refs/heads/branch-2 0fb1457d8 -> 95cedc558 http://git-wip-us.apache.org/repos/asf/hadoop/blob/95cedc55/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java -- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 712536f..b75eb76 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -30,19 +31,27 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.crypto.key.kms.KMSDelegationToken; +import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer; import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider; +import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.KMSUtil; +import org.apache.hadoop.util.KMSUtilFaultInjector; import org.apache.hadoop.util.Time; +import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -64,7 +73,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.Writer; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketTimeoutException; import java.net.URI; @@ -82,7 +90,14 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TestKMS { private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class); @@ -90,11 +105,31 @@ public class TestKMS { private static final String SSL_RELOADER_THREAD_NAME = "Truststore reloader thread"; + private final KMSUtilFaultInjector oldInjector = + KMSUtilFaultInjector.get(); + + // Injector to create providers with different ports. Can only happen in tests + private final KMSUtilFaultInjector testInjector = + new KMSUtilFaultInjector() { +@Override +public KeyProvider createKeyProviderForTests(String value, +Configuration conf) throws IOException { + return TestLoadBalancingKMSClientProvider + .createKeyProviderForTests(value, conf); +} + }; + @Rule public final Timeout testTimeout = new Timeout(18); @Before - public void cleanUp() { + public void setUp() throws Exception { +GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE); +GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE); // resetting kerberos security Configuration conf = new Configuration(); UserGroupInformation.setConfiguration(conf); @@ -114,17 +149,71 @@ public class TestKMS { } public static abstract class KMSCallable implements Callable { -private URL kmsUrl; +private List
[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop Updated Branches: refs/heads/branch-2.9 87485d40c -> 46ac59a9b http://git-wip-us.apache.org/repos/asf/hadoop/blob/46ac59a9/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java -- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 712536f..b75eb76 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -30,19 +31,27 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.crypto.key.kms.KMSDelegationToken; +import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer; import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider; +import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.KMSUtil; +import org.apache.hadoop.util.KMSUtilFaultInjector; import org.apache.hadoop.util.Time; +import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -64,7 +73,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.Writer; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketTimeoutException; import java.net.URI; @@ -82,7 +90,14 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TestKMS { private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class); @@ -90,11 +105,31 @@ public class TestKMS { private static final String SSL_RELOADER_THREAD_NAME = "Truststore reloader thread"; + private final KMSUtilFaultInjector oldInjector = + KMSUtilFaultInjector.get(); + + // Injector to create providers with different ports. Can only happen in tests + private final KMSUtilFaultInjector testInjector = + new KMSUtilFaultInjector() { +@Override +public KeyProvider createKeyProviderForTests(String value, +Configuration conf) throws IOException { + return TestLoadBalancingKMSClientProvider + .createKeyProviderForTests(value, conf); +} + }; + @Rule public final Timeout testTimeout = new Timeout(18); @Before - public void cleanUp() { + public void setUp() throws Exception { +GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE); +GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE); // resetting kerberos security Configuration conf = new Configuration(); UserGroupInformation.setConfiguration(conf); @@ -114,17 +149,71 @@ public class TestKMS { } public static abstract class KMSCallable implements Callable { -private URL kmsUrl; +private Lis
[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 5f8ab3a6b -> 714a079ff http://git-wip-us.apache.org/repos/asf/hadoop/blob/714a079f/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java -- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 308c974..b67b8a1 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -30,20 +31,28 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.crypto.key.kms.KMSDelegationToken; +import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer; import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider; +import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.KMSUtil; +import org.apache.hadoop.util.KMSUtilFaultInjector; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; +import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -63,7 +72,6 @@ import java.io.FileWriter; import java.io.IOException; import java.io.Writer; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketTimeoutException; import java.net.URI; @@ -81,17 +89,46 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class TestKMS { private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class); private static final String SSL_RELOADER_THREAD_NAME = "Truststore reloader thread"; + private final KMSUtilFaultInjector oldInjector = + KMSUtilFaultInjector.get(); + + // Injector to create providers with different ports. Can only happen in tests + private final KMSUtilFaultInjector testInjector = + new KMSUtilFaultInjector() { +@Override +public KeyProvider createKeyProviderForTests(String value, +Configuration conf) throws IOException { + return TestLoadBalancingKMSClientProvider + .createKeyProviderForTests(value, conf); +} + }; + @Rule public final Timeout testTimeout = new Timeout(18); @Before - public void cleanUp() { + public void setUp() throws Exception { +GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE); +GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE); // resetting kerberos security Configuration conf = new Configuration(); UserGroupInformation.setConfiguration(conf); @@ -111,17 +148,71 @@ public class TestKMS { } public static abstract class KMSCallable implements Callable { -private URL kmsUrl; +private List kmsUrl;