[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/trunk e81397545 -> 583fa6ed4
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 1517b04..c171143 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -31,26 +32,35 @@ import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
+import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer;
import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider;
import org.apache.hadoop.crypto.key.kms.ValueQueue;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
import
org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.KMSUtilFaultInjector;
import org.apache.hadoop.util.Time;
import org.apache.http.client.utils.URIBuilder;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
@@ -71,7 +81,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -96,6 +105,10 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
+import static
org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -113,6 +126,20 @@ public class TestKMS {
private SSLFactory sslFactory;
+ private final KMSUtilFaultInjector oldInjector =
+ KMSUtilFaultInjector.get();
+
+ // Injector to create providers with different ports. Can only happen in
tests
+ private final KMSUtilFaultInjector testInjector =
+ new KMSUtilFaultInjector() {
+@Override
+public KeyProvider createKeyProviderForTests(String value,
+Configuration conf) throws IOException {
+ return TestLoadBalancingKMSClientProvider
+ .createKeyProviderForTests(value, conf);
+}
+ };
+
// Keep track of all key providers created during a test case, so they can be
// closed at test tearDown.
private List providersCreated = new LinkedList<>();
@@ -122,7 +149,12 @@ public class TestKMS {
@Before
public void setUp() throws Exception {
-setUpMiniKdc();
+GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE);
+GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE);
// r
[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0fb1457d8 -> 95cedc558
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95cedc55/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 712536f..b75eb76 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -30,19 +31,27 @@ import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
+import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer;
import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
import
org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.KMSUtilFaultInjector;
import org.apache.hadoop.util.Time;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -64,7 +73,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -82,7 +90,14 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
+import static
org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TestKMS {
private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class);
@@ -90,11 +105,31 @@ public class TestKMS {
private static final String SSL_RELOADER_THREAD_NAME =
"Truststore reloader thread";
+ private final KMSUtilFaultInjector oldInjector =
+ KMSUtilFaultInjector.get();
+
+ // Injector to create providers with different ports. Can only happen in
tests
+ private final KMSUtilFaultInjector testInjector =
+ new KMSUtilFaultInjector() {
+@Override
+public KeyProvider createKeyProviderForTests(String value,
+Configuration conf) throws IOException {
+ return TestLoadBalancingKMSClientProvider
+ .createKeyProviderForTests(value, conf);
+}
+ };
+
@Rule
public final Timeout testTimeout = new Timeout(18);
@Before
- public void cleanUp() {
+ public void setUp() throws Exception {
+GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE);
+GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE);
// resetting kerberos security
Configuration conf = new Configuration();
UserGroupInformation.setConfiguration(conf);
@@ -114,17 +149,71 @@ public class TestKMS {
}
public static abstract class KMSCallable implements Callable {
-private URL kmsUrl;
+private List
[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.9 87485d40c -> 46ac59a9b
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46ac59a9/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 712536f..b75eb76 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -30,19 +31,27 @@ import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
+import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer;
import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
import
org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.KMSUtilFaultInjector;
import org.apache.hadoop.util.Time;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -64,7 +73,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -82,7 +90,14 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
+import static
org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TestKMS {
private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class);
@@ -90,11 +105,31 @@ public class TestKMS {
private static final String SSL_RELOADER_THREAD_NAME =
"Truststore reloader thread";
+ private final KMSUtilFaultInjector oldInjector =
+ KMSUtilFaultInjector.get();
+
+ // Injector to create providers with different ports. Can only happen in
tests
+ private final KMSUtilFaultInjector testInjector =
+ new KMSUtilFaultInjector() {
+@Override
+public KeyProvider createKeyProviderForTests(String value,
+Configuration conf) throws IOException {
+ return TestLoadBalancingKMSClientProvider
+ .createKeyProviderForTests(value, conf);
+}
+ };
+
@Rule
public final Timeout testTimeout = new Timeout(18);
@Before
- public void cleanUp() {
+ public void setUp() throws Exception {
+GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE);
+GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE);
// resetting kerberos security
Configuration conf = new Configuration();
UserGroupInformation.setConfiguration(conf);
@@ -114,17 +149,71 @@ public class TestKMS {
}
public static abstract class KMSCallable implements Callable {
-private URL kmsUrl;
+private Lis
[1/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 5f8ab3a6b -> 714a079ff
http://git-wip-us.apache.org/repos/asf/hadoop/blob/714a079f/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 308c974..b67b8a1 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -30,20 +31,28 @@ import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
+import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer;
import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
import
org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.KMSUtilFaultInjector;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -63,7 +72,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -81,17 +89,46 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
+import static
org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
public class TestKMS {
private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class);
private static final String SSL_RELOADER_THREAD_NAME =
"Truststore reloader thread";
+ private final KMSUtilFaultInjector oldInjector =
+ KMSUtilFaultInjector.get();
+
+ // Injector to create providers with different ports. Can only happen in
tests
+ private final KMSUtilFaultInjector testInjector =
+ new KMSUtilFaultInjector() {
+@Override
+public KeyProvider createKeyProviderForTests(String value,
+Configuration conf) throws IOException {
+ return TestLoadBalancingKMSClientProvider
+ .createKeyProviderForTests(value, conf);
+}
+ };
+
@Rule
public final Timeout testTimeout = new Timeout(18);
@Before
- public void cleanUp() {
+ public void setUp() throws Exception {
+GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE);
+GenericTestUtils
+.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE);
+GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE);
// resetting kerberos security
Configuration conf = new Configuration();
UserGroupInformation.setConfiguration(conf);
@@ -111,17 +148,71 @@ public class TestKMS {
}
public static abstract class KMSCallable implements Callable {
-private URL kmsUrl;
+private List kmsUrl;
