This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 66e277386e HDDS-7572. Use keyManager and trustManager provided by
keyStoreFactory in datanode grpc services (#4032)
66e277386e is described below
commit 66e277386e5fadf5ee62184654a5b0f57aa2948b
Author: Sammi Chen <[email protected]>
AuthorDate: Mon Dec 12 10:15:27 2022 +0800
HDDS-7572. Use keyManager and trustManager provided by keyStoreFactory in
datanode grpc services (#4032)
---
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 2 +-
.../ozone/container/ContainerTestHelper.java | 18 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +
.../common/transport/server/XceiverServerGrpc.java | 4 +-
.../replication/GrpcReplicationClient.java | 16 +-
.../container/replication/ReplicationServer.java | 26 +--
.../replication/SimpleContainerDownloader.java | 7 +-
.../x509/certificate/client/CertificateClient.java | 13 +-
.../client/DefaultCertificateClient.java | 35 ++++
.../hdds/security/x509/keys/SecurityUtil.java | 35 ++++
.../hdds/security/x509/CertificateClientTest.java | 17 ++
.../ozone/client/CertificateClientTestImpl.java | 136 ++++++++++++---
.../ozoneimpl/TestOzoneContainerWithTLS.java | 185 +++++++++++++++++++--
13 files changed, 427 insertions(+), 75 deletions(-)
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index cab7bee780..1fee58c3b4 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -198,7 +198,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.intercept(new GrpcClientInterceptor());
- if (secConfig.isGrpcTlsEnabled()) {
+ if (secConfig.isSecurityEnabled() && secConfig.isGrpcTlsEnabled()) {
SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
if (caCerts != null) {
sslContextBuilder.trustManager(caCerts);
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index 09fff2371e..327e03d70d 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -470,19 +470,29 @@ public final class ContainerTestHelper {
* Returns a close container request.
* @param pipeline - pipeline
* @param containerID - ID of the container.
+ * @param token - container token
* @return ContainerCommandRequestProto.
*/
public static ContainerCommandRequestProto getCloseContainer(
- Pipeline pipeline, long containerID) throws IOException {
- return ContainerCommandRequestProto.newBuilder()
+ Pipeline pipeline, long containerID, Token<?> token) throws IOException {
+ Builder builder = ContainerCommandRequestProto.newBuilder()
.setCmdType(ContainerProtos.Type.CloseContainer)
.setContainerID(containerID)
.setCloseContainer(
ContainerProtos.CloseContainerRequestProto.getDefaultInstance())
- .setDatanodeUuid(pipeline.getFirstNode().getUuidString())
- .build();
+ .setDatanodeUuid(pipeline.getFirstNode().getUuidString());
+
+ if (token != null) {
+ builder.setEncodedToken(token.encodeToUrlString());
+ }
+
+ return builder.build();
}
+ public static ContainerCommandRequestProto getCloseContainer(
+ Pipeline pipeline, long containerID) throws IOException {
+ return getCloseContainer(pipeline, containerID, null);
+ }
/**
* Returns a simple request without traceId.
* @param pipeline - pipeline
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 45b453f4a7..2de1bde537 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -604,6 +604,14 @@ public class HddsDatanodeService extends GenericCli
implements ServicePlugin {
}
}
}
+
+ if (dnCertClient != null) {
+ try {
+ dnCertClient.close();
+ } catch (IOException e) {
+ LOG.warn("Certificate client could not be closed", e);
+ }
+ }
}
@VisibleForTesting
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index c1fc950792..528f4b8bd7 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -140,10 +140,10 @@ public final class XceiverServerGrpc implements
XceiverServerSpi {
new GrpcXceiverService(dispatcher), new GrpcServerInterceptor()));
SecurityConfig secConf = new SecurityConfig(conf);
- if (secConf.isGrpcTlsEnabled()) {
+ if (secConf.isSecurityEnabled() && secConf.isGrpcTlsEnabled()) {
try {
SslContextBuilder sslClientContextBuilder =
SslContextBuilder.forServer(
- caClient.getPrivateKey(), caClient.getCertificate());
+ caClient.getServerKeyStoresFactory().getKeyManagers()[0]);
SslContextBuilder sslContextBuilder = GrpcSslContexts.configure(
sslClientContextBuilder, secConf.getGrpcSslProvider());
nettyServerBuilder.sslContext(sslContextBuilder.build());
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 023b251a52..e1348614e7 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -31,9 +31,9 @@ import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContai
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CopyContainerResponseProto;
import
org.apache.hadoop.hdds.protocol.datanode.proto.IntraDatanodeProtocolServiceGrpc;
import
org.apache.hadoop.hdds.protocol.datanode.proto.IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceStub;
+import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import com.google.common.base.Preconditions;
@@ -60,10 +60,9 @@ public class GrpcReplicationClient implements AutoCloseable {
private final Path workingDirectory;
- public GrpcReplicationClient(
- String host, int port, Path workingDir,
- SecurityConfig secConfig, CertificateClient certClient
- ) throws IOException {
+ public GrpcReplicationClient(String host, int port, Path workingDir,
+ SecurityConfig secConfig, CertificateClient certClient)
+ throws IOException {
NettyChannelBuilder channelBuilder =
NettyChannelBuilder.forAddress(host, port)
.usePlaintext()
@@ -74,12 +73,11 @@ public class GrpcReplicationClient implements AutoCloseable
{
SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
if (certClient != null) {
+ KeyStoresFactory factory = certClient.getClientKeyStoresFactory();
sslContextBuilder
- .trustManager(HAUtils.buildCAX509List(certClient,
- secConfig.getConfiguration()))
+ .trustManager(factory.getTrustManagers()[0])
.clientAuth(ClientAuth.REQUIRE)
- .keyManager(certClient.getPrivateKey(),
- certClient.getCertificate());
+ .keyManager(factory.getKeyManagers()[0]);
}
if (secConfig.useTestCert()) {
channelBuilder.overrideAuthority("localhost");
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java
index bf8d6f1025..fcad690d4f 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.conf.PostConstruct;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.tracing.GrpcServerInterceptor;
-import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
@@ -61,12 +60,9 @@ public class ReplicationServer {
private int port;
- public ReplicationServer(
- ContainerController controller,
- ReplicationConfig replicationConfig,
- SecurityConfig secConf,
- CertificateClient caClient
- ) {
+ public ReplicationServer(ContainerController controller,
+ ReplicationConfig replicationConfig, SecurityConfig secConf,
+ CertificateClient caClient) {
this.secConf = secConf;
this.caClient = caClient;
this.controller = controller;
@@ -81,17 +77,17 @@ public class ReplicationServer {
new OnDemandContainerReplicationSource(controller)
), new GrpcServerInterceptor()));
- if (secConf.isSecurityEnabled()) {
+ if (secConf.isSecurityEnabled() && secConf.isGrpcTlsEnabled()) {
try {
SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(
- caClient.getPrivateKey(), caClient.getCertificate());
+ caClient.getServerKeyStoresFactory().getKeyManagers()[0]);
sslContextBuilder = GrpcSslContexts.configure(
sslContextBuilder, secConf.getGrpcSslProvider());
sslContextBuilder.clientAuth(ClientAuth.REQUIRE);
- sslContextBuilder.trustManager(HAUtils.buildCAX509List(caClient,
- secConf.getConfiguration()));
+ sslContextBuilder.trustManager(
+ caClient.getServerKeyStoresFactory().getTrustManagers()[0]);
nettyServerBuilder.sslContext(sslContextBuilder.build());
} catch (IOException ex) {
@@ -106,14 +102,8 @@ public class ReplicationServer {
public void start() throws IOException {
server.start();
-
- if (port == 0) {
- LOG.info("{} is started using port {}", getClass().getSimpleName(),
- server.getPort());
- }
-
port = server.getPort();
-
+ LOG.info("{} is started using port {}", getClass().getSimpleName(), port);
}
public void stop() {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
index 8e661635fe..18fe058728 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
@@ -47,16 +47,15 @@ import org.slf4j.LoggerFactory;
*/
public class SimpleContainerDownloader implements ContainerDownloader {
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(SimpleContainerDownloader.class);
private final Path workingDirectory;
private final SecurityConfig securityConfig;
private final CertificateClient certClient;
- public SimpleContainerDownloader(
- ConfigurationSource conf, CertificateClient certClient) {
-
+ public SimpleContainerDownloader(ConfigurationSource conf,
+ CertificateClient certClient) {
String workDirString =
conf.get(OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR);
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
index e2863b9426..ce7cb04855 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -20,10 +20,12 @@
package org.apache.hadoop.hdds.security.x509.certificate.client;
import org.apache.hadoop.hdds.security.OzoneSecurityException;
+import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
import
org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.security.PrivateKey;
@@ -39,7 +41,7 @@ import static
org.apache.hadoop.hdds.security.OzoneSecurityException.ResultCodes
* Certificate client provides and interface to certificate operations that
* needs to be performed by all clients in the Ozone eco-system.
*/
-public interface CertificateClient {
+public interface CertificateClient extends Closeable {
/**
* Returns the private key of the specified component if it exists on the
@@ -323,4 +325,13 @@ public interface CertificateClient {
*/
boolean processCrl(CRLInfo crl);
+ /**
+ * Return the store factory for key manager and trust manager for server.
+ */
+ KeyStoresFactory getServerKeyStoresFactory() throws CertificateException;
+
+ /**
+ * Return the store factory for key manager and trust manager for client.
+ */
+ KeyStoresFactory getClientKeyStoresFactory() throws CertificateException;
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
index a55db9a427..752113f7b3 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
@@ -51,6 +51,7 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
@@ -58,6 +59,7 @@ import
org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRe
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import com.google.common.base.Preconditions;
@@ -109,6 +111,8 @@ public abstract class DefaultCertificateClient implements
CertificateClient {
private String component;
private List<String> pemEncodedCACerts = null;
private final Lock lock;
+ private KeyStoresFactory serverKeyStoresFactory;
+ private KeyStoresFactory clientKeyStoresFactory;
DefaultCertificateClient(SecurityConfig securityConfig, Logger log,
String certSerialId, String component) {
@@ -1060,4 +1064,35 @@ public abstract class DefaultCertificateClient
implements CertificateClient {
public void setLocalCrlId(long crlId) {
this.localCrlId = crlId;
}
+
+ @Override
+ public synchronized KeyStoresFactory getServerKeyStoresFactory()
+ throws CertificateException {
+ if (serverKeyStoresFactory == null) {
+ serverKeyStoresFactory = SecurityUtil.getServerKeyStoresFactory(
+ securityConfig, this, true);
+ }
+ return serverKeyStoresFactory;
+ }
+
+ @Override
+ public KeyStoresFactory getClientKeyStoresFactory()
+ throws CertificateException {
+ if (clientKeyStoresFactory == null) {
+ clientKeyStoresFactory = SecurityUtil.getClientKeyStoresFactory(
+ securityConfig, this, true);
+ }
+ return clientKeyStoresFactory;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (serverKeyStoresFactory != null) {
+ serverKeyStoresFactory.destroy();
+ }
+
+ if (clientKeyStoresFactory != null) {
+ clientKeyStoresFactory.destroy();
+ }
+ }
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
index 6147d3a990..5085b6f42e 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.hadoop.hdds.security.x509.keys;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
import java.security.KeyFactory;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
@@ -26,8 +28,13 @@ import java.security.PublicKey;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
+
+import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
+import org.apache.hadoop.hdds.security.ssl.PemFileBasedKeyStoresFactory;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1Sequence;
import org.bouncycastle.asn1.ASN1Set;
@@ -135,4 +142,32 @@ public final class SecurityUtil {
return key;
}
+ public static KeyStoresFactory getServerKeyStoresFactory(
+ SecurityConfig securityConfig, CertificateClient client,
+ boolean requireClientAuth) throws CertificateException {
+ PemFileBasedKeyStoresFactory factory =
+ new PemFileBasedKeyStoresFactory(securityConfig, client);
+ try {
+ factory.init(SSLFactory.Mode.SERVER, requireClientAuth);
+ } catch (IOException | GeneralSecurityException e) {
+ throw new CertificateException("Failed to init keyStoresFactory", e,
+ CertificateException.ErrorCode.KEYSTORE_ERROR);
+ }
+ return factory;
+ }
+
+ public static KeyStoresFactory getClientKeyStoresFactory(
+ SecurityConfig securityConfig, CertificateClient client,
+ boolean requireClientAuth) throws CertificateException {
+ PemFileBasedKeyStoresFactory factory =
+ new PemFileBasedKeyStoresFactory(securityConfig, client);
+
+ try {
+ factory.init(SSLFactory.Mode.CLIENT, requireClientAuth);
+ } catch (IOException | GeneralSecurityException e) {
+ throw new CertificateException("Failed to init keyStoresFactory", e,
+ CertificateException.ErrorCode.KEYSTORE_ERROR);
+ }
+ return factory;
+ }
}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
index 5003bf11c5..f1bbaff47b 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
@@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import
org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
@@ -217,6 +218,18 @@ public class CertificateClientTest implements
CertificateClient {
return false;
}
+ @Override
+ public KeyStoresFactory getServerKeyStoresFactory()
+ throws CertificateException {
+ return null;
+ }
+
+ @Override
+ public KeyStoresFactory getClientKeyStoresFactory()
+ throws CertificateException {
+ return null;
+ }
+
@Override
public boolean isCertificateRenewed() {
return isKeyRenewed;
@@ -231,4 +244,8 @@ public class CertificateClientTest implements
CertificateClient {
x509Certificate = newCert;
isKeyRenewed = true;
}
+
+ @Override
+ public void close() throws IOException {
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
index 253d45355a..87221827e2 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
@@ -25,11 +25,16 @@ import java.security.cert.CertStore;
import java.security.cert.X509Certificate;
import java.time.Duration;
import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.util.Collections;
+import java.util.Date;
import java.util.List;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover;
+import
org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import
org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
import
org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
@@ -37,11 +42,14 @@ import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
/**
* Test implementation for CertificateClient. To be used only for test
@@ -50,10 +58,17 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_D
public class CertificateClientTestImpl implements CertificateClient {
- private final SecurityConfig securityConfig;
- private final KeyPair keyPair;
private final OzoneConfiguration config;
- private final X509Certificate x509Certificate;
+ private final SecurityConfig securityConfig;
+ private KeyPair keyPair;
+ private X509Certificate x509Certificate;
+ private final KeyPair rootKeyPair;
+ private final X509Certificate rootCert;
+ private HDDSKeyGenerator keyGen;
+ private DefaultApprover approver;
+ private KeyStoresFactory serverKeyStoresFactory;
+ private KeyStoresFactory clientKeyStoresFactory;
+ private boolean isKeyRenewed = false;
public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception {
this(conf, true);
@@ -62,32 +77,59 @@ public class CertificateClientTestImpl implements
CertificateClient {
public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA)
throws Exception {
securityConfig = new SecurityConfig(conf);
- HDDSKeyGenerator keyGen =
- new HDDSKeyGenerator(securityConfig.getConfiguration());
+ keyGen = new HDDSKeyGenerator(securityConfig.getConfiguration());
keyPair = keyGen.generateKey();
+ rootKeyPair = keyGen.generateKey();
config = conf;
LocalDateTime start = LocalDateTime.now();
- String certDurationString = conf.get(HDDS_X509_DEFAULT_DURATION,
- HDDS_X509_DEFAULT_DURATION_DEFAULT);
- Duration certDuration = Duration.parse(certDurationString);
- LocalDateTime end = start.plus(certDuration);
+ String rootCACertDuration = conf.get(HDDS_X509_MAX_DURATION,
+ HDDS_X509_MAX_DURATION_DEFAULT);
+ LocalDateTime end = start.plus(Duration.parse(rootCACertDuration));
+ // Generate RootCA certificate
SelfSignedCertificate.Builder builder =
SelfSignedCertificate.newBuilder()
.setBeginDate(start)
.setEndDate(end)
.setClusterID("cluster1")
- .setKey(keyPair)
- .setSubject("localhost")
+ .setKey(rootKeyPair)
+ .setSubject("rootCA@localhost")
.setConfiguration(config)
- .setScmID("TestScmId1");
- if (rootCA) {
- builder.makeCA();
- }
- X509CertificateHolder certificateHolder = null;
- certificateHolder = builder.build();
- x509Certificate = new JcaX509CertificateConverter().getCertificate(
- certificateHolder);
+ .setScmID("scm1")
+ .makeCA();
+ rootCert = new JcaX509CertificateConverter().getCertificate(
+ builder.build());
+
+ // Generate normal certificate, signed by RootCA certificate
+ approver = new DefaultApprover(new DefaultProfile(), securityConfig);
+
+ CertificateSignRequest.Builder csrBuilder = getCSRBuilder();
+ // Get host name.
+ csrBuilder.setKey(keyPair)
+ .setConfiguration(config)
+ .setScmID("scm1")
+ .setClusterID("cluster1")
+ .setSubject("localhost")
+ .setDigitalSignature(true)
+ .setDigitalEncryption(true);
+
+ start = LocalDateTime.now();
+ String certDuration = conf.get(HDDS_X509_DEFAULT_DURATION,
+ HDDS_X509_DEFAULT_DURATION_DEFAULT);
+ X509CertificateHolder certificateHolder =
+ approver.sign(securityConfig, rootKeyPair.getPrivate(),
+ new X509CertificateHolder(rootCert.getEncoded()),
+ Date.from(start.atZone(ZoneId.systemDefault()).toInstant()),
+ Date.from(start.plus(Duration.parse(certDuration))
+ .atZone(ZoneId.systemDefault()).toInstant()),
+ csrBuilder.build(), "scm1", "cluster1");
+ x509Certificate =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+
+ serverKeyStoresFactory = SecurityUtil.getServerKeyStoresFactory(
+ securityConfig, this, true);
+ clientKeyStoresFactory = SecurityUtil.getClientKeyStoresFactory(
+ securityConfig, this, true);
}
@Override
@@ -119,7 +161,7 @@ public class CertificateClientTestImpl implements
CertificateClient {
@Override
public X509Certificate getCACertificate() {
- return x509Certificate;
+ return rootCert;
}
@Override
@@ -255,4 +297,58 @@ public class CertificateClientTestImpl implements
CertificateClient {
return false;
}
+ public boolean isCertificateRenewed() {
+ return isKeyRenewed;
+ }
+
+ public void renewKey() throws Exception {
+ KeyPair newKeyPair = keyGen.generateKey();
+ CertificateSignRequest.Builder csrBuilder = getCSRBuilder();
+ // Get host name.
+ csrBuilder.setKey(newKeyPair)
+ .setConfiguration(config)
+ .setScmID("scm1")
+ .setClusterID("cluster1")
+ .setSubject("localhost")
+ .setDigitalSignature(true);
+
+ String certDuration = config.get(HDDS_X509_DEFAULT_DURATION,
+ HDDS_X509_DEFAULT_DURATION_DEFAULT);
+ Date start = new Date();
+ X509CertificateHolder certificateHolder =
+ approver.sign(securityConfig, rootKeyPair.getPrivate(),
+ new X509CertificateHolder(rootCert.getEncoded()), start,
+ new Date(start.getTime() +
Duration.parse(certDuration).toMillis()),
+ csrBuilder.build(), "scm1", "cluster1");
+ X509Certificate newX509Certificate =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+
+ // Save the new private key and certificate to file
+ // Save certificate and private key to keyStore
+ keyPair = newKeyPair;
+ x509Certificate = newX509Certificate;
+ isKeyRenewed = true;
+ System.out.println(new Date() + " certificated is renewed");
+ }
+
+ @Override
+ public KeyStoresFactory getServerKeyStoresFactory() {
+ return serverKeyStoresFactory;
+ }
+
+ @Override
+ public KeyStoresFactory getClientKeyStoresFactory() {
+ return clientKeyStoresFactory;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (serverKeyStoresFactory != null) {
+ serverKeyStoresFactory.destroy();
+ }
+
+ if (clientKeyStoresFactory != null) {
+ clientKeyStoresFactory.destroy();
+ }
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
index f410f4d7fe..5bd5fa4a33 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
@@ -37,6 +39,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.ozone.test.GenericTestUtils;
@@ -53,16 +56,22 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.nio.file.Path;
import java.security.cert.CertificateExpiredException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
+import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_SSL_KEYSTORE_RELOAD_INTERVAL;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_SSL_TRUSTSTORE_RELOAD_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
@@ -120,17 +129,18 @@ public class TestOzoneContainerWithTLS {
conf.setBoolean(HddsConfigKeys.HDDS_GRPC_TLS_TEST_CERT, true);
conf.setInt(HDDS_KEY_LEN, 1024);
- conf.set(HDDS_X509_DEFAULT_DURATION, "PT5S"); // 5s
+ // certificate lives for 5s
+ conf.set(HDDS_X509_DEFAULT_DURATION, "PT5S");
+ conf.set(HDDS_SECURITY_SSL_KEYSTORE_RELOAD_INTERVAL, "1s");
+ conf.set(HDDS_SECURITY_SSL_TRUSTSTORE_RELOAD_INTERVAL, "1s");
long expiryTime = conf.getTimeDuration(
- HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME,
- HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT,
+ HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, "1s",
TimeUnit.MILLISECONDS);
caClient = new CertificateClientTestImpl(conf, false);
secretManager = new ContainerTokenSecretManager(new SecurityConfig(conf),
- expiryTime, caClient.getCertificate().
- getSerialNumber().toString());
+ expiryTime, caClient.getCertificate().getSerialNumber().toString());
}
@Test(expected = CertificateExpiredException.class)
@@ -170,13 +180,13 @@ public class TestOzoneContainerWithTLS {
if (containerTokenEnabled) {
secretManager.start(caClient);
client.connect();
- createSecureContainerForTesting(client, containerId,
+ createSecureContainer(client, containerId,
secretManager.generateToken(
UserGroupInformation.getCurrentUser().getUserName(),
ContainerID.valueOf(containerId)));
} else {
client.connect();
- createContainerForTesting(client, containerId);
+ createContainer(client, containerId);
}
} finally {
if (container != null) {
@@ -185,27 +195,170 @@ public class TestOzoneContainerWithTLS {
}
}
- public static void createContainerForTesting(XceiverClientSpi client,
+ @Test
+ public void testContainerDownload() throws Exception {
+ DatanodeDetails dn = MockDatanodeDetails.createDatanodeDetails(
+ UUID.randomUUID().toString(), "localhost", "0.0.0.0",
+ "/default-rack");
+ Pipeline pipeline = MockPipeline.createSingleNodePipeline();
+ conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.newFolder().getPath());
+ conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ pipeline.getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE)
+ .getValue());
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
+
+ OzoneContainer container = null;
+ try {
+ container = new OzoneContainer(dn, conf, getContext(dn), caClient);
+
+ // Set scmId and manually start ozone container.
+ container.start(UUID.randomUUID().toString());
+
+ if (containerTokenEnabled) {
+ secretManager.start(caClient);
+ }
+
+ // Create containers
+ long containerId = ContainerTestHelper.getTestContainerID();
+ int count = 5;
+ List<Long> containerIdList = new ArrayList<>();
+ XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf,
+ Collections.singletonList(caClient.getCACertificate()));
+ client.connect();
+ for (int i = 0; i < count; i++, containerId++) {
+ if (containerTokenEnabled) {
+ Token<ContainerTokenIdentifier> token = secretManager.generateToken(
+ UserGroupInformation.getCurrentUser().getUserName(),
+ ContainerID.valueOf(containerId));
+ createSecureContainer(client, containerId, token);
+ closeSecureContainer(client, containerId, token);
+ } else {
+ createContainer(client, containerId);
+ closeContainer(client, containerId);
+ }
+ containerIdList.add(containerId);
+ }
+
+ // Wait certificate to expire
+ GenericTestUtils.waitFor(() ->
+ caClient.getCertificate().getNotAfter().before(new Date()),
+ 500, 5000);
+
+ List<DatanodeDetails> sourceDatanodes = new ArrayList<>();
+ sourceDatanodes.add(dn);
+ if (containerTokenEnabled) {
+ // old client still function well after certificate expired
+ Token<ContainerTokenIdentifier> token = secretManager.generateToken(
+ UserGroupInformation.getCurrentUser().getUserName(),
+ ContainerID.valueOf(containerId));
+ createSecureContainer(client, containerId, token);
+ closeSecureContainer(client, containerId++, token);
+ } else {
+ createContainer(client, containerId);
+ closeContainer(client, containerId++);
+ }
+
+ // Download newly created container will fail because of cert expired
+ GenericTestUtils.LogCapturer logCapture = GenericTestUtils.LogCapturer
+ .captureLogs(SimpleContainerDownloader.LOG);
+ SimpleContainerDownloader downloader =
+ new SimpleContainerDownloader(conf, caClient);
+ Path file = downloader.getContainerDataFromReplicas(
+ containerId, sourceDatanodes);
+ downloader.close();
+ Assert.assertNull(file);
+ Assert.assertTrue(logCapture.getOutput().contains(
+ "java.security.cert.CertificateExpiredException"));
+
+ // Renew the certificate
+ caClient.renewKey();
+
+ // old client still function well after certificate renewed
+ if (containerTokenEnabled) {
+ Token<ContainerTokenIdentifier> token = secretManager.generateToken(
+ UserGroupInformation.getCurrentUser().getUserName(),
+ ContainerID.valueOf(containerId));
+ createSecureContainer(client, containerId, token);
+ closeSecureContainer(client, containerId++, token);
+ }
+
+ // Wait keyManager and trustManager to reload
+ Thread.sleep(2000);
+
+ // old client still function well after certificate reload
+ if (containerTokenEnabled) {
+ Token<ContainerTokenIdentifier> token = secretManager.generateToken(
+ UserGroupInformation.getCurrentUser().getUserName(),
+ ContainerID.valueOf(containerId));
+ createSecureContainer(client, containerId, token);
+ closeSecureContainer(client, containerId++, token);
+ } else {
+ createContainer(client, containerId);
+ closeContainer(client, containerId++);
+ }
+
+ // Download container should succeed after key and cert renewed
+ for (Long cId : containerIdList) {
+ downloader = new SimpleContainerDownloader(conf, caClient);
+ try {
+ file = downloader.getContainerDataFromReplicas(cId, sourceDatanodes);
+ downloader.close();
+ Assert.assertNotNull(file);
+ } finally {
+ if (downloader != null) {
+ downloader.close();
+ }
+ client.close();
+ }
+ }
+ } finally {
+ if (container != null) {
+ container.stop();
+ }
+ }
+ }
+
+ public static void createContainer(XceiverClientSpi client,
long containerID) throws Exception {
- ContainerProtos.ContainerCommandRequestProto request =
- ContainerTestHelper.getCreateContainerRequest(
- containerID, client.getPipeline());
- ContainerProtos.ContainerCommandResponseProto response =
- client.sendCommand(request);
+ ContainerCommandRequestProto request = ContainerTestHelper
+ .getCreateContainerRequest(containerID, client.getPipeline());
+ ContainerCommandResponseProto response = client.sendCommand(request);
Assert.assertNotNull(response);
+ Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS);
}
- public static void createSecureContainerForTesting(XceiverClientSpi client,
+ public static void createSecureContainer(XceiverClientSpi client,
long containerID, Token<ContainerTokenIdentifier> token)
throws Exception {
- ContainerProtos.ContainerCommandRequestProto request =
+ ContainerCommandRequestProto request =
ContainerTestHelper.getCreateContainerSecureRequest(
containerID, client.getPipeline(), token);
- ContainerProtos.ContainerCommandResponseProto response =
+ ContainerCommandResponseProto response =
client.sendCommand(request);
Assert.assertNotNull(response);
+ Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS);
+ }
+
+ public static void closeContainer(XceiverClientSpi client,
+ long containerID) throws Exception {
+ ContainerCommandRequestProto request = ContainerTestHelper
+ .getCloseContainer(client.getPipeline(), containerID);
+ ContainerCommandResponseProto response = client.sendCommand(request);
+ Assert.assertNotNull(response);
+ Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS);
}
+ public static void closeSecureContainer(XceiverClientSpi client,
+ long containerID, Token<ContainerTokenIdentifier> token)
+ throws Exception {
+ ContainerCommandRequestProto request =
+ ContainerTestHelper.getCloseContainer(client.getPipeline(),
+ containerID, token);
+ ContainerCommandResponseProto response =
+ client.sendCommand(request);
+ Assert.assertNotNull(response);
+ Assert.assertTrue(response.getResult() == ContainerProtos.Result.SUCCESS);
+ }
private StateContext getContext(DatanodeDetails datanodeDetails) {
DatanodeStateMachine stateMachine = Mockito.mock(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
