This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-7391-ca-cert-rot
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-7391-ca-cert-rot by this
push:
new 3c15fd407d HDDS-8694. Add SCM HA aware Root CA certificate monitor
task (#4792)
3c15fd407d is described below
commit 3c15fd407d937775b98100c7e5413ab61f0fb8d1
Author: Sammi Chen <[email protected]>
AuthorDate: Wed Jun 7 14:51:29 2023 +0800
HDDS-8694. Add SCM HA aware Root CA certificate monitor task (#4792)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 41 ++++
.../common/src/main/resources/ozone-default.xml | 18 ++
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 2 +
.../{crl => security}/CRLStatusReportHandler.java | 2 +-
.../hdds/scm/security/RootCARotationManager.java | 272 +++++++++++++++++++++
.../hdds/scm/{crl => security}/package-info.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 25 +-
.../TestCRLStatusReportHandler.java | 2 +-
.../scm/security/TestRootCARotationManager.java | 214 ++++++++++++++++
.../dist/src/main/compose/ozonesecure-ha/test.sh | 4 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 3 +
.../ozoneimpl/TestOzoneContainerWithTLS.java | 2 +
13 files changed, 593 insertions(+), 5 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 2ace9ad49f..4cad18fac6 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -203,6 +203,15 @@ public final class HddsConfigKeys {
public static final String HDDS_X509_RENEW_GRACE_DURATION_DEFAULT = "P28D";
public static final String HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX = "-next";
public static final String HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX =
"-previous";
+ public static final String HDDS_X509_CA_ROTATION_CHECK_INTERNAL =
+ "hdds.x509.ca.rotation.check.interval";
+ public static final String HDDS_X509_CA_ROTATION_CHECK_INTERNAL_DEFAULT =
+ "P1D";
+ public static final String HDDS_X509_CA_ROTATION_TIME_OF_DAY =
+ "hdds.x509.ca.rotation.time-of-day";
+ // format hh:mm:ss, representing hour, minute, and second
+ public static final String HDDS_X509_CA_ROTATION_TIME_OF_DAY_DEFAULT =
+ "02:00:00";
public static final String HDDS_CONTAINER_REPLICATION_COMPRESSION =
"hdds.container.replication.compression";
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 6a8a91e7b2..519b3a4d16 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -25,6 +25,8 @@ import java.security.Provider;
import java.security.Security;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -38,6 +40,10 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL_DEFAULT;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_OF_DAY;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_OF_DAY_DEFAULT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PRIVATE_KEY_FILE;
@@ -116,6 +122,10 @@ public class SecurityConfig {
private final String externalRootCaPublicKeyPath;
private final String externalRootCaPrivateKeyPath;
private final String externalRootCaCert;
+ private final Duration caCheckInterval;
+ private final String caRotationTimeOfDay;
+ private final Pattern caRotationTimeOfDayPattern =
+ Pattern.compile("\\d{2}:\\d{2}:\\d{2}");
/**
* Constructs a SecurityConfig.
@@ -180,6 +190,23 @@ public class SecurityConfig {
HDDS_X509_RENEW_GRACE_DURATION_DEFAULT);
renewalGracePeriod = Duration.parse(renewalGraceDurationString);
+ String caCheckIntervalString = configuration.get(
+ HDDS_X509_CA_ROTATION_CHECK_INTERNAL,
+ HDDS_X509_CA_ROTATION_CHECK_INTERNAL_DEFAULT);
+ caCheckInterval = Duration.parse(caCheckIntervalString);
+
+ String timeOfDayString = configuration.get(
+ HDDS_X509_CA_ROTATION_TIME_OF_DAY,
+ HDDS_X509_CA_ROTATION_TIME_OF_DAY_DEFAULT);
+
+ Matcher matcher = caRotationTimeOfDayPattern.matcher(timeOfDayString);
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Property value of " +
+ HDDS_X509_CA_ROTATION_TIME_OF_DAY +
+ " should follow the hh:mm:ss format.");
+ }
+ caRotationTimeOfDay = "1970-01-01T" + timeOfDayString;
+
validateCertificateValidityConfig();
this.externalRootCaCert = this.configuration.get(
@@ -244,6 +271,12 @@ public class SecurityConfig {
LOG.error(msg);
throw new IllegalArgumentException(msg);
}
+
+ if (caCheckInterval.compareTo(renewalGracePeriod) >= 0) {
+ throw new IllegalArgumentException("Property value of " +
+ HDDS_X509_CA_ROTATION_CHECK_INTERNAL +
+ " should be smaller than " + HDDS_X509_RENEW_GRACE_DURATION);
+ }
}
/**
@@ -450,6 +483,14 @@ public class SecurityConfig {
return externalRootCaCert;
}
+ public Duration getCaCheckInterval() {
+ return caCheckInterval;
+ }
+
+ public String getCaRotationTimeOfDay() {
+ return caRotationTimeOfDay;
+ }
+
/**
* Return true if using test certificates with authority as localhost. This
* should be used only for unit test where certificates are generated by
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0de5c43034..e6d43a15bf 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2234,6 +2234,24 @@
they can be configured separately.
</description>
</property>
+ <property>
+ <name>hdds.x509.ca.rotation.check.interval</name>
+ <value>P1D</value>
+ <tag>OZONE, HDDS, SECURITY</tag>
+ <description>Check interval of whether internal root certificate is going
+ to expire and needs to start rotation or not. Default is 1 day. The
property
+ value should be less than the value of property
hdds.x509.renew.grace.duration.
+ </description>
+ </property>
+ <property>
+ <name>hdds.x509.ca.rotation.time-of-day</name>
+ <value>02:00:00</value>
+ <tag>OZONE, HDDS, SECURITY</tag>
+ <description>Time of day to start the rotation. Default 02:00 AM to avoid
impacting
+ daily workload. The supported format is 'hh:mm:ss', representing hour,
minute,
+ and second.
+ </description>
+ </property>
<property>
<name>ozone.scm.security.handler.count.key</name>
<value>2</value>
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java
index a95e12c828..0b856fd2c3 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.util.ServicePlugin;
import org.apache.commons.io.FileUtils;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION;
import static org.apache.hadoop.ozone.HddsDatanodeService.getLogger;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
@@ -99,6 +100,7 @@ public class TestHddsSecureDatanodeInit {
TestHddsDatanodeService.MockService.class,
ServicePlugin.class);
conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT5S"); // 5s
+ conf.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "PT1S"); // 1s
securityConfig = new SecurityConfig(conf);
service = HddsDatanodeService.createHddsDatanodeService(args);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl/CRLStatusReportHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/CRLStatusReportHandler.java
similarity index 98%
rename from
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl/CRLStatusReportHandler.java
rename to
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/CRLStatusReportHandler.java
index 35c5825da3..69cb9a7c34 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl/CRLStatusReportHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/CRLStatusReportHandler.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdds.scm.crl;
+package org.apache.hadoop.hdds.scm.security;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
new file mode 100644
index 0000000000..33f44146a1
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.security;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.ha.SCMService;
+import org.apache.hadoop.hdds.scm.ha.SCMServiceException;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Root CA Rotation Manager is a service in SCM to control the CA rotation.
+ */
+public class RootCARotationManager implements SCMService {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(RootCARotationManager.class);
+
+ private StorageContainerManager scm;
+ private final SCMContext scmContext;
+ private OzoneConfiguration ozoneConf;
+ private SecurityConfig secConf;
+ private ScheduledExecutorService executorService;
+ private Duration checkInterval;
+ private Duration renewalGracePeriod;
+ private Date timeOfDay;
+ private CertificateClient scmCertClient;
+ private AtomicBoolean isRunning = new AtomicBoolean(false);
+ private AtomicBoolean isScheduled = new AtomicBoolean(false);
+ private String threadName = this.getClass().getSimpleName();
+
+ /**
+ * Constructs RootCARotationManager with the specified arguments.
+ *
+ * @param scm the storage container manager
+ */
+ public RootCARotationManager(StorageContainerManager scm) {
+ this.scm = scm;
+ this.ozoneConf = scm.getConfiguration();
+ this.secConf = new SecurityConfig(ozoneConf);
+ this.scmContext = scm.getScmContext();
+
+ checkInterval = secConf.getCaCheckInterval();
+ timeOfDay = Date.from(LocalDateTime.parse(secConf.getCaRotationTimeOfDay())
+ .atZone(ZoneId.systemDefault()).toInstant());
+ renewalGracePeriod = secConf.getRenewalGracePeriod();
+
+ executorService = Executors.newScheduledThreadPool(1,
+ new ThreadFactoryBuilder().setNameFormat(threadName)
+ .setDaemon(true).build());
+
+ scmCertClient = scm.getScmCertificateClient();
+ scm.getSCMServiceManager().register(this);
+ }
+
+ /**
+ * Receives a notification for raft or safe mode related status changes.
+ * Stops monitor task if it's running and the current SCM becomes a
+ * follower or enter safe mode. Starts monitor if the current SCM becomes
+ * leader and not in safe mode.
+ */
+ @Override
+ public void notifyStatusChanged() {
+ // stop the monitor task
+ if (!scmContext.isLeader() || scmContext.isInSafeMode()) {
+ if (isRunning.compareAndSet(true, false)) {
+ LOG.info("notifyStatusChanged: disable monitor task.");
+ }
+ return;
+ }
+
+ if (isRunning.compareAndSet(false, true)) {
+ LOG.info("notifyStatusChanged: enable monitor task");
+ }
+ return;
+ }
+
+ /**
+ * Checks if monitor task should start (after a leader change, restart
+ * etc.) by reading persisted state.
+ * @return true if the persisted state is true, otherwise false
+ */
+ @Override
+ public boolean shouldRun() {
+ return true;
+ }
+
+ /**
+ * @return Name of this service.
+ */
+ @Override
+ public String getServiceName() {
+ return RootCARotationManager.class.getSimpleName();
+ }
+
+ /**
+ * Schedule monitor task.
+ */
+ @Override
+ public void start() throws SCMServiceException {
+ executorService.scheduleAtFixedRate(
+ new MonitorTask(scmCertClient), 0, checkInterval.toMillis(),
+ TimeUnit.MILLISECONDS);
+ LOG.info("Monitor task for root certificate {} is started with " +
+ "interval {}.", scmCertClient.getCACertificate().getSerialNumber(),
+ checkInterval);
+ }
+
+ public boolean isRunning() {
+ return isRunning.get();
+ }
+
+ /**
+ * Task to monitor certificate lifetime and start rotation if needed.
+ */
+ public class MonitorTask implements Runnable {
+ private CertificateClient certClient;
+
+ public MonitorTask(CertificateClient client) {
+ this.certClient = client;
+ }
+
+ @Override
+ public void run() {
+ Thread.currentThread().setName(threadName +
+ (isRunning() ? "-Active" : "-Inactive"));
+ if (!isRunning.get() || isScheduled.get()) {
+ return;
+ }
+ // Lock to protect the root CA certificate rotation process,
+ // to make sure there is only one task is ongoing at one time.
+ synchronized (RootCARotationManager.class) {
+ X509Certificate rootCACert = certClient.getCACertificate();
+ Duration timeLeft = timeBefore2ExpiryGracePeriod(rootCACert);
+ if (timeLeft.isZero()) {
+ LOG.info("Root certificate {} has entered the 2 * expiry" +
+ " grace period({}).",
rootCACert.getSerialNumber().toString(),
+ renewalGracePeriod);
+ // schedule root CA rotation task
+ LocalDateTime now = LocalDateTime.now();
+ LocalDateTime timeToSchedule = LocalDateTime.of(
+ now.getYear(), now.getMonthValue(), now.getDayOfMonth(),
+ timeOfDay.getHours(), timeOfDay.getMinutes(),
+ timeOfDay.getSeconds());
+ if (timeToSchedule.isBefore(now)) {
+ timeToSchedule = timeToSchedule.plusDays(1);
+ }
+ long delay = Duration.between(now, timeToSchedule).toMillis();
+ if (timeToSchedule.isAfter(rootCACert.getNotAfter().toInstant()
+ .atZone(ZoneId.systemDefault()).toLocalDateTime())) {
+ LOG.info("Configured rotation time {} is after root" +
+ " certificate {} end time {}. Start the rotation immediately.",
+ timeToSchedule, rootCACert.getSerialNumber().toString(),
+ rootCACert.getNotAfter());
+ delay = 0;
+ }
+
+ executorService.schedule(new RotationTask(certClient), delay,
+ TimeUnit.MILLISECONDS);
+ isScheduled.set(true);
+ LOG.info("Root certificate {} rotation task is scheduled with {} ms "
+ + "delay", rootCACert.getSerialNumber().toString(), delay);
+ }
+ }
+ }
+ }
+
+ /**
+ * Task to rotate root certificate.
+ */
+ public class RotationTask implements Runnable {
+ private CertificateClient certClient;
+
+ public RotationTask(CertificateClient client) {
+ this.certClient = client;
+ }
+
+ @Override
+ public void run() {
+ isScheduled.set(false);
+ if (!isRunning.get()) {
+ return;
+ }
+ // Lock to protect the root CA certificate rotation process,
+ // to make sure there is only one task is ongoing at one time.
+ // Root CA rotation steps:
+ // 1. generate new root CA keys and certificate, persist to disk
+ // 2. start new Root CA server
+ // 3. send scm Sub-CA rotation preparation request through RATIS
+ // 4. send scm Sub-CA rotation commit request through RATIS
+ // 5. send scm Sub-CA rotation finish request through RATIS
+ synchronized (RootCARotationManager.class) {
+ X509Certificate rootCACert = certClient.getCACertificate();
+ Duration timeLeft = timeBefore2ExpiryGracePeriod(rootCACert);
+ if (timeLeft.isZero()) {
+ LOG.info("Root certificate {} rotation is started.",
+ rootCACert.getSerialNumber().toString());
+ // TODO: start the root CA rotation process
+ } else {
+ LOG.warn("Root certificate {} hasn't entered the 2 * expiry" +
+ " grace period {}. Skip root certificate rotation this
time.",
+ rootCACert.getSerialNumber().toString(), renewalGracePeriod);
+ }
+ }
+ }
+ }
+
+ /**
+ * Calculate time before root certificate will enter 2 * expiry grace period.
+ * @return Duration, time before certificate enters the 2 * grace
+ * period defined by "hdds.x509.renew.grace.duration"
+ */
+ public Duration timeBefore2ExpiryGracePeriod(X509Certificate certificate) {
+ LocalDateTime gracePeriodStart = certificate.getNotAfter().toInstant()
+ .minus(renewalGracePeriod).minus(renewalGracePeriod)
+ .atZone(ZoneId.systemDefault()).toLocalDateTime();
+ LocalDateTime currentTime = LocalDateTime.now();
+ if (gracePeriodStart.isBefore(currentTime)) {
+ // Cert is already in grace period time.
+ return Duration.ZERO;
+ } else {
+ return Duration.between(currentTime, gracePeriodStart);
+ }
+ }
+
+ /**
+ * Stops scheduled monitor task.
+ */
+ @Override
+ public void stop() {
+ try {
+ executorService.shutdown();
+ if (!executorService.awaitTermination(3, TimeUnit.SECONDS)) {
+ executorService.shutdownNow();
+ }
+ } catch (InterruptedException ie) {
+ // Ignore, we don't really care about the failure.
+ Thread.currentThread().interrupt();
+ }
+ }
+}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl/package-info.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/package-info.java
similarity index 89%
rename from
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl/package-info.java
rename to
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/package-info.java
index 997aac2b3c..118c9dc6d6 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl/package-info.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/package-info.java
@@ -17,6 +17,6 @@
*/
/**
- * This package contains CRL related classes.
+ * This package contains security related classes.
*/
-package org.apache.hadoop.hdds.scm.crl;
+package org.apache.hadoop.hdds.scm.security;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index e6880add77..681625463c 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -50,7 +50,8 @@ import
org.apache.hadoop.hdds.scm.container.balancer.MoveManager;
import
org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingOps;
import
org.apache.hadoop.hdds.scm.container.replication.DatanodeCommandCountUpdatedHandler;
import
org.apache.hadoop.hdds.scm.container.replication.LegacyReplicationManager;
-import org.apache.hadoop.hdds.scm.crl.CRLStatusReportHandler;
+import org.apache.hadoop.hdds.scm.ha.SCMServiceException;
+import org.apache.hadoop.hdds.scm.security.CRLStatusReportHandler;
import org.apache.hadoop.hdds.scm.ha.BackgroundSCMService;
import org.apache.hadoop.hdds.scm.ha.HASecurityUtils;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
@@ -67,6 +68,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
+import org.apache.hadoop.hdds.scm.security.RootCARotationManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManagerImpl;
import org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManager;
@@ -269,6 +271,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
private SCMSafeModeManager scmSafeModeManager;
private CertificateClient scmCertificateClient;
+ private RootCARotationManager rootCARotationManager;
private ContainerTokenSecretManager containerTokenMgr;
private JvmPauseMonitor jvmPauseMonitor;
@@ -882,6 +885,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
if (securityConfig.isContainerTokenEnabled()) {
containerTokenMgr = createContainerTokenSecretManager(configuration);
}
+ rootCARotationManager = new RootCARotationManager(this);
}
/** Persist primary SCM root ca cert and sub-ca certs to DB.
@@ -1502,6 +1506,14 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
persistSCMCertificates();
}
+ if (rootCARotationManager != null) {
+ try {
+ rootCARotationManager.start();
+ } catch (SCMServiceException e) {
+ throw new IOException("Failed to start root CA rotation manager", e);
+ }
+ }
+
scmBlockManager.start();
leaseManager.start();
@@ -1623,6 +1635,10 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
getSecurityProtocolServer().stop();
}
+ if (rootCARotationManager != null) {
+ rootCARotationManager.stop();
+ }
+
try {
LOG.info("Stopping Block Manager Service.");
scmBlockManager.stop();
@@ -1834,6 +1850,13 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
return moveManager;
}
+ /**
+ * Returns SCM root CA rotation manager.
+ */
+ public RootCARotationManager getRootCARotationManager() {
+ return rootCARotationManager;
+ }
+
/**
* Check if the current scm is the leader and ready for accepting requests.
* @return - if the current scm is the leader and is ready.
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/crl/TestCRLStatusReportHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestCRLStatusReportHandler.java
similarity index 99%
rename from
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/crl/TestCRLStatusReportHandler.java
rename to
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestCRLStatusReportHandler.java
index 91c9dbc09a..e5b206728f 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/crl/TestCRLStatusReportHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestCRLStatusReportHandler.java
@@ -14,7 +14,7 @@
* License for the specific language governing permissions and limitations
under
* the License.
*/
-package org.apache.hadoop.hdds.scm.crl;
+package org.apache.hadoop.hdds.scm.security;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
new file mode 100644
index 0000000000..16c2b45606
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.security;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.container.TestContainerManagerImpl;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.ha.SCMServiceManager;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import
org.apache.hadoop.hdds.security.x509.certificate.utils.SelfSignedCertificate;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.ozone.test.GenericTestUtils;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeParseException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_OF_DAY;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION;
+import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.when;
+import static org.slf4j.event.Level.INFO;
+
+/**
+ * Test for root CA rotation manager.
+ */
+public class TestRootCARotationManager {
+
+ private OzoneConfiguration ozoneConfig;
+ private RootCARotationManager rootCARotationManager;
+ private StorageContainerManager scm;
+ private CertificateClient scmCertClient;
+ private SCMServiceManager scmServiceManager;
+ private SCMContext scmContext;
+ private File testDir;
+
+ @BeforeEach
+ public void init() throws IOException, TimeoutException {
+ ozoneConfig = new OzoneConfiguration();
+ testDir = GenericTestUtils.getTestDir(
+ TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
+ ozoneConfig.set(HddsConfigKeys.OZONE_METADATA_DIRS,
+ testDir.getAbsolutePath());
+ scm = Mockito.mock(StorageContainerManager.class);
+ scmCertClient = Mockito.mock(CertificateClient.class);
+ scmServiceManager = new SCMServiceManager();
+ scmContext = Mockito.mock(SCMContext.class);
+ when(scmContext.isLeader()).thenReturn(true);
+ when(scm.getConfiguration()).thenReturn(ozoneConfig);
+ when(scm.getScmCertificateClient()).thenReturn(scmCertClient);
+ when(scm.getScmContext()).thenReturn(scmContext);
+ when(scm.getSCMServiceManager()).thenReturn(scmServiceManager);
+ }
+
+ @AfterEach
+ public void tearDown() throws Exception {
+ if (rootCARotationManager != null) {
+ rootCARotationManager.stop();
+ }
+
+ FileUtil.fullyDelete(testDir);
+ }
+
+ @Test
+ public void testProperties() {
+ // invalid check interval
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "P28");
+ try {
+ rootCARotationManager = new RootCARotationManager(scm);
+ fail("Should fail");
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof DateTimeParseException);
+ }
+
+ // check interval should be less than grace period
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "P28D");
+ try {
+ rootCARotationManager = new RootCARotationManager(scm);
+ fail("Should fail");
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof IllegalArgumentException);
+ Assertions.assertTrue(e.getMessage().contains("should be smaller than"));
+ }
+
+ // invalid time of day format
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "P1D");
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_TIME_OF_DAY, "01:00");
+ try {
+ rootCARotationManager = new RootCARotationManager(scm);
+ fail("Should fail");
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof IllegalArgumentException);
+ Assertions.assertTrue(
+ e.getMessage().contains("should follow the hh:mm:ss format"));
+ }
+
+ // valid properties
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "P1D");
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_TIME_OF_DAY, "01:00:00");
+
+ try {
+ rootCARotationManager = new RootCARotationManager(scm);
+ } catch (Exception e) {
+ fail("Should succeed");
+ }
+ }
+
+ @Test
+ public void testRotationOnSchedule() throws Exception {
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "PT1S");
+ ozoneConfig.set(HDDS_X509_RENEW_GRACE_DURATION, "PT15S");
+ Date date = Calendar.getInstance().getTime();
+ date.setSeconds(date.getSeconds() + 10);
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_TIME_OF_DAY,
+ String.format("%02d", date.getHours()) + ":" +
+ String.format("%02d", date.getMinutes()) + ":" +
+ String.format("%02d", date.getSeconds()));
+
+ X509Certificate cert = generateX509Cert(ozoneConfig,
+ LocalDateTime.now(), Duration.ofSeconds(35));
+ when(scmCertClient.getCACertificate()).thenReturn(cert);
+
+ rootCARotationManager = new RootCARotationManager(scm);
+ GenericTestUtils.LogCapturer logs =
+ GenericTestUtils.LogCapturer.captureLogs(RootCARotationManager.LOG);
+ GenericTestUtils.setLogLevel(RootCARotationManager.LOG, INFO);
+ rootCARotationManager.start();
+ rootCARotationManager.notifyStatusChanged();
+
+ String msg = "Root certificate " +
+ cert.getSerialNumber().toString() + " rotation is started.";
+ GenericTestUtils.waitFor(
+ () -> !logs.getOutput().contains("Start the rotation immediately") &&
+ logs.getOutput().contains(msg),
+ 100, 10000);
+ assertEquals(1, StringUtils.countMatches(logs.getOutput(), msg));
+ }
+
+ @Test
+ public void testRotationImmediately() throws Exception {
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "PT1S");
+ ozoneConfig.set(HDDS_X509_RENEW_GRACE_DURATION, "PT15S");
+ Date date = Calendar.getInstance().getTime();
+ date.setMinutes(date.getMinutes() + 5);
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_TIME_OF_DAY,
+ String.format("%02d", date.getHours()) + ":" +
+ String.format("%02d", date.getMinutes()) + ":" +
+ String.format("%02d", date.getSeconds()));
+
+ X509Certificate cert = generateX509Cert(ozoneConfig,
+ LocalDateTime.now(), Duration.ofSeconds(35));
+ when(scmCertClient.getCACertificate()).thenReturn(cert);
+
+ rootCARotationManager = new RootCARotationManager(scm);
+ GenericTestUtils.LogCapturer logs =
+ GenericTestUtils.LogCapturer.captureLogs(RootCARotationManager.LOG);
+ GenericTestUtils.setLogLevel(RootCARotationManager.LOG, INFO);
+ rootCARotationManager.start();
+ rootCARotationManager.notifyStatusChanged();
+
+ GenericTestUtils.waitFor(
+ () -> logs.getOutput().contains("Start the rotation immediately") &&
+ logs.getOutput().contains("Root certificate " +
+ cert.getSerialNumber().toString() + " rotation is started."),
+ 100, 10000);
+ }
+
+ private X509Certificate generateX509Cert(
+ OzoneConfiguration conf, LocalDateTime startDate,
+ Duration certLifetime) throws Exception {
+ KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ LocalDateTime start = startDate == null ? LocalDateTime.now() : startDate;
+ LocalDateTime end = start.plus(certLifetime);
+ return new JcaX509CertificateConverter().getCertificate(
+ SelfSignedCertificate.newBuilder().setBeginDate(start)
+ .setEndDate(end).setClusterID("cluster").setKey(keyPair)
+ .setSubject("localhost").setConfiguration(conf).setScmID("test")
+ .build());
+ }
+}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
index 10d71c08cf..9fe32168e3 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
@@ -51,7 +51,11 @@ done
execute_robot_test s3g admincli
execute_robot_test s3g omha/om-leader-transfer.robot
+
+# verify root CA rotation monitor task
+wait_for_execute_command scm1.org 30 "jps | grep
StorageContainerManagerStarter | awk -F' ' '{print $1}' | xargs -I {} jstack {}
| grep 'RootCARotationManager-MonitorTask-Active'"
execute_robot_test s3g scmha/scm-leader-transfer.robot
+wait_for_execute_command scm1.org 30 "jps | grep
StorageContainerManagerStarter | awk -F' ' '{print $1}' | xargs -I {} jstack {}
| grep 'RootCARotationManager-MonitorTask-Inactive'"
execute_robot_test s3g httpfs
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 16c1cb62d8..0e8d2c882c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -118,6 +118,7 @@ import org.apache.commons.lang3.StringUtils;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT;
@@ -251,6 +252,8 @@ public final class TestSecureOzoneCluster {
conf.set(OZONE_METADATA_DIRS, omMetaDirPath.toString());
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name());
+ conf.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL,
+ Duration.ofMillis(certGraceTime - 1000).toString());
conf.set(HDDS_X509_RENEW_GRACE_DURATION,
Duration.ofMillis(certGraceTime).toString());
conf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
index 22d55992cf..7d26663ea8 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
@@ -73,6 +73,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
@@ -138,6 +139,7 @@ public class TestOzoneContainerWithTLS {
conf.set(HDDS_X509_DEFAULT_DURATION,
Duration.ofMillis(certLifetime).toString());
conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT2S");
+ conf.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "PT1S"); // 1s
long expiryTime = conf.getTimeDuration(
HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, "1s",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
