This is an automated email from the ASF dual-hosted git repository.
sadanand48 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new fb9f30b8869 HDDS-16057. OM bootstrap fails when checkpoint install is
rejected during BOOTSTRAPPING. (#10917)
fb9f30b8869 is described below
commit fb9f30b88694f8fcc876af786d38b86a2aa89d76
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Mon Aug 3 16:05:18 2026 +0530
HDDS-16057. OM bootstrap fails when checkpoint install is rejected during
BOOTSTRAPPING. (#10917)
Co-authored-by: Wei-Chiu Chuang <[email protected]>
---
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 138 +++++++++++++++++++++
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 16 ++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +-
3 files changed, 160 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
index 1925b6a204a..7de95f58bc2 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
@@ -28,6 +28,7 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
import java.io.File;
import java.io.IOException;
@@ -51,12 +52,14 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.utils.FaultInjector;
+import org.apache.hadoop.hdds.utils.RDBSnapshotProvider;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils;
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -72,6 +75,7 @@
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServerConfig;
+import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
@@ -93,8 +97,14 @@
*/
public class TestOMRatisSnapshots {
private static final String OM_SERVICE_ID = "om-service-test1";
+ private static final String BOOTSTRAP_OM_SERVICE_ID = "om-service-bootstrap";
private static final int NUM_OF_OMS = 3;
+ private static final int BOOTSTRAP_LOG_PURGE_GAP = 5;
+ private static final long BOOTSTRAP_TARGET_LOG_INDEX = 200;
+ private static final int BOOTSTRAP_INSTALL_START_DEADLINE_MS = 30_000;
+ private static final int BOOTSTRAP_COMPLETION_DEADLINE_MS = 60_000;
+
private MiniOzoneHAClusterImpl cluster = null;
private ObjectStore objectStore;
private OzoneConfiguration conf;
@@ -597,6 +607,134 @@ public void
testInstallSnapshotFromLeaderFailedDownloadCleanupSucceeds()
followerOM.getOmSnapshotProvider().setInjector(null);
}
+ /**
+ * Regression test for bootstrap when leader logs are purged: checkpoint
install
+ * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint
API
+ * and complete successfully.
+ */
+ @Test
+ public void testBootstrapInstallSnapshotDuringBootstrapping() throws
Exception {
+ IOUtils.closeQuietly(client);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+
+ OzoneConfiguration bootstrapConf = new OzoneConfiguration();
+
bootstrapConf.setInt(OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, 5);
+ bootstrapConf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP,
BOOTSTRAP_LOG_PURGE_GAP);
+
bootstrapConf.setLong(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
+ SNAPSHOT_THRESHOLD);
+ bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY,
16,
+ StorageUnit.KB);
+
bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
+ 16, StorageUnit.KB);
+
+ OzoneManagerRatisServerConfig omRatisConf =
+ bootstrapConf.getObject(OzoneManagerRatisServerConfig.class);
+ omRatisConf.setLogAppenderWaitTimeMin(10);
+ bootstrapConf.setFromObject(omRatisConf);
+
+ cluster = (MiniOzoneHAClusterImpl)
MiniOzoneCluster.newHABuilder(bootstrapConf)
+ .setOMServiceId(BOOTSTRAP_OM_SERVICE_ID)
+ .setNumOfOzoneManagers(2)
+ .setNumDatanodes(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+
+ client = OzoneClientFactory.getRpcClient(BOOTSTRAP_OM_SERVICE_ID,
bootstrapConf);
+ objectStore = client.getObjectStore();
+ String bootstrapVolume = uniqueObjectName("volume");
+ String bootstrapBucket = uniqueObjectName("bucket");
+ objectStore.createVolume(bootstrapVolume);
+ OzoneVolume volume = objectStore.getVolume(bootstrapVolume);
+ volume.createBucket(bootstrapBucket,
+ BucketArgs.newBuilder().setBucketLayout(TEST_BUCKET_LAYOUT).build());
+ ozoneBucket = volume.getBucket(bootstrapBucket);
+
+ OzoneManager leader = cluster.getOMLeader();
+ writeKeysToIncreaseLogIndex(leader.getOmRatisServer(),
BOOTSTRAP_TARGET_LOG_INDEX);
+ assertThat(leader.getRatisSnapshotIndex())
+ .as("leader should have purged early logs")
+ .isGreaterThan((long) BOOTSTRAP_LOG_PURGE_GAP);
+
+ LogCapturer omLog = LogCapturer.captureLogs(OzoneManager.class);
+ LogCapturer stateMachineLog =
+ LogCapturer.captureLogs(OzoneManagerStateMachine.class);
+ LogCapturer snapshotProviderLog =
+ LogCapturer.captureLogs(RDBSnapshotProvider.class);
+ String newNodeId = "omNode-bootstrap-ratis-snapshots";
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<?> bootstrapFuture = executor.submit(() -> {
+ try {
+ cluster.bootstrapOzoneManager(newNodeId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ try {
+ waitForBootstrapCheckpointInstallToStart(omLog, snapshotProviderLog);
+ bootstrapFuture.get(BOOTSTRAP_COMPLETION_DEADLINE_MS,
TimeUnit.MILLISECONDS);
+ assertBootstrapOmJoinedRatisGroup(newNodeId);
+ } finally {
+ bootstrapFuture.cancel(true);
+ omLog.stopCapturing();
+ stateMachineLog.stopCapturing();
+ snapshotProviderLog.stopCapturing();
+ executor.shutdownNow();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ assertThat(stateMachineLog.getOutput())
+ .as("Ratis should notify the bootstrapping OM to install a checkpoint")
+ .contains("Received install snapshot notification from OM leader");
+ assertThat(omLog.getOutput())
+ .as("checkpoint install must not be aborted during BOOTSTRAPPING")
+ .doesNotContain("Abort install snapshot from Leader");
+ assertThat(omLog.getOutput())
+ .as("checkpoint installation should finish")
+ .contains("Install Checkpoint is finished");
+ assertThat(snapshotProviderLog.getOutput())
+ .as("checkpoint download should start after install is accepted")
+ .contains("Prepare to download the snapshot from leader OM");
+ assertThat(snapshotProviderLog.getOutput())
+ .as("checkpoint tarball should be assembled on the bootstrapping OM")
+ .contains("DB snapshot transfer is complete.");
+ }
+
+ private void assertBootstrapOmJoinedRatisGroup(String newNodeId) {
+ OzoneManager newOm = cluster.getOzoneManager(newNodeId);
+ assertNotNull(newOm, "Bootstrapped OM should be registered on the
cluster");
+ for (OzoneManager om : cluster.getOzoneManagersList()) {
+ assertTrue(om.doesPeerExist(newNodeId),
+ "New OM node " + newNodeId + " not present in peer list of OM " +
om.getOMNodeId());
+ assertTrue(om.getOmRatisServer().doesPeerExist(newNodeId),
+ "New OM node " + newNodeId + " not present in Ratis peer list of OM "
+ + om.getOMNodeId());
+ }
+ }
+
+ private void waitForBootstrapCheckpointInstallToStart(
+ LogCapturer omLog,
+ LogCapturer snapshotProviderLog)
+ throws InterruptedException, TimeoutException {
+ try {
+ GenericTestUtils.waitFor(() -> {
+ if (omLog.getOutput().contains("Abort install snapshot from Leader")) {
+ fail("Checkpoint install was aborted during BOOTSTRAPPING.");
+ }
+ return snapshotProviderLog.getOutput()
+ .contains("Prepare to download the snapshot from leader OM");
+ }, 200, BOOTSTRAP_INSTALL_START_DEADLINE_MS);
+ } catch (TimeoutException e) {
+ fail("Checkpoint download did not start within " +
BOOTSTRAP_INSTALL_START_DEADLINE_MS
+ + "ms. OzoneManager log: " + omLog.getOutput()
+ + ", RDBSnapshotProvider log: " + snapshotProviderLog.getOutput());
+ }
+ }
+
/**
* Moves all contents from the checkpoint location into the omDbDir.
* This reorganizes the checkpoint structure so that all checkpoint files
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 507777d8bfe..8df0f587c60 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -881,10 +881,24 @@ private OzoneConfiguration addNewOMToConfig(String
omServiceId,
/**
* Update the configurations of the given list of OMs.
+ * Merges {@code newConf} with each OM's existing node-local storage paths so
+ * bootstrap peer updates do not clobber per-node {@code
ozone.metadata.dirs}.
*/
private void updateOMConfigs(OzoneConfiguration newConf) {
for (OzoneManager om : omhaService.getActiveServices()) {
- om.setConfiguration(newConf);
+ OzoneConfiguration merged = new OzoneConfiguration(newConf);
+ OzoneConfiguration current = om.getConfiguration();
+ copyConfigIfSet(current, merged, OZONE_METADATA_DIRS);
+ copyConfigIfSet(current, merged, OMConfigKeys.OZONE_OM_DB_DIRS);
+ om.setConfiguration(merged);
+ }
+ }
+
+ private static void copyConfigIfSet(OzoneConfiguration from,
+ OzoneConfiguration to, String key) {
+ String value = from.get(key);
+ if (StringUtils.isNotEmpty(value)) {
+ to.set(key, value);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 2db20c59f31..e15231dcfc2 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -4201,9 +4201,9 @@ public List<OzoneAcl> getAcl(OzoneObj obj) throws
IOException {
* @throws IOException if download or cleanup fails
*/
public synchronized TermIndex installSnapshotFromLeader(String leaderId)
throws IOException {
- if (!isRunning() || testInstallSnapshot) {
- LOG.warn("OzoneManager is not in running state, state {}. Abort install
snapshot from Leader.",
- omState);
+ if (!isRunningOrBootstrapping() || testInstallSnapshot) {
+ LOG.warn("OzoneManager is not in running state nor bootstrapping, state
{}. "
+ + "Abort install snapshot from Leader.", omState);
return null;
}
@@ -4249,6 +4249,10 @@ public synchronized TermIndex
installSnapshotFromLeader(String leaderId) throws
return termIndex;
}
+ private boolean isRunningOrBootstrapping() {
+ return omState == State.RUNNING || omState == State.BOOTSTRAPPING;
+ }
+
private void cleanupCheckpoint(DBCheckpoint omDBCheckpoint) throws
IOException {
if (omDBCheckpoint != null) {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
