errose28 commented on code in PR #11925:
URL: https://github.com/apache/ozone/pull/11925#discussion_r4516610438
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java:
##########
@@ -339,6 +356,68 @@ public void testUpgradeExceptionDuringApplyTerminates()
throws Exception {
}
}
+ /**
+ * Raft log entries written by the pre-ZDU finalization must still apply
after the upgrade, for example on an SCM
+ * that was down while the other SCMs finalized. They finalize up to their
layout version, but not beyond it.
+ */
+ @Test
+ public void
testPreZduFinalizationEntriesFinalizeUpToTheirLayoutVersion(@TempDir Path
tempDir) throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, tempDir.toString());
+ SCMStorageConfig storage = new SCMStorageConfig(conf);
+ storage.setScmId("scm");
+ storage.setApparentVersion(HBASE_SUPPORT.serialize());
+ storage.initialize();
+ ExitUtils.disableSystemExit();
+ try (ScmVersionManager versionManager =
+ new ScmVersionManager(storage, mock(StorageContainerManager.class),
HashMap::new)) {
+ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
+ @SuppressWarnings("unchecked")
+ Table<String, String> finalizationStore = mock(Table.class);
+ FinalizationStateManager finalizationStateManager = new
FinalizationStateManagerTestImpl.Builder()
+ .setFinalizationStore(finalizationStore)
+ .setTransactionBuffer(buffer)
+ .setVersionManager(versionManager)
+ .build();
+ SCMStateMachine applyingStateMachine = newStateMachine(buffer,
RequestType.FINALIZE,
+ new FinalizationStateManagerInvoker(finalizationStateManager,
mock(SCMRatisServer.class)));
+
+ // The entries a pre-ZDU leader writes to finalize from HBASE_SUPPORT,
then one of them replayed.
+ long index = 0;
+ for (SCMRatisRequest request : Arrays.asList(
+ finalizeRequest("addFinalizingMark"),
+ finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()),
+ finalizeRequest("finalizeLayoutFeature",
STORAGE_SPACE_DISTRIBUTION.serialize()),
+ finalizeRequest("removeFinalizingMark"),
+ finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()))) {
+
assertThat(applyingStateMachine.applyTransaction(newTransaction(request,
++index)))
+ .isCompletedWithValue(Message.EMPTY);
+ }
+
assertThat(versionManager.getApparentVersion()).isEqualTo(STORAGE_SPACE_DISTRIBUTION);
+
assertThat(versionManager.getPersistedApparentVersion()).isEqualTo(STORAGE_SPACE_DISTRIBUTION.serialize());
+ verify(buffer, times(2))
+ .addToBuffer(finalizationStore, APPARENT_VERSION_KEY,
String.valueOf(STORAGE_SPACE_DISTRIBUTION.serialize()));
Review Comment:
The condition being tested here is not clear without a comment IMO.
Additionally, a stricter `InOrder` enforces this more than just a count.
```suggestion
InOrder order = inOrder(buffer);
order.verify(buffer).addToBuffer(finalizationStore,
APPARENT_VERSION_KEY,
String.valueOf(WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()));
// Every apply will call finalize upgrade and write the resulting
apparent version to the DB.
// When an older apparent version is encountered, finalize will be a
no-op, but the current apparent version is
// still added to the buffer.
order.verify(buffer, times(2))
.addToBuffer(finalizationStore, APPARENT_VERSION_KEY,
String.valueOf(STORAGE_SPACE_DISTRIBUTION.serialize()));
```
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java:
##########
@@ -339,6 +356,68 @@ public void testUpgradeExceptionDuringApplyTerminates()
throws Exception {
}
}
+ /**
+ * Raft log entries written by the pre-ZDU finalization must still apply
after the upgrade, for example on an SCM
+ * that was down while the other SCMs finalized. They finalize up to their
layout version, but not beyond it.
+ */
+ @Test
+ public void
testPreZduFinalizationEntriesFinalizeUpToTheirLayoutVersion(@TempDir Path
tempDir) throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, tempDir.toString());
+ SCMStorageConfig storage = new SCMStorageConfig(conf);
+ storage.setScmId("scm");
+ storage.setApparentVersion(HBASE_SUPPORT.serialize());
+ storage.initialize();
+ ExitUtils.disableSystemExit();
+ try (ScmVersionManager versionManager =
+ new ScmVersionManager(storage, mock(StorageContainerManager.class),
HashMap::new)) {
+ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
+ @SuppressWarnings("unchecked")
+ Table<String, String> finalizationStore = mock(Table.class);
+ FinalizationStateManager finalizationStateManager = new
FinalizationStateManagerTestImpl.Builder()
+ .setFinalizationStore(finalizationStore)
+ .setTransactionBuffer(buffer)
+ .setVersionManager(versionManager)
+ .build();
+ SCMStateMachine applyingStateMachine = newStateMachine(buffer,
RequestType.FINALIZE,
+ new FinalizationStateManagerInvoker(finalizationStateManager,
mock(SCMRatisServer.class)));
+
+ // The entries a pre-ZDU leader writes to finalize from HBASE_SUPPORT,
then one of them replayed.
+ long index = 0;
+ for (SCMRatisRequest request : Arrays.asList(
+ finalizeRequest("addFinalizingMark"),
+ finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()),
+ finalizeRequest("finalizeLayoutFeature",
STORAGE_SPACE_DISTRIBUTION.serialize()),
+ finalizeRequest("removeFinalizingMark"),
+ finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()))) {
+
assertThat(applyingStateMachine.applyTransaction(newTransaction(request,
++index)))
+ .isCompletedWithValue(Message.EMPTY);
+ }
Review Comment:
The comment about the log entry being replayed is not really accurate. On
startup, old logs that could be replayed onto the state machine are fenced by
the Ratis apply index written to the DB. They will never reach the
`applyTransaction` path this PR changed. A replay would also have an identical
transaction index. Here we are creating a new transaction that happens after
the previous ones as ordered by Ratis, but has an old layout version. We should
still make sure this unexpected case is handled safely, just update the comment
to explain what is actually being tested.
Which logs to replay is handled by Ratis so we don't need to test that again
here. In the original issue, the logs being applied from the old version to the
new version were never applied to the state machine to begin with.
```suggestion
long index = 0;
for (SCMRatisRequest request : Arrays.asList(
finalizeRequest("addFinalizingMark"),
finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()),
finalizeRequest("finalizeLayoutFeature",
STORAGE_SPACE_DISTRIBUTION.serialize()),
finalizeRequest("removeFinalizingMark"),
// Test that a commit with an older layout version does not move
the apparent version backwards.
finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()))) {
assertThat(applyingStateMachine.applyTransaction(newTransaction(request,
++index)))
.isCompletedWithValue(Message.EMPTY);
}
```
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java:
##########
@@ -339,6 +356,68 @@ public void testUpgradeExceptionDuringApplyTerminates()
throws Exception {
}
}
+ /**
+ * Raft log entries written by the pre-ZDU finalization must still apply
after the upgrade, for example on an SCM
+ * that was down while the other SCMs finalized. They finalize up to their
layout version, but not beyond it.
+ */
+ @Test
+ public void
testPreZduFinalizationEntriesFinalizeUpToTheirLayoutVersion(@TempDir Path
tempDir) throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, tempDir.toString());
+ SCMStorageConfig storage = new SCMStorageConfig(conf);
+ storage.setScmId("scm");
+ storage.setApparentVersion(HBASE_SUPPORT.serialize());
+ storage.initialize();
+ ExitUtils.disableSystemExit();
+ try (ScmVersionManager versionManager =
+ new ScmVersionManager(storage, mock(StorageContainerManager.class),
HashMap::new)) {
+ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
+ @SuppressWarnings("unchecked")
+ Table<String, String> finalizationStore = mock(Table.class);
+ FinalizationStateManager finalizationStateManager = new
FinalizationStateManagerTestImpl.Builder()
+ .setFinalizationStore(finalizationStore)
+ .setTransactionBuffer(buffer)
+ .setVersionManager(versionManager)
+ .build();
+ SCMStateMachine applyingStateMachine = newStateMachine(buffer,
RequestType.FINALIZE,
+ new FinalizationStateManagerInvoker(finalizationStateManager,
mock(SCMRatisServer.class)));
+
+ // The entries a pre-ZDU leader writes to finalize from HBASE_SUPPORT,
then one of them replayed.
+ long index = 0;
+ for (SCMRatisRequest request : Arrays.asList(
+ finalizeRequest("addFinalizingMark"),
+ finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()),
+ finalizeRequest("finalizeLayoutFeature",
STORAGE_SPACE_DISTRIBUTION.serialize()),
+ finalizeRequest("removeFinalizingMark"),
+ finalizeRequest("finalizeLayoutFeature",
WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()))) {
+
assertThat(applyingStateMachine.applyTransaction(newTransaction(request,
++index)))
+ .isCompletedWithValue(Message.EMPTY);
+ }
+
assertThat(versionManager.getApparentVersion()).isEqualTo(STORAGE_SPACE_DISTRIBUTION);
+
assertThat(versionManager.getPersistedApparentVersion()).isEqualTo(STORAGE_SPACE_DISTRIBUTION.serialize());
+ verify(buffer, times(2))
+ .addToBuffer(finalizationStore, APPARENT_VERSION_KEY,
String.valueOf(STORAGE_SPACE_DISTRIBUTION.serialize()));
+
+
assertThat(applyingStateMachine.applyTransaction(newTransaction(finalizeRequest("finalizeUpgrade"),
++index)))
+ .isCompletedWithValue(Message.EMPTY);
+
assertThat(versionManager.getApparentVersion()).isEqualTo(SOFTWARE_VERSION);
+
+ final long unknownIndex = ++index;
+ assertThrows(ExitUtils.ExitException.class,
+ () ->
applyingStateMachine.applyTransaction(newTransaction(finalizeRequest("unknownMethod"),
unknownIndex)));
+
+ ExitUtils.clear();
+ final long unknownLayoutIndex = ++index;
+ SCMRatisRequest unknownLayoutRequest =
+ finalizeRequest("finalizeLayoutFeature",
STORAGE_SPACE_DISTRIBUTION.serialize() + 1);
+ ExitUtils.ExitException unknownLayout =
assertThrows(ExitUtils.ExitException.class,
+ () ->
applyingStateMachine.applyTransaction(newTransaction(unknownLayoutRequest,
unknownLayoutIndex)));
+ assertThat(unknownLayout).hasMessageContaining("Unknown pre-ZDU layout
version 11");
Review Comment:
These are sort of "meta tests" to make sure that our test setup would have
failed if there was a compat issue. This is good to have, but the intent would
be clearer with a comment:
```
// Verify that the previous cases would have failed and caught a
compatibility issue if the request type or
// layout feature was not known.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]