yandrey321 commented on code in PR #9779:
URL: https://github.com/apache/ozone/pull/9779#discussion_r2819039707
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java:
##########
@@ -116,97 +110,97 @@ public void
postFinalizeUpgrade(SCMUpgradeFinalizationContext context)
context.getFinalizationStateManager();
if (!stateManager.crossedCheckpoint(
FinalizationCheckpoint.FINALIZATION_COMPLETE)) {
- createPipelinesAfterFinalization(context);
+ waitForDatanodesToFinalize(context);
stateManager.removeFinalizingMark();
}
}
- @Override
- public void runPrefinalizeStateActions(Storage storage,
- SCMUpgradeFinalizationContext context) throws IOException {
- super.runPrefinalizeStateActions(
- lf -> ((HDDSLayoutFeature) lf)::scmAction, storage, context);
- }
-
- private void closePipelinesBeforeFinalization(PipelineManager
pipelineManager)
- throws IOException {
- /*
- * Before we can call finalize the feature, we need to make sure that
- * all existing pipelines are closed and pipeline Manger would freeze
- * all new pipeline creation.
- */
- String msg = " Existing pipelines and containers will be closed " +
- "during Upgrade.";
- msg += "\n New pipelines creation will remain frozen until Upgrade " +
- "is finalized.";
-
- // Pipeline creation should already be frozen when the finalization state
- // manager set the checkpoint.
- if (!pipelineManager.isPipelineCreationFrozen()) {
- throw new SCMException("Error during finalization. Pipeline creation" +
- "should have been frozen before closing existing pipelines.",
- SCMException.ResultCodes.INTERNAL_ERROR);
- }
-
- for (Pipeline pipeline : pipelineManager.getPipelines()) {
- if (pipeline.getPipelineState() != CLOSED) {
- pipelineManager.closePipeline(pipeline.getId());
- }
- }
-
- // We can not yet move all the existing data nodes to HEALTHY-READONLY
- // state since the next heartbeat will move them back to HEALTHY state.
- // This has to wait till postFinalizeUpgrade, when SCM MLV version is
- // already upgraded as part of finalize processing.
- // While in this state, it should be safe to do finalize processing for
- // all new features. This will also update ondisk mlv version. Any
- // disrupting upgrade can add a hook here to make sure that SCM is in a
- // consistent state while finalizing the upgrade.
-
- logAndEmit(msg);
- }
+ /**
+ * Wait for all HEALTHY datanodes to complete finalization before finishing
+ * SCM finalization. This ensures that when the client receives a
+ * FINALIZATION_DONE status, all healthy datanodes have also finalized.
+ *
+ * A datanode is considered finalized when its metadata layout version (MLV)
+ * equals its software layout version (SLV), indicating it has completed
+ * processing all layout features.
+ *
+ * @param context The finalization context containing node manager reference
+ * @throws SCMException if waiting is interrupted or SCM loses leadership
+ * @throws NotLeaderException if SCM is no longer the leader
+ */
+ private void waitForDatanodesToFinalize(SCMUpgradeFinalizationContext
context)
+ throws SCMException, NotLeaderException {
+ NodeManager nodeManager = context.getNodeManager();
- private void createPipelinesAfterFinalization(
- SCMUpgradeFinalizationContext context) throws SCMException,
- NotLeaderException {
- // Pipeline creation should already be resumed when the finalization state
- // manager set the checkpoint.
- PipelineManager pipelineManager = context.getPipelineManager();
- if (pipelineManager.isPipelineCreationFrozen()) {
- throw new SCMException("Error during finalization. Pipeline creation " +
- "should have been resumed before waiting for new pipelines.",
- SCMException.ResultCodes.INTERNAL_ERROR);
- }
+ LOG.info("Waiting for all HEALTHY datanodes to complete finalization
before finishing SCM finalization.");
- // Wait for at least one pipeline to be created before finishing
- // finalization, so clients can write.
- boolean hasPipeline = false;
- while (!hasPipeline) {
+ boolean allDatanodesFinalized = false;
+ while (!allDatanodesFinalized) {
// Break out of the wait and step down from driving finalization if this
// SCM is no longer the leader by throwing NotLeaderException.
context.getSCMContext().getTermOfLeader();
- ReplicationConfig ratisThree =
- ReplicationConfig.fromProtoTypeAndFactor(
- HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.THREE);
- int pipelineCount =
- pipelineManager.getPipelines(ratisThree, Pipeline.PipelineState.OPEN)
- .size();
-
- hasPipeline = (pipelineCount >= 1);
- if (!hasPipeline) {
- LOG.info("Waiting for at least one open Ratis 3 pipeline after SCM " +
- "finalization.");
+ allDatanodesFinalized = true;
+ int totalHealthyNodes = 0;
+ int finalizedNodes = 0;
+ int unfinalizedNodes = 0;
+
+ for (DatanodeDetails dn : nodeManager.getAllNodes()) {
+ try {
+ // Only check HEALTHY nodes. STALE/DEAD nodes will be told to
+ // finalize when they recover.
+ if (nodeManager.getNodeStatus(dn).isHealthy()) {
+ totalHealthyNodes++;
+ DatanodeInfo datanodeInfo = nodeManager.getDatanodeInfo(dn);
+ if (datanodeInfo == null) {
+ LOG.warn("Could not get DatanodeInfo for {}, skipping in " +
+ "finalization wait.", dn.getHostName());
+ continue;
+ }
+
+ LayoutVersionProto dnLayout =
datanodeInfo.getLastKnownLayoutVersion();
+ int dnMlv = dnLayout.getMetadataLayoutVersion();
+ int dnSlv = dnLayout.getSoftwareLayoutVersion();
+
+ if (dnMlv < dnSlv) {
+ // Datanode has not yet finalized
+ allDatanodesFinalized = false;
+ unfinalizedNodes++;
+ LOG.debug("Datanode {} has not yet finalized: MLV={}, SLV={}",
+ dn.getHostName(), dnMlv, dnSlv);
+ } else {
+ finalizedNodes++;
+ }
+ }
+ } catch (NodeNotFoundException e) {
+ // Node was removed while we were iterating. This is OK, skip it.
+ LOG.debug("Node {} not found while waiting for finalization, " +
+ "skipping.", dn);
+ }
+ }
+
+ if (!allDatanodesFinalized) {
+ LOG.info("Waiting for datanodes to finalize. Status: {}/{} healthy " +
+ "datanodes have finalized ({} remaining).",
+ finalizedNodes, totalHealthyNodes, unfinalizedNodes);
Review Comment:
its would be nice to have a ids for unhealthy nodes and unfinalizedNodes.
Also it would be nice to have a time in wait state.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
