This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9a8007b718 HDDS-9898. Recon SCM doesn't store EC containers. (#5812)
9a8007b718 is described below
commit 9a8007b7188f8506b5f34fd3832594460c916905
Author: Arafat2198 <[email protected]>
AuthorDate: Thu Jan 4 14:19:34 2024 +0530
HDDS-9898. Recon SCM doesn't store EC containers. (#5812)
---
.../ozone/recon/scm/ReconContainerManager.java | 33 ++++++++++++----------
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +++++
.../ozone/recon/api/TestOpenContainerCount.java | 8 ++++++
.../ozone/recon/scm/TestReconContainerManager.java | 24 ++++++++++++++++
4 files changed, 57 insertions(+), 15 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
index d1d8373a29..e4108c2834 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
@@ -218,31 +218,34 @@ public class ReconContainerManager extends
ContainerManagerImpl {
/**
* Adds a new container to Recon's container manager.
+ *
* @param containerWithPipeline containerInfo with pipeline info
* @throws IOException on Error.
*/
public void addNewContainer(ContainerWithPipeline containerWithPipeline)
throws IOException {
+ ReconPipelineManager reconPipelineManager = (ReconPipelineManager)
pipelineManager;
ContainerInfo containerInfo = containerWithPipeline.getContainerInfo();
try {
if (containerInfo.getState().equals(HddsProtos.LifeCycleState.OPEN)) {
PipelineID pipelineID = containerWithPipeline.getPipeline().getId();
- if (pipelineManager.containsPipeline(pipelineID)) {
- getContainerStateManager().addContainer(containerInfo.getProtobuf());
- pipelineManager.addContainerToPipeline(
- containerWithPipeline.getPipeline().getId(),
- containerInfo.containerID());
- // update open container count on all datanodes on this pipeline
- pipelineToOpenContainer.put(pipelineID,
- pipelineToOpenContainer.getOrDefault(pipelineID, 0) + 1);
- LOG.info("Successfully added container {} to Recon.",
- containerInfo.containerID());
- } else {
- // Get open container for a pipeline that Recon does not know
- // about yet. Cannot update internal state until pipeline is synced.
- LOG.warn("Pipeline {} not found. Cannot add container {}",
- pipelineID, containerInfo.containerID());
+ // Check if the pipeline is present in Recon
+ if (!pipelineManager.containsPipeline(pipelineID)) {
+ // Pipeline is not present, add it first.
+ LOG.info("Adding new pipeline {} from SCM.", pipelineID);
+
reconPipelineManager.addPipeline(containerWithPipeline.getPipeline());
}
+
+ getContainerStateManager().addContainer(containerInfo.getProtobuf());
+ pipelineManager.addContainerToPipeline(
+ containerWithPipeline.getPipeline().getId(),
+ containerInfo.containerID());
+ // update open container count on all datanodes on this pipeline
+ pipelineToOpenContainer.put(pipelineID,
+ pipelineToOpenContainer.getOrDefault(pipelineID, 0) + 1);
+ LOG.info("Successfully added container {} to Recon.",
+ containerInfo.containerID());
+
} else {
getContainerStateManager().addContainer(containerInfo.getProtobuf());
LOG.info("Successfully added no open container {} to Recon.",
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
index d376f8793c..497bf51a2d 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
@@ -49,6 +49,7 @@ import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
@@ -79,6 +80,7 @@ import org.apache.hadoop.ozone.recon.common.CommonUtils;
import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
+import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
@@ -183,6 +185,8 @@ public class TestEndpoints extends AbstractReconSqlDBTest {
private ContainerHealthSchemaManager containerHealthSchemaManager;
private CommonUtils commonUtils;
+ private PipelineManager pipelineManager;
+ private ReconPipelineManager reconPipelineManager;
private void initializeInjector() throws Exception {
reconOMMetadataManager = getTestReconOmMetadataManager(
@@ -293,6 +297,9 @@ public class TestEndpoints extends AbstractReconSqlDBTest {
metricsProxyEndpoint =
new MetricsProxyEndpoint(metricsServiceProviderFactory);
dslContext = getDslContext();
+ pipelineManager = reconScm.getPipelineManager();
+ reconPipelineManager = (ReconPipelineManager) pipelineManager;
+ reconPipelineManager.addPipeline(pipeline);
}
@SuppressWarnings("checkstyle:MethodLength")
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java
index a74b9f7978..eee7f44cd9 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.recon.api;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -51,6 +52,7 @@ import
org.apache.hadoop.ozone.recon.api.types.DatanodesResponse;
import org.apache.hadoop.ozone.recon.common.CommonUtils;
import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
+import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
@@ -117,6 +119,8 @@ public class TestOpenContainerCount {
private List<ContainerWithPipeline> cpw;
private CommonUtils commonUtils;
+ private PipelineManager pipelineManager;
+ private ReconPipelineManager reconPipelineManager;
private void initializeInjector() throws Exception {
reconOMMetadataManager = getTestReconOmMetadataManager(
@@ -220,6 +224,10 @@ public class TestOpenContainerCount {
nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class);
reconScm = (ReconStorageContainerManagerFacade)
reconTestInjector.getInstance(OzoneStorageContainerManager.class);
+ pipelineManager = reconScm.getPipelineManager();
+ reconPipelineManager = (ReconPipelineManager) pipelineManager;
+ reconPipelineManager.addPipeline(pipeline);
+ reconPipelineManager.addPipeline(pipeline2);
}
@BeforeEach
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
index a373ff9aec..642ed7a096 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
@@ -281,4 +281,28 @@ public class TestReconContainerManager
assertEquals(uuid2,
repHistMap.get(cIDlong1).keySet().iterator().next());
}
+
+ @Test
+ public void testAddNewContainerWithMissingPipeline()
+ throws IOException, TimeoutException {
+ // Create a container with a pipeline that Recon does not know about yet
+ Pipeline newPipeline = getRandomPipeline();
+ ContainerInfo containerInfo = newContainerInfo(101L, newPipeline);
+ ContainerWithPipeline containerWithPipeline =
+ new ContainerWithPipeline(containerInfo, newPipeline);
+
+ // Ensure pipeline is not present
+ ReconContainerManager containerManager = getContainerManager();
+ assertFalse(getPipelineManager().containsPipeline(
+ newPipeline.getId()));
+
+ // Add a new container, and it should add the missing pipeline first
+ containerManager.addNewContainer(containerWithPipeline);
+
+ // Pipeline should be added now
+ assertTrue(getPipelineManager().containsPipeline(
+ newPipeline.getId()));
+ assertTrue(containerManager.containerExist(containerInfo.containerID()));
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
