HDDS-85. Send Container State Info while sending the container report from Datanode to SCM. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fed2bef6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fed2bef6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fed2bef6 Branch: refs/heads/HDDS-48 Commit: fed2bef647d9a15fe020ad5d3bb89fcb77ed30e6 Parents: 745f203 Author: Mukul Kumar Singh <msi...@apache.org> Authored: Wed May 23 14:15:35 2018 +0530 Committer: Mukul Kumar Singh <msi...@apache.org> Committed: Wed May 23 14:15:35 2018 +0530 ---------------------------------------------------------------------- .../main/proto/DatanodeContainerProtocol.proto | 1 + .../container/common/helpers/ContainerData.java | 8 ++++ .../common/impl/ContainerManagerImpl.java | 45 ++++++++++++++++++-- .../common/interfaces/ContainerManager.java | 2 +- .../commandhandler/ContainerReportHandler.java | 4 +- .../container/ozoneimpl/OzoneContainer.java | 4 +- .../common/impl/TestContainerPersistence.java | 2 +- 7 files changed, 57 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto index 1138297..53da18a 100644 --- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto @@ -131,6 +131,7 @@ enum Result { UNCLOSED_CONTAINER_IO = 25; DELETE_ON_OPEN_CONTAINER = 26; CLOSED_CONTAINER_RETRY = 27; + INVALID_CONTAINER_STATE = 28; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java index 14ee33a..d1746f2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java @@ -340,6 +340,14 @@ public class ContainerData { } /** + * checks if the container is closed. + * @return - boolean + */ + public synchronized boolean isClosed() { + return ContainerLifeCycleState.CLOSED == state; + } + + /** * Marks this container as closed. */ public synchronized void closeContainer() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java index faee5d0..9355364 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java @@ -26,6 +26,8 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -100,6 +102,8 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.UNCLOSED_CONTAINER_IO; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Result.UNSUPPORTED_REQUEST; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos. + Result.INVALID_CONTAINER_STATE; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION; /** @@ -707,6 +711,39 @@ public class ContainerManagerImpl implements ContainerManager { } /** + * Returns LifeCycle State of the container + * @param containerID - Id of the container + * @return LifeCycle State of the container + * @throws StorageContainerException + */ + private HddsProtos.LifeCycleState getState(long containerID) + throws StorageContainerException { + LifeCycleState state; + final ContainerData data = containerMap.get(containerID); + if (data == null) { + throw new StorageContainerException( + "Container status not found: " + containerID, CONTAINER_NOT_FOUND); + } + switch (data.getState()) { + case OPEN: + state = LifeCycleState.OPEN; + break; + case CLOSING: + state = LifeCycleState.CLOSING; + break; + case CLOSED: + state = LifeCycleState.CLOSED; + break; + default: + throw new StorageContainerException( + "Invalid Container state found: " + containerID, + INVALID_CONTAINER_STATE); + } + + return state; + } + + /** * Supports clean shutdown of container. * * @throws IOException @@ -835,14 +872,14 @@ public class ContainerManagerImpl implements ContainerManager { * @throws IOException */ @Override - public List<ContainerData> getContainerReports() throws IOException { + public List<ContainerData> getClosedContainerReports() throws IOException { LOG.debug("Starting container report iteration."); // No need for locking since containerMap is a ConcurrentSkipListMap // And we can never get the exact state since close might happen // after we iterate a point. return containerMap.entrySet().stream() .filter(containerData -> - !containerData.getValue().isOpen()) + containerData.getValue().isClosed()) .map(containerData -> containerData.getValue()) .collect(Collectors.toList()); } @@ -870,6 +907,7 @@ public class ContainerManagerImpl implements ContainerManager { .setType(ContainerReportsRequestProto.reportType.fullReport); for (ContainerData container: containers) { + long containerId = container.getContainerID(); StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); ciBuilder.setContainerID(container.getContainerID()) @@ -879,7 +917,8 @@ public class ContainerManagerImpl implements ContainerManager { .setReadCount(container.getReadCount()) .setWriteCount(container.getWriteCount()) .setReadBytes(container.getReadBytes()) - .setWriteBytes(container.getWriteBytes()); + .setWriteBytes(container.getWriteBytes()) + .setState(getState(containerId)); crBuilder.addReports(ciBuilder.build()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java index 3a1a73d..ba70953 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java @@ -185,7 +185,7 @@ public interface ContainerManager extends RwLock { * @return List of all closed containers. * @throws IOException */ - List<ContainerData> getContainerReports() throws IOException; + List<ContainerData> getClosedContainerReports() throws IOException; /** * Increase pending deletion blocks count number of specified container. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java index ba6b418..fbea290 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java @@ -63,13 +63,13 @@ public class ContainerReportHandler implements CommandHandler { invocationCount++; long startTime = Time.monotonicNow(); try { - ContainerReportsRequestProto contianerReport = + ContainerReportsRequestProto containerReport = container.getContainerReport(); // TODO : We send this report to all SCMs.Check if it is enough only to // send to the leader once we have RAFT enabled SCMs. for (EndpointStateMachine endPoint : connectionManager.getValues()) { - endPoint.getEndPoint().sendContainerReport(contianerReport); + endPoint.getEndPoint().sendContainerReport(containerReport); } } catch (IOException ex) { LOG.error("Unable to process the Container Report command.", ex); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index b497cdc..6758479 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -265,8 +265,8 @@ public class OzoneContainer { * @return - List of closed containers. * @throws IOException */ - public List<ContainerData> getContainerReports() throws IOException { - return this.manager.getContainerReports(); + public List<ContainerData> getClosedContainerReports() throws IOException { + return this.manager.getClosedContainerReports(); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/hadoop/blob/fed2bef6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index 89ee673..4975fd3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -307,7 +307,7 @@ public class TestContainerPersistence { } // The container report only returns reports of closed containers. - List<ContainerData> reports = containerManager.getContainerReports(); + List<ContainerData> reports = containerManager.getClosedContainerReports(); Assert.assertEquals(4, reports.size()); for(ContainerData report : reports) { long actualContainerID = report.getContainerID(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
