HDDS-259. Implement ContainerReportPublisher and NodeReportPublisher. Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68b57ad3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68b57ad3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68b57ad3 Branch: refs/heads/HADOOP-15461 Commit: 68b57ad32cb0978ad5cd20b5fdc821f087a2c9dc Parents: e9c44ec Author: Xiaoyu Yao <x...@apache.org> Authored: Fri Jul 20 09:07:58 2018 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Fri Jul 20 09:12:48 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 26 ++++++++++++++-- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 5 --- .../common/src/main/resources/ozone-default.xml | 26 +++++++++++++--- .../apache/hadoop/hdds/scm/HddsServerUtil.java | 13 ++++---- .../report/CommandStatusReportPublisher.java | 24 ++++++++++++--- .../common/report/ContainerReportPublisher.java | 25 ++++++++++++--- .../common/report/NodeReportPublisher.java | 32 ++++++++++++++++++-- .../common/report/ReportPublisher.java | 14 +++++++-- .../common/report/TestReportPublisher.java | 11 ++----- .../scm/container/closer/ContainerCloser.java | 12 ++++---- .../container/closer/TestContainerCloser.java | 8 ++--- .../hadoop/hdds/scm/node/TestNodeManager.java | 17 +++++------ .../hadoop/ozone/MiniOzoneClusterImpl.java | 5 +-- .../ozone/TestStorageContainerManager.java | 8 ++--- .../hadoop/ozone/scm/node/TestQueryNode.java | 5 ++- 16 files changed, 162 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index 8b449fb..0283615 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@ -17,15 +17,35 @@ */ package org.apache.hadoop.hdds; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; /** - * Config class for HDDS. + * This class contains constants for configuration keys and default values + * used in hdds. */ public final class HddsConfigKeys { + + /** + * Do not instantiate. + */ private HddsConfigKeys() { } + + public static final String HDDS_HEARTBEAT_INTERVAL = + "hdds.heartbeat.interval"; + public static final String HDDS_HEARTBEAT_INTERVAL_DEFAULT = + "30s"; + + public static final String HDDS_NODE_REPORT_INTERVAL = + "hdds.node.report.interval"; + public static final String HDDS_NODE_REPORT_INTERVAL_DEFAULT = + "60s"; + + public static final String HDDS_CONTAINER_REPORT_INTERVAL = + "hdds.container.report.interval"; + public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT = + "60s"; + public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL = "hdds.command.status.report.interval"; public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT = - ScmConfigKeys.OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT; + "60s"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 46eb8aa..71184cf 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -156,11 +156,6 @@ public final class ScmConfigKeys { "ozone.scm.handler.count.key"; public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10; - public static final String OZONE_SCM_HEARTBEAT_INTERVAL = - "ozone.scm.heartbeat.interval"; - public static final String OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT = - "30s"; - public static final String OZONE_SCM_DEADNODE_INTERVAL = "ozone.scm.dead.node.interval"; public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT = http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index fc10fbb..0273677 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -200,11 +200,6 @@ public final class OzoneConfigKeys { public static final int OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT = 10; - public static final String OZONE_CONTAINER_REPORT_INTERVAL = - "ozone.container.report.interval"; - public static final String OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT = - "60s"; - public static final String DFS_CONTAINER_RATIS_ENABLED_KEY = ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY; public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index da3870e..5a1d26a 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -153,13 +153,29 @@ <description>The timeout duration for ratis server request.</description> </property> <property> - <name>ozone.container.report.interval</name> + <name>hdds.node.report.interval</name> + <value>60000ms</value> + <tag>OZONE, CONTAINER, MANAGEMENT</tag> + <description>Time interval of the datanode to send node report. Each + datanode periodically send node report to SCM. Unit could be + defined with postfix (ns,ms,s,m,h,d)</description> + </property> + <property> + <name>hdds.container.report.interval</name> <value>60000ms</value> <tag>OZONE, CONTAINER, MANAGEMENT</tag> <description>Time interval of the datanode to send container report. Each - datanode periodically send container report upon receive - sendContainerReport from SCM. Unit could be defined with - postfix (ns,ms,s,m,h,d)</description> + datanode periodically send container report to SCM. Unit could be + defined with postfix (ns,ms,s,m,h,d)</description> + </property> + <property> + <name>hdds.command.status.report.interval</name> + <value>60000ms</value> + <tag>OZONE, CONTAINER, MANAGEMENT</tag> + <description>Time interval of the datanode to send status of command + execution. Each datanode periodically the execution status of commands + received from SCM to SCM. Unit could be defined with postfix + (ns,ms,s,m,h,d)</description> </property> <!--Ozone Settings--> <property> @@ -677,7 +693,7 @@ </description> </property> <property> - <name>ozone.scm.heartbeat.interval</name> + <name>hdds.heartbeat.interval</name> <value>30s</value> <tag>OZONE, MANAGEMENT</tag> <description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java index cc7adbf..a8b919d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java @@ -29,13 +29,15 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_HEARTBEAT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_HEARTBEAT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_DEADNODE_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT; @@ -181,9 +183,8 @@ public final class HddsServerUtil { * @return - HB interval in seconds. */ public static long getScmHeartbeatInterval(Configuration conf) { - return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, - ScmConfigKeys.OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT, - TimeUnit.SECONDS); + return conf.getTimeDuration(HDDS_HEARTBEAT_INTERVAL, + HDDS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS); } /** @@ -225,7 +226,7 @@ public final class HddsServerUtil { sanitizeUserArgs(staleNodeIntervalMs, heartbeatIntervalMs, 3, 1000); } catch (IllegalArgumentException ex) { LOG.error("Stale Node Interval MS is cannot be honored due to " + - "mis-configured {}. ex: {}", OZONE_SCM_HEARTBEAT_INTERVAL, ex); + "mis-configured {}. ex: {}", HDDS_HEARTBEAT_INTERVAL, ex); throw ex; } return staleNodeIntervalMs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java index ca5174a..3898d15 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java @@ -19,12 +19,20 @@ package org.apache.hadoop.ozone.container.common.report; import java.util.Iterator; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatus.Status; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.CommandStatus.Status; import org.apache.hadoop.hdds.protocol.proto. StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto; +import org.apache.hadoop.hdds.scm.HddsServerUtil; import org.apache.hadoop.ozone.protocol.commands.CommandStatus; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_COMMAND_STATUS_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT; + /** * Publishes CommandStatusReport which will be sent to SCM as part of * heartbeat. CommandStatusReport consist of the following information: @@ -42,9 +50,17 @@ public class CommandStatusReportPublisher extends protected long getReportFrequency() { if (cmdStatusReportInterval == -1) { cmdStatusReportInterval = getConf().getTimeDuration( - HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL, - HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT, + HDDS_COMMAND_STATUS_REPORT_INTERVAL, + HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + + long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval( + getConf()); + + Preconditions.checkState( + heartbeatFrequency < cmdStatusReportInterval, + HDDS_COMMAND_STATUS_REPORT_INTERVAL + + " cannot be configured lower than heartbeat frequency."); } return cmdStatusReportInterval; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java index ea2b987..3e73bb4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java @@ -17,13 +17,20 @@ package org.apache.hadoop.ozone.container.common.report; +import com.google.common.base.Preconditions; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.hdds.scm.HddsServerUtil; +import java.io.IOException; import java.util.concurrent.TimeUnit; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_CONTAINER_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT; + /** * Publishes ContainerReport which will be sent to SCM as part of heartbeat. @@ -49,9 +56,17 @@ public class ContainerReportPublisher extends protected long getReportFrequency() { if (containerReportInterval == null) { containerReportInterval = getConf().getTimeDuration( - OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL, - OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT, + HDDS_CONTAINER_REPORT_INTERVAL, + HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + + long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval( + getConf()); + + Preconditions.checkState( + heartbeatFrequency < containerReportInterval, + HDDS_CONTAINER_REPORT_INTERVAL + + " cannot be configured lower than heartbeat frequency."); } // Add a random delay (0~30s) on top of the container report // interval (60s) so tha the SCM is overwhelmed by the container reports @@ -64,7 +79,7 @@ public class ContainerReportPublisher extends } @Override - protected ContainerReportsProto getReport() { - return ContainerReportsProto.getDefaultInstance(); + protected ContainerReportsProto getReport() throws IOException { + return getContext().getParent().getContainer().getContainerReport(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java index 704b1f5..b98cba4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java @@ -17,8 +17,18 @@ package org.apache.hadoop.ozone.container.common.report; +import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.hdds.scm.HddsServerUtil; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_NODE_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_NODE_REPORT_INTERVAL_DEFAULT; /** * Publishes NodeReport which will be sent to SCM as part of heartbeat. @@ -28,13 +38,29 @@ import org.apache.hadoop.hdds.protocol.proto */ public class NodeReportPublisher extends ReportPublisher<NodeReportProto> { + private Long nodeReportInterval; + @Override protected long getReportFrequency() { - return 90000L; + if (nodeReportInterval == null) { + nodeReportInterval = getConf().getTimeDuration( + HDDS_NODE_REPORT_INTERVAL, + HDDS_NODE_REPORT_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + + long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval( + getConf()); + + Preconditions.checkState( + heartbeatFrequency < nodeReportInterval, + HDDS_NODE_REPORT_INTERVAL + + " cannot be configured lower than heartbeat frequency."); + } + return nodeReportInterval; } @Override - protected NodeReportProto getReport() { - return NodeReportProto.getDefaultInstance(); + protected NodeReportProto getReport() throws IOException { + return getContext().getParent().getContainer().getNodeReport(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java index 105f073..e3910db 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java @@ -23,7 +23,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ozone.container.common.statemachine .DatanodeStateMachine.DatanodeStates; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -34,6 +37,9 @@ import java.util.concurrent.TimeUnit; public abstract class ReportPublisher<T extends GeneratedMessage> implements Configurable, Runnable { + private static final Logger LOG = LoggerFactory.getLogger( + ReportPublisher.class); + private Configuration config; private StateContext context; private ScheduledExecutorService executor; @@ -76,7 +82,11 @@ public abstract class ReportPublisher<T extends GeneratedMessage> * Generates and publishes the report to datanode state context. */ private void publishReport() { - context.addReport(getReport()); + try { + context.addReport(getReport()); + } catch (IOException e) { + LOG.error("Exception while publishing report.", e); + } } /** @@ -91,7 +101,7 @@ public abstract class ReportPublisher<T extends GeneratedMessage> * * @return datanode report */ - protected abstract T getReport(); + protected abstract T getReport() throws IOException; /** * Returns {@link StateContext}. http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java index d4db55b..a0db2e8 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java @@ -180,14 +180,9 @@ public class TestReportPublisher { @Test public void testAddingReportToHeartbeat() { - Configuration conf = new OzoneConfiguration(); - ReportPublisherFactory factory = new ReportPublisherFactory(conf); - ReportPublisher nodeReportPublisher = factory.getPublisherFor( - NodeReportProto.class); - ReportPublisher containerReportPubliser = factory.getPublisherFor( - ContainerReportsProto.class); - GeneratedMessage nodeReport = nodeReportPublisher.getReport(); - GeneratedMessage containerReport = containerReportPubliser.getReport(); + GeneratedMessage nodeReport = NodeReportProto.getDefaultInstance(); + GeneratedMessage containerReport = ContainerReportsProto + .getDefaultInstance(); SCMHeartbeatRequestProto.Builder heartbeatBuilder = SCMHeartbeatRequestProto.newBuilder(); heartbeatBuilder.setDatanodeDetails( http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java index eb591be..ba691ca 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java @@ -37,10 +37,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_CONTAINER_REPORT_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_CONTAINER_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT; /** * A class that manages closing of containers. This allows transition from a @@ -75,8 +75,8 @@ public class ContainerCloser { this.threadRunCount = new AtomicInteger(0); this.isRunning = new AtomicBoolean(false); this.reportInterval = this.configuration.getTimeDuration( - OZONE_CONTAINER_REPORT_INTERVAL, - OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT, TimeUnit.SECONDS); + HDDS_CONTAINER_REPORT_INTERVAL, + HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT, TimeUnit.SECONDS); Preconditions.checkState(this.reportInterval > 0, "report interval has to be greater than 0"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java index 74238a7..f7863bc 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java @@ -44,6 +44,8 @@ import java.io.File; import java.io.IOException; import java.util.concurrent.TimeUnit; +import static org.apache.hadoop.hdds.HddsConfigKeys + .HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_CONTAINER_SIZE_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys @@ -52,8 +54,6 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent .CREATE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent .CREATED; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_CONTAINER_REPORT_INTERVAL; /** * Test class for Closing Container. @@ -72,7 +72,7 @@ public class TestContainerCloser { configuration = SCMTestUtils.getConf(); size = configuration.getLong(OZONE_SCM_CONTAINER_SIZE_GB, OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024; - configuration.setTimeDuration(OZONE_CONTAINER_REPORT_INTERVAL, + configuration.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); testDir = GenericTestUtils .getTestDir(TestContainerMapping.class.getSimpleName()); @@ -137,7 +137,7 @@ public class TestContainerCloser { // second report is discarded by the system if it lands in the 3 * report // frequency window. - configuration.setTimeDuration(OZONE_CONTAINER_REPORT_INTERVAL, 1, + configuration.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); ContainerWithPipeline containerWithPipeline = mapping.allocateContainer( http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java index cefd179..5275992 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java @@ -58,11 +58,10 @@ import java.util.concurrent.TimeoutException; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_STALENODE_INTERVAL; @@ -359,7 +358,7 @@ public class TestNodeManager { final int interval = 100; conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL @@ -388,7 +387,7 @@ public class TestNodeManager { final int interval = 100; conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, TimeUnit.MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, TimeUnit.SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, TimeUnit.SECONDS); // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL @@ -413,7 +412,7 @@ public class TestNodeManager { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); @@ -551,7 +550,7 @@ public class TestNodeManager { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); @@ -729,7 +728,7 @@ public class TestNodeManager { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); @@ -820,7 +819,7 @@ public class TestNodeManager { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000, MILLISECONDS); @@ -985,7 +984,7 @@ public class TestNodeManager { conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 3ef74b0..9b7e399 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -59,6 +59,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState .HEALTHY; import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY; @@ -392,11 +393,11 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster { private void configureSCMheartbeat() { if (hbInterval.isPresent()) { - conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL, + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, hbInterval.get(), TimeUnit.MILLISECONDS); } else { - conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL, + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, DEFAULT_HB_INTERVAL_MS, TimeUnit.MILLISECONDS); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java index cc367b3..7ca5fa1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.ozone; -import static org.junit.Assert.fail; import java.io.IOException; import org.apache.commons.lang3.RandomUtils; @@ -68,6 +67,9 @@ import org.junit.rules.Timeout; import org.mockito.Mockito; import org.apache.hadoop.test.GenericTestUtils; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; +import static org.junit.Assert.fail; + /** * Test class that exercises the StorageContainerManager. */ @@ -186,9 +188,7 @@ public class TestStorageContainerManager { public void testBlockDeletionTransactions() throws Exception { int numKeys = 5; OzoneConfiguration conf = new OzoneConfiguration(); - conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL, - 5, - TimeUnit.SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 5, TimeUnit.SECONDS); conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 3000, TimeUnit.MILLISECONDS); http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java index 22528e4..a4fcd53 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; @@ -38,8 +39,6 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_STALENODE_INTERVAL; @@ -61,7 +60,7 @@ public class TestQueryNode { conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, TimeUnit.MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org