smengcl commented on code in PR #11962:
URL: https://github.com/apache/ozone/pull/11962#discussion_r4510162667
##########
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java:
##########
@@ -228,46 +233,106 @@ public void testReportQueueWithAddReports() throws
IOException {
}
@Test
- public void testGetAllAvailableReportsRespectsLimit() throws IOException {
- StateContext ctx = createSubject();
- HostAndPort scm1 = new HostAndPort("scm1", 9001);
- ctx.addEndpoint(scm1);
-
- // Queue 10 ICRs; no full report is refreshed, so only ICRs are returned.
- batchAddIncrementalReport(ctx,
- StateContext.INCREMENTAL_CONTAINER_REPORT_PROTO_NAME, 10);
-
- // A limited collection returns at most the limit and leaves the rest.
- assertEquals(4, ctx.getAllAvailableReports(scm1, 4).size());
- assertTrue(ctx.hasPendingReports(scm1));
- assertEquals(4, ctx.getAllAvailableReports(scm1, 4).size());
- assertTrue(ctx.hasPendingReports(scm1));
- // Last chunk drains the queue.
- assertEquals(2, ctx.getAllAvailableReports(scm1, 4).size());
- assertFalse(ctx.hasPendingReports(scm1));
- assertEquals(0, ctx.getAllAvailableReports(scm1, 4).size());
+ public void testCommandStatusReportsRespectByteLimit() throws IOException {
+ StateContext context = createSubject();
+ HostAndPort scm = new HostAndPort("scm", 9001);
+ context.addEndpoint(scm);
+
+ CommandStatusReportsProto first = newCommandStatusReport(1, 128);
+ CommandStatusReportsProto second = newCommandStatusReport(2, 128);
+ context.addIncrementalReport(first);
+ context.addIncrementalReport(second);
+
+ long firstBytes = commandStatusReportWireSize(first);
+ CommandStatusReportSelection selection =
+ context.getCommandStatusReports(scm, 10, firstBytes, firstBytes,
firstBytes);
+ assertEquals(Collections.singletonList(first), selection.getReports());
+ assertEquals(firstBytes, selection.getSelectedBytes());
+ assertEquals(commandStatusReportWireSize(second),
context.getCommandStatusReportQueueBytes(scm));
}
@Test
- public void testHasPendingReportsWithFullReports() throws IOException {
- StateContext ctx = createSubject();
- HostAndPort scm1 = new HostAndPort("scm1", 9001);
- ctx.addEndpoint(scm1);
-
- // addEndpoint marks the full-report ready flags true, but no full report
- // value has been generated yet, so nothing is actually pending.
- assertFalse(ctx.hasPendingReports(scm1));
+ public void testAggregateCommandStatusReportIsSplitAtStatusBoundary() throws
IOException {
+ StateContext context = createSubject();
+ HostAndPort scm = new HostAndPort("scm", 9001);
+ context.addEndpoint(scm);
+
+ CommandStatus first = newCommandStatus(1, 128);
+ CommandStatus second = newCommandStatus(2, 128);
+ CommandStatus third = newCommandStatus(3, 128);
+ CommandStatusReportsProto aggregate =
+ CommandStatusReportsProto.newBuilder()
+ .addCmdStatus(first)
+ .addCmdStatus(second)
+ .addCmdStatus(third)
+ .build();
+ CommandStatusReportsProto firstBatch =
+ CommandStatusReportsProto.newBuilder()
+ .addCmdStatus(first)
+ .addCmdStatus(second)
+ .build();
+ CommandStatusReportsProto secondBatch =
+ CommandStatusReportsProto.newBuilder()
+ .addCmdStatus(third)
+ .build();
+ context.addIncrementalReport(aggregate);
+
+ CommandStatusReportSelection selection = context.getCommandStatusReports(
+ scm, 10, commandStatusReportWireSize(firstBatch),
+ commandStatusReportWireSize(firstBatch),
+ commandStatusReportWireSize(firstBatch));
+ assertEquals(Collections.singletonList(firstBatch),
+ selection.getReports());
+ assertEquals(commandStatusReportWireSize(secondBatch),
context.getCommandStatusReportQueueBytes(scm));
+
+ selection = context.getCommandStatusReports(
+ scm, 10, commandStatusReportWireSize(firstBatch),
+ commandStatusReportWireSize(firstBatch),
+ commandStatusReportWireSize(firstBatch));
+ assertEquals(Collections.singletonList(secondBatch),
+ selection.getReports());
+ assertEquals(0, context.getCommandStatusReportQueueBytes(scm));
+ }
- // Generate all three full reports; now values exist and flags are ready.
- batchRefreshfullReports(ctx, StateContext.CONTAINER_REPORTS_PROTO_NAME, 1);
- batchRefreshfullReports(ctx, StateContext.NODE_REPORT_PROTO_NAME, 1);
- batchRefreshfullReports(ctx, StateContext.PIPELINE_REPORTS_PROTO_NAME, 1);
- assertTrue(ctx.hasPendingReports(scm1));
+ @Test
+ public void testNonCommandReportsAreSelectedIndependently() throws
IOException {
+ StateContext context = createSubject();
+ HostAndPort scm = new HostAndPort("scm", 9001);
+ context.addEndpoint(scm);
+ CommandStatusReportsProto commandStatus =
+ newCommandStatusReport(1, 128);
+ IncrementalContainerReportProto icr =
+ IncrementalContainerReportProto.getDefaultInstance();
+ context.addIncrementalReport(commandStatus);
+ context.addIncrementalReport(icr);
+
+ assertEquals(Collections.singletonList(icr),
+ context.getAvailableNonCommandStatusReports(scm, 10));
+ assertEquals(commandStatusReportWireSize(commandStatus),
+ context.getCommandStatusReportQueueBytes(scm));
+ }
- // With a cap of one, a heartbeat takes a single full report but the others
- // stay ready, so a follow-up is still needed.
- assertEquals(1, ctx.getAllAvailableReports(scm1, 1).size());
- assertTrue(ctx.hasPendingReports(scm1));
+ @Test
+ public void testOversizedCommandStatusMakesProgressWhenSafe() throws
IOException {
Review Comment:
Thanks for adding the progress path. The tests use a report with a single
status, so the `iterator.set(singleStatus.getRemainder())` branch is not
covered. That is the common case in practice, since
`CommandStatusReportPublisher` puts all statuses of an interval into one
aggregate report. Could we add a case like `[small, oversized, small]` in one
report? The first call should return only the first small status, the second
call only the oversized one, and the last small status should stay queued in
order.
##########
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java:
##########
@@ -112,7 +112,12 @@ public class ScmConfig extends ReconfigurableConfig {
+ "The block num is counted at the replica level."
+ "If SCM has 100000 blocks which need to be deleted and the "
+ "configuration is 5000 then it would only send 5000 blocks "
- + "for deletion to the datanodes."
+ + "for deletion to the datanodes. Increasing this value can "
+ + "increase delete command acknowledgement size; each "
+ + "acknowledgement must fit within the datanode heartbeat "
Review Comment:
nit: with the new progress exception, an acknowledgement larger than
`hdds.heartbeat.command.status.report.max.size` is still sent alone, so the
hard bound is only the safe RPC payload (90% of `ipc.maximum.data.length`,
minus the other heartbeat fields). Maybe say that a single acknowledgement must
fit within the IPC size limit, and drop the command status limit part, so it
matches the ozone-default.xml description.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]