Copilot commented on code in PR #4924:
URL: https://github.com/apache/bookkeeper/pull/4924#discussion_r4449585289
##########
bookkeeper-server/src/test/java/org/apache/bookkeeper/client/PendingAddOpTest.java:
##########
@@ -73,6 +86,173 @@ public void setup() {
payload.writerIndex(data.length);
}
+ @Test(timeout = 30000)
+ public void testReconcileStaleEnsembleOnNoOpChange() throws Exception {
+ checkStaleEnsembleReconciled(Collections.emptySet());
+ }
+
+ @Test(timeout = 30000)
+ public void testReconcileStaleEnsembleWhenOnlyOtherIndexChanges() throws
Exception {
+ checkStaleEnsembleReconciled(Collections.singleton(1));
+ }
+
+ private void checkStaleEnsembleReconciled(Set<Integer> changedIndices)
throws Exception {
+ BookieId removed = BookieId.parse("removed:3181");
+ BookieId replacement = BookieId.parse("replacement:3181");
+ BookieId other = BookieId.parse("other:3181");
+ BookieId newOther = BookieId.parse("new-other:3181");
+ List<BookieId> target = Arrays.asList(replacement,
+ changedIndices.isEmpty() ? other : newOther);
+ MockClientContext client = MockClientContext.create();
+ CompletableFuture<Void> oldWrite = new CompletableFuture<>();
+ try {
+ Versioned<LedgerMetadata> metadata =
ClientUtil.setupLedger(client, 10L,
+
LedgerMetadataBuilder.create().withEnsembleSize(2).withWriteQuorumSize(2).withAckQuorumSize(2)
+ .newEnsembleEntry(0L, target));
+ LedgerHandle handle = new LedgerHandle(client, 10L, metadata,
BookKeeper.DigestType.CRC32C,
+ ClientUtil.PASSWD, WriteFlag.NONE);
+ client.getMockBookieClient().setPreWriteHook((bookie, ledgerId,
entryId) ->
+ bookie.equals(removed) ? oldWrite :
CompletableFuture.completedFuture(null));
+ CompletableFuture<Integer> first = new CompletableFuture<>();
+ // Reproduce the interleaving deterministically: metadata is
already updated,
+ // but the operation captured the old ensemble before it entered
the queue.
+ PendingAddOp stale = PendingAddOp.create(handle, client,
Arrays.asList(removed, other),
+ Unpooled.wrappedBuffer(new byte[] {1}), WriteFlag.NONE,
+ (rc, lh, entryId, latency, ctx) -> first.complete(rc),
null);
+ handle.doAsyncAddEntry(stale);
+ CompletableFuture<Integer> second = new CompletableFuture<>();
+ handle.asyncAddEntry(new byte[] {2}, (rc, lh, entryId, ctx) ->
second.complete(rc), null);
+ handle.executor.submit(() -> { }).get();
+ assertFalse(first.isDone());
+ assertFalse(second.isDone());
+ handle.executor.submit(() ->
handle.unsetSuccessAndSendWriteRequest(target, changedIndices)).get();
+ assertEquals(Integer.valueOf(Code.OK), first.get(10,
TimeUnit.SECONDS));
+ assertEquals(Integer.valueOf(Code.OK), second.get(10,
TimeUnit.SECONDS));
+ assertEquals(1L, handle.getLastAddConfirmed());
+ } finally {
+ oldWrite.complete(null);
+ client.getMainWorkerPool().shutdown();
+ client.getScheduler().shutdown();
+ }
+ }
+
+ @Test(timeout = 30000)
+ public void testMultiIndexAckCleanupAndLateResponsesAcrossChanges() throws
Exception {
+ MockClientContext client = MockClientContext.create();
+ // Control every response explicitly, including replies from older
generations.
+ client.setBookieClient(mock(BookieClient.class));
+ List<BookieId> original = Arrays.asList(BookieId.parse("a:3181"),
+ BookieId.parse("b:3181"), BookieId.parse("c:3181"));
+ List<BookieId> next = Arrays.asList(BookieId.parse("d:3181"),
+ BookieId.parse("e:3181"), BookieId.parse("f:3181"));
+ List<BookieId> last = Arrays.asList(BookieId.parse("g:3181"),
+ BookieId.parse("h:3181"), BookieId.parse("i:3181"));
+ try {
+ Versioned<LedgerMetadata> metadata =
ClientUtil.setupLedger(client, 11L,
+
LedgerMetadataBuilder.create().withEnsembleSize(3).withWriteQuorumSize(2).withAckQuorumSize(2)
+ .newEnsembleEntry(0L, original));
+ LedgerHandle handle = new LedgerHandle(client, 11L, metadata,
BookKeeper.DigestType.CRC32C,
+ ClientUtil.PASSWD, WriteFlag.NONE);
+ AtomicInteger callbacks = new AtomicInteger();
+ handle.executor.submit(() -> {
+ PendingAddOp[] ops = new PendingAddOp[2];
+ handle.changingEnsemble = true;
+ handle.pendingAddsSequenceHead = 0;
+ for (int entry = 0; entry < ops.length; entry++) {
+ PendingAddOp op = PendingAddOp.create(handle, client,
original,
+ Unpooled.wrappedBuffer(new byte[] {1}),
WriteFlag.NONE,
+ (rc, lh, eid, latency, ctx) -> {
+ assertEquals(Code.OK, rc);
+ callbacks.incrementAndGet();
+ }, null);
+ ops[entry] = op;
+ op.entryId = entry + 1;
+ op.toSend = op.payload;
+ handle.pendingAddOps.add(op);
+ for (int index = 0; index < 3; index++) {
+ if (handle.distributionSchedule.hasEntry(op.entryId,
index)) {
+ op.pendingWriteRequests++;
+ op.writeComplete(Code.OK, 11L, op.entryId,
original.get(index), index);
+ }
+ }
+ assertTrue(op.completed);
+ }
+ handle.changingEnsemble = false;
+ Set<Integer> changed = new LinkedHashSet<>(Arrays.asList(2, 0,
1));
Review Comment:
Index 2 belongs to both entries' write sets ({1,2} and {2,0}), so processing
it first breaks their ACK quorums before any write-set-external index can
trigger callbacks. This does not exercise the premature-callback ordering
regression described in the PR. Start with index 0, which is outside the head
entry's write set, and process index 2 afterward so the test detects callbacks
running before the affected ACKs are invalidated.
##########
bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java:
##########
@@ -188,56 +189,31 @@ synchronized void timeoutQuorumWait() {
lh.handleUnrecoverableErrorDuringAdd(BKException.Code.AddEntryQuorumTimeoutException);
}
- synchronized void unsetSuccessAndSendWriteRequest(List<BookieId> ensemble,
int bookieIndex) {
- // update the ensemble
- this.ensemble = ensemble;
-
- if (toSend == null) {
- // this addOp hasn't yet had its mac computed. When the mac is
- // computed, its write requests will be sent, so no need to send it
- // now
- return;
+ synchronized Set<Integer> prepareEnsembleChange(List<BookieId> target,
Set<Integer> changed) {
+ Set<Integer> resend = new HashSet<>(changed);
+ resend.addAll(EnsembleUtils.diffEnsemble(ensemble, target));
+ List<BookieId> previous = ensemble;
+ ensemble = target;
+ if (toSend == null || callbackTriggered) {
+ return Collections.emptySet();
}
- // Suppose that unset doesn't happen on the write set of an entry. In
this
- // case we don't need to resend the write request upon an ensemble
change.
- // We do need to invoke #sendAddSuccessCallbacks() for such entries
because
- // they may have already completed, but they are just waiting for the
ensemble
- // to change.
- // E.g.
- // ensemble (A, B, C, D), entry k is written to (A, B, D). An ensemble
change
- // happens to replace C with E. Entry k does not complete until C is
- // replaced with E successfully. When the ensemble change completes,
it tries
- // to unset entry k. C however is not in k's write set, so no entry is
written
- // again, and no one triggers #sendAddSuccessCallbacks. Consequently,
k never
- // completes.
- //
- // We call sendAddSuccessCallback when unsetting t cover this case.
- if (!lh.distributionSchedule.hasEntry(entryId, bookieIndex)) {
- lh.sendAddSuccessCallbacks();
- return;
+ resend.removeIf(index -> !lh.distributionSchedule.hasEntry(entryId,
index));
+ for (int index : resend) {
+ addEntrySuccessBookies.remove(previous.get(index));
+ if (!ackSet.removeBookieAndCheck(index)) {
+ completed = false;
+ }
}
+ return resend;
+ }
- if (callbackTriggered) {
+ synchronized void resendEnsembleChange(Set<Integer> indices) {
+ if (toSend == null || callbackTriggered) {
return;
}
-
-
- log.debug()
- .ctx(lh.log)
- .attr("entryId", entryId)
- .attr("bookieIndex", bookieIndex)
- .log("Unsetting success");
-
-
- // if we had already heard a success from this array index, need to
- // increment our number of responses that are pending, since we are
- // going to unset this success
- if (!ackSet.removeBookieAndCheck(bookieIndex)) {
- // unset completed if this results in loss of ack quorum
- completed = false;
+ for (int index : indices) {
+ sendWriteRequest(ensemble, index);
}
Review Comment:
`BookieClientImpl.addEntry()` can invoke `writeComplete()` inline when pool
lookup fails. If the entry still has ACK quorum, that response can trigger its
callback and clear `toSend`, so the next iteration can send a null buffer or
access recycled state. Repeat the `toSend`/`callbackTriggered` guard before
each resend.
##########
bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java:
##########
@@ -188,56 +189,31 @@ synchronized void timeoutQuorumWait() {
lh.handleUnrecoverableErrorDuringAdd(BKException.Code.AddEntryQuorumTimeoutException);
}
- synchronized void unsetSuccessAndSendWriteRequest(List<BookieId> ensemble,
int bookieIndex) {
- // update the ensemble
- this.ensemble = ensemble;
-
- if (toSend == null) {
- // this addOp hasn't yet had its mac computed. When the mac is
- // computed, its write requests will be sent, so no need to send it
- // now
- return;
+ synchronized Set<Integer> prepareEnsembleChange(List<BookieId> target,
Set<Integer> changed) {
+ Set<Integer> resend = new HashSet<>(changed);
+ resend.addAll(EnsembleUtils.diffEnsemble(ensemble, target));
+ List<BookieId> previous = ensemble;
+ ensemble = target;
+ if (toSend == null || callbackTriggered) {
+ return Collections.emptySet();
}
- // Suppose that unset doesn't happen on the write set of an entry. In
this
- // case we don't need to resend the write request upon an ensemble
change.
- // We do need to invoke #sendAddSuccessCallbacks() for such entries
because
- // they may have already completed, but they are just waiting for the
ensemble
- // to change.
- // E.g.
- // ensemble (A, B, C, D), entry k is written to (A, B, D). An ensemble
change
- // happens to replace C with E. Entry k does not complete until C is
- // replaced with E successfully. When the ensemble change completes,
it tries
- // to unset entry k. C however is not in k's write set, so no entry is
written
- // again, and no one triggers #sendAddSuccessCallbacks. Consequently,
k never
- // completes.
- //
- // We call sendAddSuccessCallback when unsetting t cover this case.
- if (!lh.distributionSchedule.hasEntry(entryId, bookieIndex)) {
- lh.sendAddSuccessCallbacks();
- return;
+ resend.removeIf(index -> !lh.distributionSchedule.hasEntry(entryId,
index));
+ for (int index : resend) {
+ addEntrySuccessBookies.remove(previous.get(index));
+ if (!ackSet.removeBookieAndCheck(index)) {
+ completed = false;
+ }
}
+ return resend;
Review Comment:
With fault-domain enforcement enabled, removing the only ACK from one rack
can leave the numeric ACK quorum intact but violate the placement policy.
Because `completed` stays true, the final callback pass can report success and
advance LAC before the replacement acknowledges. After invalidating all
affected ACKs, also clear `completed` if the remaining successful bookies fail
the placement-policy check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]