This is an automated email from the ASF dual-hosted git repository.
1996fanrui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 23d73809bf4 [FLINK-40345][checkpoint] Do not recycle the failed buffer
twice in FilteringHandler
23d73809bf4 is described below
commit 23d73809bf40ea6e06fa629f476bb55e58ecb059
Author: Rui Fan <[email protected]>
AuthorDate: Fri Aug 7 00:00:34 2026 +0200
[FLINK-40345][checkpoint] Do not recycle the failed buffer twice in
FilteringHandler
onRecoveredStateBuffer() takes over the buffer before anything can fail, so
the
compensating loop in the catch block must skip the buffer it failed on.
---
.../channel/RecoveredChannelStateHandler.java | 4 +-
.../InputChannelRecoveredStateHandlerTest.java | 56 ++++++++++++++++++-
.../consumer/FailingRecoveredInputChannel.java | 63 ++++++++++++++++++++++
3 files changed, 120 insertions(+), 3 deletions(-)
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java
index 72d0b995f29..6a68f47c639 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java
@@ -328,7 +328,9 @@ class FilteringHandler extends
AbstractInputChannelRecoveredStateHandler {
channel.onRecoveredStateBuffer(filteredBuffers.get(i));
}
} catch (Throwable t) {
- for (int j = i; j < filteredBuffers.size(); j++) {
+ // Start at i + 1: onRecoveredStateBuffer() takes over the buffer
before anything can
+ // fail, so recycling the buffer at index i again would corrupt
its reference count.
+ for (int j = i + 1; j < filteredBuffers.size(); j++) {
filteredBuffers.get(j).recycleBuffer();
}
throw t;
diff --git
a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java
b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java
index 18f8a863352..76f82a82ff7 100644
---
a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java
+++
b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.checkpoint.RescaleMappings;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.buffer.NetworkBuffer;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
+import
org.apache.flink.runtime.io.network.partition.consumer.FailingRecoveredInputChannel;
import
org.apache.flink.runtime.io.network.partition.consumer.InputChannelBuilder;
import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
@@ -33,10 +34,13 @@ import org.apache.flink.runtime.memory.MemoryManager;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import static
org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptorUtil.mappings;
import static
org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptorUtil.to;
+import static
org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSomeBuffer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -121,9 +125,14 @@ class InputChannelRecoveredStateHandlerTest extends
RecoveredChannelStateHandler
private FilteringHandler buildFilteringInputChannelStateHandler() {
// Empty GateFilterHandler array: filtering is "enabled" structurally,
but no gate-level
// filter logic runs. Suitable for exercising getBuffer() routing only.
- ChannelStateFilteringHandler stubFilteringHandler =
+ return buildFilteringInputChannelStateHandler(
+ inputGate,
new ChannelStateFilteringHandler(
- new ChannelStateFilteringHandler.GateFilterHandler[0]);
+ new
ChannelStateFilteringHandler.GateFilterHandler[0]));
+ }
+
+ private FilteringHandler buildFilteringInputChannelStateHandler(
+ SingleInputGate inputGate, ChannelStateFilteringHandler
stubFilteringHandler) {
return (FilteringHandler)
AbstractInputChannelRecoveredStateHandler.create(
new InputGate[] {inputGate},
@@ -283,6 +292,49 @@ class InputChannelRecoveredStateHandlerTest extends
RecoveredChannelStateHandler
}
}
+ @Test
+ void testFilteredBuffersRecycledOnceWhenDeliveryFails() throws Exception {
+ List<Buffer> filteredBuffers =
+ Arrays.asList(buildSomeBuffer(), buildSomeBuffer(),
buildSomeBuffer());
+ ChannelStateFilteringHandler filteringHandler =
+ new ChannelStateFilteringHandler(
+ new ChannelStateFilteringHandler.GateFilterHandler[0])
{
+ @Override
+ public List<Buffer> filterAndRewrite(
+ int gateIndex,
+ int oldSubtaskIndex,
+ int oldChannelIndex,
+ Buffer sourceBuffer,
+ BufferSupplier bufferSupplier) {
+ sourceBuffer.recycleBuffer();
+ return filteredBuffers;
+ }
+ };
+ // The gate fails while taking over the first filtered buffer.
+ SingleInputGate failingGate =
+ new SingleInputGateBuilder()
+ .setChannelFactory(
+ (builder, gate) -> new
FailingRecoveredInputChannel(gate, 0))
+ .setSegmentProvider(networkBufferPool)
+ .build();
+
+ try (FilteringHandler handler =
+ buildFilteringInputChannelStateHandler(failingGate,
filteringHandler)) {
+ RecoveredChannelStateHandler.BufferWithContext<Buffer>
bufferWithContext =
+ handler.getBuffer(channelInfo);
+ bufferWithContext.context.setSize(1);
+
+ assertThatThrownBy(() -> handler.recover(channelInfo, 0,
bufferWithContext))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("Delivery failed on purpose.");
+ }
+
+ // The first buffer is owned by the channel, the undelivered ones are
recycled exactly once.
+ assertThat(filteredBuffers.get(0).isRecycled()).isFalse();
+ assertThat(filteredBuffers.get(1).isRecycled()).isTrue();
+ assertThat(filteredBuffers.get(2).isRecycled()).isTrue();
+ }
+
@Test
void testPreFilterSegmentFreedOnClose() throws Exception {
FilteringHandler filteringHandler =
buildFilteringInputChannelStateHandler();
diff --git
a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/FailingRecoveredInputChannel.java
b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/FailingRecoveredInputChannel.java
new file mode 100644
index 00000000000..fda01f41d88
--- /dev/null
+++
b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/FailingRecoveredInputChannel.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.partition.consumer;
+
+import org.apache.flink.metrics.SimpleCounter;
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import
org.apache.flink.runtime.io.network.partition.ResultSubpartitionIndexSet;
+
+/**
+ * A {@link RecoveredInputChannel} that starts failing {@link
#onRecoveredStateBuffer} once the
+ * given number of buffers has been taken over. The failing call takes over
its buffer as well, like
+ * the production implementations do.
+ */
+public class FailingRecoveredInputChannel extends RecoveredInputChannel {
+
+ private final int failAfterBuffers;
+
+ private int deliveredBuffers;
+
+ public FailingRecoveredInputChannel(SingleInputGate inputGate, int
failAfterBuffers) {
+ super(
+ inputGate,
+ 0,
+ new ResultPartitionID(),
+ new ResultSubpartitionIndexSet(0),
+ 0,
+ 0,
+ new SimpleCounter(),
+ new SimpleCounter(),
+ 1);
+ this.failAfterBuffers = failAfterBuffers;
+ }
+
+ @Override
+ public void onRecoveredStateBuffer(Buffer buffer) {
+ super.onRecoveredStateBuffer(buffer);
+ if (++deliveredBuffers > failAfterBuffers) {
+ throw new IllegalStateException("Delivery failed on purpose.");
+ }
+ }
+
+ @Override
+ protected InputChannel toInputChannelInternal(boolean needsRecovery) {
+ return new TestInputChannel(inputGate, 0);
+ }
+}
