[
https://issues.apache.org/jira/browse/FLINK-7524?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16176202#comment-16176202
]
ASF GitHub Bot commented on FLINK-7524:
---------------------------------------
Github user aljoscha commented on a diff in the pull request:
https://github.com/apache/flink/pull/4639#discussion_r140452104
--- Diff:
flink-core/src/test/java/org/apache/flink/core/fs/AbstractCloseableRegistryTest.java
---
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.fs;
+
+import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.util.AbstractCloseableRegistry;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.spy;
+
+public abstract class AbstractCloseableRegistryTest<C extends Closeable,
T> {
+
+ protected ProducerThread[] streamOpenThreads;
+ protected AbstractCloseableRegistry<C, T> closeableRegistry;
+ protected AtomicInteger unclosedCounter;
+
+ protected abstract C createCloseable();
+
+ protected abstract AbstractCloseableRegistry<C, T> createRegistry();
+
+ protected abstract ProducerThread<C, T> createProducerThread(
+ AbstractCloseableRegistry<C, T> registry,
+ AtomicInteger unclosedCounter,
+ int maxStreams);
+
+ public void setup() {
+
Assert.assertFalse(SafetyNetCloseableRegistry.isReaperThreadRunning());
+ this.closeableRegistry = createRegistry();
+ this.unclosedCounter = new AtomicInteger(0);
+ this.streamOpenThreads = new ProducerThread[10];
+ for (int i = 0; i < streamOpenThreads.length; ++i) {
+ streamOpenThreads[i] =
createProducerThread(closeableRegistry, unclosedCounter, Integer.MAX_VALUE);
+ }
+ }
+
+ protected void startThreads(int maxStreams) {
+ for (ProducerThread t : streamOpenThreads) {
+ t.setMaxStreams(maxStreams);
+ t.start();
+ }
+ }
+
+ protected void joinThreads() throws InterruptedException {
+ for (Thread t : streamOpenThreads) {
+ t.join();
+ }
+ }
+
+ @Test
+ public void testClose() throws Exception {
+
+ setup();
+ startThreads(Integer.MAX_VALUE);
+
+ for (int i = 0; i < 5; ++i) {
+ System.gc();
+ Thread.sleep(40);
+ }
+
+ closeableRegistry.close();
+
+ joinThreads();
+
+ Assert.assertEquals(0, unclosedCounter.get());
+ Assert.assertEquals(0,
closeableRegistry.getNumberOfRegisteredCloseables());
+
+ final C testCloseable = spy(createCloseable());
+
+ try {
+
+ closeableRegistry.registerClosable(testCloseable);
+
+ Assert.fail("Closed registry should not accept
closeables!");
+
+ } catch (IOException expected) {
+ //expected
+ }
+
+ Assert.assertEquals(0, unclosedCounter.get());
+ Assert.assertEquals(0,
closeableRegistry.getNumberOfRegisteredCloseables());
+ verify(testCloseable).close();
+ }
+
+ @Test
+ public void testNonBlockingClose() throws Exception {
+ setup();
+
+ final OneShotLatch waitRegistryClosedLatch = new OneShotLatch();
+ final OneShotLatch blockCloseLatch = new OneShotLatch();
+
+ final C spyCloseable = spy(createCloseable());
+
+ doAnswer(invocationOnMock -> {
+ invocationOnMock.callRealMethod();
+ waitRegistryClosedLatch.trigger();
+ blockCloseLatch.await();
+ return null;
+ }).when(spyCloseable).close();
+
+ closeableRegistry.registerClosable(spyCloseable);
+
+ Assert.assertEquals(1,
closeableRegistry.getNumberOfRegisteredCloseables());
+
+ Thread closer = new Thread(() -> {
+ try {
+ closeableRegistry.close();
+ } catch (IOException ignore) {
+
+ }
+ });
+
+ closer.start();
+ waitRegistryClosedLatch.await();
+
+ final C testCloseable = spy(createCloseable());
+
+ try {
+ closeableRegistry.registerClosable(testCloseable);
+ Assert.fail("Closed registry should not accept
closeables!");
+ }catch (IOException ignore) {
+ }
+
+ blockCloseLatch.trigger();
+ closer.join();
+
+ verify(spyCloseable).close();
+ verify(testCloseable).close();
+ Assert.assertEquals(0,
closeableRegistry.getNumberOfRegisteredCloseables());
+ }
+
+ protected static abstract class ProducerThread<C extends Closeable, T>
extends Thread {
+
+ protected final AbstractCloseableRegistry<C, T> registry;
+ protected final AtomicInteger refCount;
+ protected int maxStreams;
+
+ public ProducerThread(AbstractCloseableRegistry<C, T> registry,
AtomicInteger refCount, int maxStreams) {
+ this.registry = registry;
+ this.refCount = refCount;
+ this.maxStreams = maxStreams;
+ }
+
+ public int getMaxStreams() {
+ return maxStreams;
+ }
+
+ public void setMaxStreams(int maxStreams) {
+ this.maxStreams = maxStreams;
+ }
+
+ protected abstract void createAndRegisterStream() throws
IOException;
+
+ @Override
+ public void run() {
+ try {
+ while (maxStreams > 0) {
+
+ createAndRegisterStream();
+
+ try {
+ Thread.sleep(2);
+ } catch (InterruptedException ignored)
{}
+
+ if (maxStreams != Integer.MAX_VALUE) {
+ --maxStreams;
--- End diff --
nit: I always find it strange when a field that should be a final is also
used as a "counter". In those cases there should probably be a field
`numStream` and you loop until `maxStreams == openStreams`.
> Task "xxx" did not react to cancelling signal, but is stuck in method
> ---------------------------------------------------------------------
>
> Key: FLINK-7524
> URL: https://issues.apache.org/jira/browse/FLINK-7524
> Project: Flink
> Issue Type: Bug
> Components: State Backends, Checkpointing
> Affects Versions: 1.3.0
> Reporter: Bowen Li
> Priority: Blocker
> Fix For: 1.4.0
>
>
> Hi,
> I observed the following errors in taskmanager.log
> {code:java}
> 2017-08-25 17:03:40,141 WARN org.apache.flink.runtime.taskmanager.Task
> - Task 'TriggerWindow(SlidingEventTimeWindows(259200000,
> 3600000),
> AggregatingStateDescriptor{serializer=org.apache.flink.api.java.typeutils.runtime.TupleSerializer@f65b6aa2,
> aggFunction=com.streamingApp$MyAggregateFunction@1f193686},
> EventTimeTrigger(), WindowedStream.aggregate(WindowedStream.java:858)) ->
> Sink: prod_item (2/6)' did not react to cancelling signal, but is stuck in
> method:
>
> org.apache.flink.util.AbstractCloseableRegistry.unregisterClosable(AbstractCloseableRegistry.java:84)
> org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl.closeAndUnregisterStream(StateSnapshotContextSynchronousImpl.java:137)
> org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl.close(StateSnapshotContextSynchronousImpl.java:147)
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:399)
> org.apache.flink.streaming.runtime.tasks.StreamTask$CheckpointingOperation.checkpointStreamOperator(StreamTask.java:1157)
> org.apache.flink.streaming.runtime.tasks.StreamTask$CheckpointingOperation.executeCheckpointing(StreamTask.java:1089)
> org.apache.flink.streaming.runtime.tasks.StreamTask.checkpointState(StreamTask.java:653)
> org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:589)
> org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:542)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.notifyCheckpoint(BarrierBuffer.java:378)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.processBarrier(BarrierBuffer.java:281)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.getNextNonBlocked(BarrierBuffer.java:183)
> org.apache.flink.streaming.runtime.io.StreamInputProcessor.processInput(StreamInputProcessor.java:213)
> org.apache.flink.streaming.runtime.tasks.OneInputStreamTask.run(OneInputStreamTask.java:69)
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:262)
> org.apache.flink.runtime.taskmanager.Task.run(Task.java:702)
> java.lang.Thread.run(Thread.java:748)
> ...
> 2017-08-25 17:05:10,139 INFO org.apache.flink.runtime.taskmanager.Task
> - Notifying TaskManager about fatal error. Task
> 'TriggerWindow(SlidingEventTimeWindows(259200000, 3600000),
> AggregatingStateDescriptor{serializer=org.apache.flink.api.java.typeutils.runtime.TupleSerializer@f65b6aa2,
> aggFunction=com.streamingApp$MyAggregateFunction@1f193686},
> EventTimeTrigger(), WindowedStream.aggregate(WindowedStream.java:858)) ->
> Sink: prod_item (2/6)' did not react to cancelling signal in the last 30
> seconds, but is stuck in method:
>
> org.apache.flink.util.AbstractCloseableRegistry.unregisterClosable(AbstractCloseableRegistry.java:84)
> org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl.closeAndUnregisterStream(StateSnapshotContextSynchronousImpl.java:137)
> org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl.close(StateSnapshotContextSynchronousImpl.java:147)
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:399)
> org.apache.flink.streaming.runtime.tasks.StreamTask$CheckpointingOperation.checkpointStreamOperator(StreamTask.java:1157)
> org.apache.flink.streaming.runtime.tasks.StreamTask$CheckpointingOperation.executeCheckpointing(StreamTask.java:1089)
> org.apache.flink.streaming.runtime.tasks.StreamTask.checkpointState(StreamTask.java:653)
> org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:589)
> org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:542)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.notifyCheckpoint(BarrierBuffer.java:378)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.processBarrier(BarrierBuffer.java:281)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.getNextNonBlocked(BarrierBuffer.java:183)
> org.apache.flink.streaming.runtime.io.StreamInputProcessor.processInput(StreamInputProcessor.java:213)
> org.apache.flink.streaming.runtime.tasks.OneInputStreamTask.run(OneInputStreamTask.java:69)
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:262)
> org.apache.flink.runtime.taskmanager.Task.run(Task.java:702)
> java.lang.Thread.run(Thread.java:748)
> .
> 2017-08-25 17:05:10,140 ERROR org.apache.flink.yarn.YarnTaskManager
> -
> ==============================================================
> ====================== FATAL =======================
> ==============================================================
> A fatal error occurred, forcing the TaskManager to shut down: Task
> 'TriggerWindow(SlidingEventTimeWindows(259200000, 3600000),
> AggregatingStateDescriptor{serializer=org.apache.flink.api.java.typeutils.runtime.TupleSerializer@f65b6aa2,
> aggFunction=com.streamingApp$MyAggregateFunction@1f193686},
> EventTimeTrigger(), WindowedStream.aggregate(WindowedStream.java:858)) ->
> Sink: prod_item (2/6)' did not react to cancelling signal in the last 30
> seconds, but is stuck in method:
>
> org.apache.flink.util.AbstractCloseableRegistry.unregisterClosable(AbstractCloseableRegistry.java:84)
> org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl.closeAndUnregisterStream(StateSnapshotContextSynchronousImpl.java:137)
> org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl.close(StateSnapshotContextSynchronousImpl.java:147)
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:399)
> org.apache.flink.streaming.runtime.tasks.StreamTask$CheckpointingOperation.checkpointStreamOperator(StreamTask.java:1157)
> org.apache.flink.streaming.runtime.tasks.StreamTask$CheckpointingOperation.executeCheckpointing(StreamTask.java:1089)
> org.apache.flink.streaming.runtime.tasks.StreamTask.checkpointState(StreamTask.java:653)
> org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:589)
> org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:542)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.notifyCheckpoint(BarrierBuffer.java:378)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.processBarrier(BarrierBuffer.java:281)
> org.apache.flink.streaming.runtime.io.BarrierBuffer.getNextNonBlocked(BarrierBuffer.java:183)
> org.apache.flink.streaming.runtime.io.StreamInputProcessor.processInput(StreamInputProcessor.java:213)
> org.apache.flink.streaming.runtime.tasks.OneInputStreamTask.run(OneInputStreamTask.java:69)
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:262)
> org.apache.flink.runtime.taskmanager.Task.run(Task.java:702)
> java.lang.Thread.run(Thread.java:748)
> {code}
> Why is the task stucked?
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
