This is an automated email from the ASF dual-hosted git repository. guoweijie pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
from cdf314d30b5 [FLINK-34116][test] Enhance GlobalConfigurationTest.testInvalidStandardYamlFile for JDK compatibility. add 2839d06559c [FLINK-33743][runtime] Distinguish channel and subpartition add c65d5f18ad5 [FLINK-33743][runtime] Replace consumer-side SubpartitionId with InputChannelId add 0e8b9808839 [FLINK-33743][runtime] Modify parameter subpartitionIndex to subpartitionIndexSet add 6b44f29a1ce [FLINK-33743][runtime] Identify subpartitionId in notifyRequiredSegment add f0436e53313 [FLINK-33743][runtime] Disable tier shuffle during recovery add 439d1091daa [FLINK-33743][runtime] Acquire subpartition id for input channel dynamically in tiered shuffle add 4e6796dd147 [FLINK-33743][runtime] Change numCreditsAvailable to isCreditAvailable in ResultSubpartitionView#getAvailabilityAndBacklog add 8651f734b16 [FLINK-33743][runtime] Add ResultSubpartitionView parameter to notifyDataAvailable() method add 1199c7106a4 [FLINK-33743][runtime] Support consuming multiple subpartition in one inputchannel add f2417a74bd7 [FLINK-33743][runtime] Rename InputChannelStatus to SubpartitionStatus add 32de7521ac7 [FLINK-33743][runtime] Align watermark at subpartition granularity add 8358e3aa2ee [FLINK-33743][runtime] Ignore RecordAttributes when adaptive parallelism is used add be127310958 [FLINK-33743][runtime] Flush one accumulated buffer at a time add a886339dbb3 [FLINK-33743][runtime] Optimize partial record split logic No new revisions were added by this update. Summary of changes: .../ResultPartitionDeploymentDescriptor.java | 4 + .../flink/runtime/executiongraph/IndexRange.java | 4 +- .../IntermediateResultPartition.java | 8 + .../io/network/NetworkSequenceViewReader.java | 15 +- .../runtime/io/network/PartitionRequestClient.java | 11 +- .../runtime/io/network/api/RecoveryMetadata.java | 72 ++++ .../network/api/serialization/EventSerializer.java | 13 + .../api/writer/ChannelSelectorRecordWriter.java | 10 +- .../io/network/api/writer/RecordWriter.java | 30 +- .../network/api/writer/ResultPartitionWriter.java | 8 +- .../flink/runtime/io/network/buffer/Buffer.java | 50 ++- .../runtime/io/network/buffer/BufferBuilder.java | 10 + .../metrics/ExclusiveBuffersUsageGauge.java | 4 +- .../network/metrics/FloatingBuffersUsageGauge.java | 2 +- .../io/network/metrics/InputGateMetrics.java | 12 +- .../CreditBasedPartitionRequestClientHandler.java | 5 +- .../CreditBasedSequenceNumberingViewReader.java | 56 ++- .../runtime/io/network/netty/NettyMessage.java | 58 ++- .../network/netty/NettyPartitionRequestClient.java | 21 +- .../netty/NettyPartitionRequestListener.java | 13 +- .../io/network/netty/PartitionRequestQueue.java | 8 +- .../netty/PartitionRequestServerHandler.java | 5 +- ...edBlockingSubpartitionDirectTransferReader.java | 11 +- .../BoundedBlockingSubpartitionReader.java | 11 +- .../partition/BufferAvailabilityListener.java | 8 +- .../network/partition/BufferReaderWriterUtil.java | 18 + ...ithChannel.java => BufferWithSubpartition.java} | 14 +- .../partition/BufferWritingResultPartition.java | 9 +- .../runtime/io/network/partition/DataBuffer.java | 19 +- .../io/network/partition/DeduplicatedQueue.java | 70 ++++ .../io/network/partition/HashBasedDataBuffer.java | 60 ++-- .../partition/NoOpResultSubpartitionView.java | 7 +- .../network/partition/PartitionedFileWriter.java | 39 ++- .../network/partition/PipelinedSubpartition.java | 4 +- .../partition/PipelinedSubpartitionView.java | 11 +- .../io/network/partition/ResultPartition.java | 38 ++ .../network/partition/ResultPartitionFactory.java | 11 +- .../network/partition/ResultPartitionManager.java | 13 +- .../network/partition/ResultPartitionProvider.java | 6 +- .../partition/ResultSubpartitionIndexSet.java | 77 ++++ .../network/partition/ResultSubpartitionView.java | 15 +- .../partition/RoundRobinSubpartitionSelector.java | 73 ++++ .../io/network/partition/SortBasedDataBuffer.java | 12 +- .../runtime/io/network/partition/SortBuffer.java | 34 +- .../partition/SortMergeResultPartition.java | 44 +-- .../partition/SortMergeSubpartitionReader.java | 11 +- .../io/network/partition/SubpartitionSelector.java | 56 +++ .../partition/UnionResultSubpartitionView.java | 258 ++++++++++++++ .../network/partition/consumer/InputChannel.java | 60 +++- .../partition/consumer/LocalInputChannel.java | 57 ++- .../consumer/LocalRecoveredInputChannel.java | 7 +- .../partition/consumer/RecoveredInputChannel.java | 13 +- .../partition/consumer/RemoteInputChannel.java | 81 ++++- .../consumer/RemoteRecoveredInputChannel.java | 7 +- .../partition/consumer/SingleInputGate.java | 389 ++++++++++++++------- .../partition/consumer/SingleInputGateFactory.java | 150 +++++--- .../partition/consumer/UnknownInputChannel.java | 16 +- .../partition/hybrid/HsResultPartition.java | 10 +- .../partition/hybrid/HsSubpartitionConsumer.java | 13 +- .../tiered/common/TieredStorageInputChannelId.java | 58 +++ .../hybrid/tiered/netty/NettyConnectionReader.java | 10 +- .../tiered/netty/NettyConnectionReaderImpl.java | 25 +- .../tiered/netty/NettyConnectionWriterImpl.java | 12 +- .../netty/TieredStorageNettyServiceImpl.java | 37 +- .../netty/TieredStorageResultSubpartitionView.java | 18 +- .../tiered/shuffle/TieredResultPartition.java | 4 +- .../shuffle/TieredResultPartitionFactory.java | 20 +- .../tiered/storage/AvailabilityNotifier.java | 11 + .../hybrid/tiered/storage/BufferAccumulator.java | 9 +- .../tiered/storage/HashBufferAccumulator.java | 24 +- .../storage/HashSubpartitionBufferAccumulator.java | 44 ++- .../HashSubpartitionBufferAccumulatorContext.java | 10 +- .../tiered/storage/SortBufferAccumulator.java | 55 ++- .../storage/TieredStorageConsumerClient.java | 22 ++ .../tiered/storage/TieredStorageConsumerSpec.java | 21 +- .../storage/TieredStorageProducerClient.java | 80 ++--- .../tiered/storage/TieredStorageSortBuffer.java | 46 ++- .../hybrid/tiered/tier/TierConsumerAgent.java | 13 + .../hybrid/tiered/tier/TierProducerAgent.java | 17 +- .../hybrid/tiered/tier/disk/DiskCacheManager.java | 11 +- .../tiered/tier/disk/DiskTierConsumerAgent.java | 38 +- .../tiered/tier/disk/DiskTierProducerAgent.java | 23 +- .../tier/memory/MemoryTierConsumerAgent.java | 38 +- .../tier/memory/MemoryTierProducerAgent.java | 18 +- .../tier/remote/RemoteTierConsumerAgent.java | 85 ++++- .../tiered/tier/remote/RemoteTierFactory.java | 5 +- .../tier/remote/RemoteTierProducerAgent.java | 14 +- .../flink/runtime/shuffle/PartitionDescriptor.java | 13 +- .../SequentialChannelStateReaderImplTest.java | 4 +- .../ResultPartitionDeploymentDescriptorTest.java | 3 +- .../io/network/TestingPartitionRequestClient.java | 6 +- .../api/serialization/EventSerializerTest.java | 14 +- .../api/writer/BroadcastRecordWriterTest.java | 44 +-- .../api/writer/RecordWriterDelegateTest.java | 9 +- .../io/network/api/writer/RecordWriterTest.java | 71 ++-- .../network/netty/CancelPartitionRequestTest.java | 28 +- .../netty/ClientTransportErrorHandlingTest.java | 7 +- ...editBasedPartitionRequestClientHandlerTest.java | 24 +- ...CreditBasedSequenceNumberingViewReaderTest.java | 12 +- .../NettyMessageClientDecoderDelegateTest.java | 4 +- .../NettyMessageClientSideSerializationTest.java | 3 +- .../NettyMessageServerSideSerializationTest.java | 6 +- .../netty/NettyPartitionRequestClientTest.java | 14 +- .../runtime/io/network/netty/NettyTestUtil.java | 1 + .../network/netty/PartitionRequestQueueTest.java | 70 ++-- .../netty/PartitionRequestRegistrationTest.java | 13 +- .../netty/PartitionRequestServerHandlerTest.java | 3 +- .../netty/ServerTransportErrorHandlingTest.java | 9 +- .../AwaitableBufferAvailablityListener.java | 2 +- ...oundedBlockingSubpartitionAvailabilityTest.java | 6 +- .../BoundedBlockingSubpartitionWriteReadTest.java | 9 +- .../partition/CountingAvailabilityListener.java | 2 +- .../io/network/partition/DataBufferTest.java | 32 +- .../partition/FileChannelBoundedDataTest.java | 5 +- .../network/partition/InputChannelTestUtils.java | 5 +- .../network/partition/InputGateFairnessTest.java | 20 +- .../partition/MockResultPartitionWriter.java | 3 +- .../partition/NoOpBufferAvailablityListener.java | 2 +- .../PartitionRequestListenerManagerTest.java | 18 +- .../io/network/partition/PartitionTestUtils.java | 11 +- .../partition/PartitionedFileWriteReadTest.java | 39 ++- .../PipelinedApproximateSubpartitionTest.java | 4 +- .../PipelinedSubpartitionWithReadViewTest.java | 24 +- .../network/partition/ResultPartitionBuilder.java | 3 +- .../partition/ResultPartitionManagerTest.java | 27 +- .../io/network/partition/ResultPartitionTest.java | 25 +- .../SortMergeResultPartitionReadSchedulerTest.java | 6 +- .../partition/SortMergeResultPartitionTest.java | 53 +-- .../partition/SortMergeSubpartitionReaderTest.java | 12 +- .../io/network/partition/SubpartitionTestBase.java | 2 +- .../partition/TestingPartitionRequestListener.java | 2 +- .../network/partition/TestingResultPartition.java | 2 +- .../partition/TestingResultPartitionProvider.java | 16 +- .../TestingSubpartitionCreatedViewReader.java | 19 +- .../partition/UnionResultSubpartitionViewTest.java | 185 ++++++++++ .../consumer/InputBuffersMetricsTest.java | 2 +- .../partition/consumer/InputChannelBuilder.java | 18 +- .../partition/consumer/InputChannelTest.java | 10 +- .../partition/consumer/LocalInputChannelTest.java | 61 ++-- .../consumer/RecoveredInputChannelTest.java | 5 +- .../partition/consumer/RemoteInputChannelTest.java | 83 ++--- .../partition/consumer/SingleInputGateBuilder.java | 16 +- .../partition/consumer/SingleInputGateTest.java | 152 ++++---- .../partition/consumer/TestInputChannel.java | 12 +- .../partition/hybrid/HsResultPartitionTest.java | 39 ++- .../partition/hybrid/HsSubpartitionViewTest.java | 18 +- .../hybrid/tiered/TestingBufferAccumulator.java | 17 +- .../hybrid/tiered/TestingTierProducerAgent.java | 8 +- .../tiered/netty/NettyConnectionReaderTest.java | 17 +- .../tiered/netty/NettyConnectionWriterTest.java | 20 +- .../tiered/netty/TestingNettyConnectionReader.java | 10 +- .../tiered/netty/TestingTierConsumerAgent.java | 10 + .../netty/TieredStorageConsumerClientTest.java | 14 +- .../TieredStorageResultSubpartitionViewTest.java | 10 +- .../tiered/shuffle/TieredResultPartitionTest.java | 7 +- .../tiered/storage/HashBufferAccumulatorTest.java | 54 ++- .../tiered/storage/SortBufferAccumulatorTest.java | 58 ++- .../storage/TieredStorageSortBufferTest.java | 44 ++- .../tiered/tier/disk/DiskCacheManagerTest.java | 8 +- .../tier/disk/DiskTierProducerAgentTest.java | 25 +- .../tier/memory/MemoryTierProducerAgentTest.java | 16 +- .../tier/remote/RemoteTierConsumerAgentTest.java | 29 +- .../tier/remote/TestingAvailabilityNotifier.java | 7 + .../runtime/io/network/util/TestBufferFactory.java | 18 +- .../io/network/util/TestSubpartitionConsumer.java | 2 +- .../runtime/shuffle/NettyShuffleUtilsTest.java | 5 +- .../shuffle/PartitionDescriptorBuilder.java | 3 +- .../runtime/state/ChannelPersistenceITCase.java | 10 +- .../streaming/api/watermark/InternalWatermark.java | 55 +++ .../flink/streaming/api/watermark/Watermark.java | 7 +- .../streaming/runtime/io/RecordWriterOutput.java | 36 +- .../io/StreamMultipleInputProcessorFactory.java | 3 +- .../runtime/io/StreamTwoInputProcessorFactory.java | 6 +- .../runtime/streamrecord/StreamElement.java | 2 +- .../streamrecord/StreamElementSerializer.java | 18 +- .../runtime/tasks/OneInputStreamTask.java | 3 +- .../watermarkstatus/StatusWatermarkValve.java | 316 +++++++++++------ .../benchmark/SingleInputGateBenchmarkFactory.java | 25 +- .../checkpointing/AlternatingCheckpointsTest.java | 100 +++--- .../checkpointing/CheckpointedInputGateTest.java | 8 +- .../io/checkpointing/UnalignedCheckpointsTest.java | 1 + .../streamrecord/StreamElementSerializerTest.java | 4 + .../watermarkstatus/StatusWatermarkValveTest.java | 8 +- .../flink/test/runtime/BatchShuffleITCaseBase.java | 58 ++- .../flink/test/runtime/BlockingShuffleITCase.java | 12 +- .../flink/test/runtime/HybridShuffleITCase.java | 28 +- 186 files changed, 3805 insertions(+), 1504 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/RecoveryMetadata.java rename flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/{BufferWithChannel.java => BufferWithSubpartition.java} (77%) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/DeduplicatedQueue.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartitionIndexSet.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/RoundRobinSubpartitionSelector.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SubpartitionSelector.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/UnionResultSubpartitionView.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/common/TieredStorageInputChannelId.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/UnionResultSubpartitionViewTest.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/InternalWatermark.java