This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 894feed5f3e Revert "[FLINK-34123][core][type] Introduce built-in
serialization support for Map, List, and Collection"
new 1d62a358719 [FLINK-35533][runtime] Create teir factories according to
the configured remote tier class
new fe74f74d3e6 [FLINK-35533][runtime] Introduce TierShuffleDescriptor and
RemoteTierShuffleDescriptor
new 32b563dd29c [FLINK-35533][runtime] Refactor the APIs and
implementations for supporting remote tier plugin
new 23fa2aeb018 [FLINK-35533][runtime] Add a option to config the external
remote tier factory class
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../generated/all_taskmanager_network_section.html | 6 +
.../netty_shuffle_environment_configuration.html | 6 +
.../NettyShuffleEnvironmentOptions.java | 18 +
.../io/network/NettyShuffleServiceFactory.java | 3 +-
.../network/partition/ResultPartitionFactory.java | 56 ++-
.../partition/consumer/InputGateSpecUtils.java | 12 +-
.../partition/consumer/SingleInputGate.java | 3 +
.../partition/consumer/SingleInputGateFactory.java | 35 +-
.../tiered/common/TieredStorageConfiguration.java | 516 +--------------------
.../hybrid/tiered/common/TieredStorageUtils.java | 81 ++++
.../tiered/shuffle/TierFactoryInitializer.java | 129 ++++++
.../shuffle/TieredInternalShuffleMaster.java | 74 ++-
.../shuffle/TieredResultPartitionFactory.java | 52 ++-
.../storage/TieredStorageConsumerClient.java | 65 ++-
.../tiered/storage/TieredStorageConsumerSpec.java | 8 +
.../tiered/storage/TieredStorageMasterClient.java | 44 +-
.../tiered/storage/TieredStorageMemoryManager.java | 7 +
.../storage/TieredStorageMemoryManagerImpl.java | 5 +
.../hybrid/tiered/tier/NoOpMasterAgent.java | 24 +-
.../tiered/tier/NoOpTierShuffleDescriptor.java} | 14 +-
.../hybrid/tiered/tier/TierConsumerAgent.java | 12 +-
.../partition/hybrid/tiered/tier/TierFactory.java | 21 +-
.../hybrid/tiered/tier/TierMasterAgent.java | 25 +-
.../hybrid/tiered/tier/TierShuffleDescriptor.java} | 20 +-
...oOpMasterAgent.java => TierShuffleHandler.java} | 28 +-
.../tiered/tier/UnknownTierShuffleDescriptor.java} | 18 +-
.../tiered/tier/disk/DiskTierConsumerAgent.java | 6 +
.../hybrid/tiered/tier/disk/DiskTierFactory.java | 73 +--
.../tiered/tier/disk/DiskTierProducerAgent.java | 3 +-
.../tier/memory/MemoryTierConsumerAgent.java | 6 +
.../tiered/tier/memory/MemoryTierFactory.java | 52 ++-
.../tier/memory/MemoryTierProducerAgent.java | 8 +-
.../tier/remote/RemoteTierConsumerAgent.java | 6 +
.../tiered/tier/remote/RemoteTierFactory.java | 60 ++-
.../tiered/tier/remote/RemoteTierMasterAgent.java | 36 +-
.../tier/remote/RemoteTierProducerAgent.java | 3 +-
.../RemoteTierShuffleDescriptor.java} | 22 +-
.../runtime/shuffle/NettyShuffleDescriptor.java | 20 +
.../flink/runtime/shuffle/NettyShuffleMaster.java | 23 +-
.../flink/runtime/shuffle/NettyShuffleUtils.java | 3 +-
.../NettyShuffleEnvironmentConfiguration.java | 22 +-
.../io/network/NettyShuffleEnvironmentBuilder.java | 9 +
.../network/partition/ResultPartitionBuilder.java | 36 ++
.../partition/ResultPartitionFactoryTest.java | 4 +-
.../partition/consumer/GateBuffersSpecTest.java | 20 +-
.../tiered/TestingTieredStorageMemoryManager.java | 18 +
.../tiered/netty/TestingTierConsumerAgent.java | 20 +
.../netty/TieredStorageConsumerClientTest.java | 7 +
.../tiered/shuffle/TierFactoryInitializerTest.java | 118 +++++
.../tiered/shuffle/TieredResultPartitionTest.java | 17 +-
.../hybrid/tiered/storage/TestingTierFactory.java | 93 +++-
.../tier/remote/RemoteTierConsumerAgentTest.java | 2 +
.../tier/remote/RemoteTierMasterAgentTest.java | 10 +-
.../adaptivebatch/BatchJobRecoveryTest.java | 4 +-
.../flink/runtime/shuffle/ShuffleMasterTest.java | 2 +-
.../flink/runtime/shuffle/ShuffleTestUtils.java | 3 +-
.../util/NettyShuffleDescriptorBuilder.java | 11 +-
.../NoOpTierShuffleDescriptor.java} | 15 +-
58 files changed, 1246 insertions(+), 768 deletions(-)
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TierFactoryInitializer.java
copy
flink-runtime/src/{test/java/org/apache/flink/runtime/shuffle/ShuffleTestUtils.java
=>
main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/NoOpTierShuffleDescriptor.java}
(66%)
copy
flink-runtime/src/{test/java/org/apache/flink/runtime/shuffle/ShuffleTestUtils.java
=>
main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/TierShuffleDescriptor.java}
(65%)
copy
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/{NoOpMasterAgent.java
=> TierShuffleHandler.java} (55%)
copy
flink-runtime/src/{test/java/org/apache/flink/runtime/shuffle/ShuffleTestUtils.java
=>
main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/UnknownTierShuffleDescriptor.java}
(62%)
copy
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/{NoOpMasterAgent.java
=> remote/RemoteTierShuffleDescriptor.java} (66%)
create mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/shuffle/TierFactoryInitializerTest.java
copy
flink-runtime/src/test/java/org/apache/flink/runtime/{shuffle/ShuffleTestUtils.java
=> util/NoOpTierShuffleDescriptor.java} (63%)
