This is an automated email from the ASF dual-hosted git repository.
zhijiang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from caff2ed [FLINK-13195][sql client] Add create/drop table support in
sql client
add b2022ba [FLINK-14845][runtime][table] Move block compression utils
from table module to runtime module.
add 66d4d7d [FLINK-14845][runtime] Introduce data compression to reduce
disk and network IO of shuffle.
No new revisions were added by this update.
Summary of changes:
.../netty_shuffle_environment_configuration.html | 12 ++
.../NettyShuffleEnvironmentOptions.java | 35 ++++
flink-runtime/pom.xml | 13 ++
.../io}/compression/BlockCompressionFactory.java | 2 +-
.../runtime/io}/compression/BlockCompressor.java | 2 +-
.../runtime/io}/compression/BlockDecompressor.java | 2 +-
.../io}/compression/DataCorruptionException.java | 2 +-
.../compression/InsufficientBufferException.java | 2 +-
.../compression/Lz4BlockCompressionFactory.java | 4 +-
.../io}/compression/Lz4BlockCompressor.java | 6 +-
.../io}/compression/Lz4BlockDecompressor.java | 6 +-
.../io/network/NettyShuffleServiceFactory.java | 5 +-
.../flink/runtime/io/network/buffer/Buffer.java | 10 +
.../io/network/buffer/BufferCompressor.java | 116 +++++++++++
.../io/network/buffer/BufferDecompressor.java | 104 ++++++++++
.../runtime/io/network/buffer/NetworkBuffer.java | 35 ++++
.../buffer/ReadOnlySlicedNetworkBuffer.java | 21 +-
.../CreditBasedPartitionRequestClientHandler.java | 1 +
.../CreditBasedSequenceNumberingViewReader.java | 2 +-
.../runtime/io/network/netty/NettyMessage.java | 13 +-
.../partition/BoundedBlockingSubpartition.java | 8 +-
.../BoundedBlockingSubpartitionReader.java | 2 +-
.../network/partition/BufferReaderWriterUtil.java | 47 ++---
.../partition/NoOpResultSubpartitionView.java | 2 +-
.../network/partition/PipelinedSubpartition.java | 11 +-
.../partition/PipelinedSubpartitionView.java | 4 +-
.../ReleaseOnConsumptionResultPartition.java | 14 +-
.../io/network/partition/ResultPartition.java | 7 +
.../network/partition/ResultPartitionFactory.java | 24 ++-
.../io/network/partition/ResultSubpartition.java | 8 +
.../network/partition/ResultSubpartitionView.java | 4 +-
.../io/network/partition/consumer/InputGate.java | 4 +
.../partition/consumer/LocalInputChannel.java | 2 +-
.../partition/consumer/RemoteInputChannel.java | 5 +
.../partition/consumer/SingleInputGate.java | 80 +++++---
.../partition/consumer/SingleInputGateFactory.java | 22 ++-
.../NettyShuffleEnvironmentConfiguration.java | 49 ++++-
flink-runtime/src/main/resources/META-INF/NOTICE | 1 +
.../io}/compression/BlockCompressionTest.java | 4 +-
.../io/network/NettyShuffleEnvironmentBuilder.java | 26 ++-
.../io/network/buffer/BufferCompressionTest.java | 217 +++++++++++++++++++++
.../network/netty/CancelPartitionRequestTest.java | 2 +-
...editBasedPartitionRequestClientHandlerTest.java | 63 ++++--
.../netty/NettyMessageSerializationTest.java | 74 ++++++-
.../network/netty/PartitionRequestQueueTest.java | 8 +-
...oundedBlockingSubpartitionAvailabilityTest.java | 4 +-
.../BoundedBlockingSubpartitionWriteReadTest.java | 65 ++++--
.../io/network/partition/BoundedDataTestBase.java | 32 ++-
.../partition/FileChannelBoundedDataTest.java | 4 +-
.../network/partition/InputGateFairnessTest.java | 3 +-
.../io/network/partition/PartitionTestUtils.java | 14 ++
.../partition/PipelinedSubpartitionTest.java | 4 +-
.../PipelinedSubpartitionWithReadViewTest.java | 41 +++-
.../network/partition/ResultPartitionBuilder.java | 26 ++-
.../partition/ResultPartitionFactoryTest.java | 5 +-
.../io/network/partition/SubpartitionTestBase.java | 2 +-
.../partition/consumer/LocalInputChannelTest.java | 60 ++++--
.../partition/consumer/SingleInputGateBuilder.java | 11 +-
.../partition/consumer/SingleInputGateTest.java | 117 ++++++++---
.../io/network/util/TestSubpartitionConsumer.java | 2 +-
flink-table/flink-table-runtime-blink/pom.xml | 31 ---
.../runtime/hashtable/BaseHybridHashTable.java | 2 +-
.../runtime/hashtable/BinaryHashPartition.java | 2 +-
.../runtime/hashtable/LongHybridHashTable.java | 2 +-
.../runtime/io/CompressedBlockChannelReader.java | 6 +-
.../runtime/io/CompressedBlockChannelWriter.java | 4 +-
...CompressedHeaderlessChannelReaderInputView.java | 6 +-
...ompressedHeaderlessChannelWriterOutputView.java | 4 +-
.../sort/AbstractBinaryExternalMerger.java | 2 +-
.../operators/sort/BinaryExternalMerger.java | 2 +-
.../operators/sort/BinaryExternalSorter.java | 2 +-
.../operators/sort/BinaryKVExternalMerger.java | 2 +-
.../operators/sort/BufferedKVExternalSorter.java | 2 +-
.../flink/table/runtime/util/FileChannelUtil.java | 2 +-
.../src/main/resources/META-INF/NOTICE | 1 -
.../io/CompressedHeaderlessChannelTest.java | 4 +-
.../test/runtime/ShuffleCompressionITCase.java | 173 ++++++++++++++++
77 files changed, 1459 insertions(+), 259 deletions(-)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/BlockCompressionFactory.java
(98%)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/BlockCompressor.java
(97%)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/BlockDecompressor.java
(97%)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/DataCorruptionException.java
(96%)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/InsufficientBufferException.java
(96%)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/Lz4BlockCompressionFactory.java
(93%)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/Lz4BlockCompressor.java
(93%)
rename
{flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime
=>
flink-runtime/src/main/java/org/apache/flink/runtime/io}/compression/Lz4BlockDecompressor.java
(94%)
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferCompressor.java
create mode 100644
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferDecompressor.java
rename
{flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/runtime
=>
flink-runtime/src/test/java/org/apache/flink/runtime/io}/compression/BlockCompressionTest.java
(97%)
create mode 100644
flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferCompressionTest.java
create mode 100644
flink-tests/src/test/java/org/apache/flink/test/runtime/ShuffleCompressionITCase.java
