This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-1.2
in repository https://gitbox.apache.org/repos/asf/paimon.git
from f8d89e67cb [spark] Upgrade spark version to 4.0.0 (#5711)
new 2aabf52d0f [hive] Hive catalog configuration to get owner for table
creation (#5668)
new 95322af748 [flink] Add 'scan.dedicated-split-generation' to Flink
Batch Source (#5715)
new 0033a73721 [hotfix] Minor refactor for Flink Dedicated Split Gen Source
new 4efd31c85d [test][spark] Fix the unstable testHiveCatalogOptions
(#5738)
new 958e85bac2 [flink] Add support for AvroType enum for debezium format
(#5742)
new dc779c1512 [rest] Adjust writeTimeout of HttpClient to 3 minutes
new 79a3ab2b58 [flink] Postpone mode should support
'partition.sink-strategy' (#5743)
new 1e00d28699 [doc] fix docs gcs in filesystem.md (#5747)
new 68e2692f48 [core] Limit max task number in ManifestReadThreadPool
(#5737)
new 08050cf2bc [fs] Add a requiredOptions method to COSNLoader (#5750)
new a44ae96578 [core] Support dedicated full compact to external paths
(#5674)
new 7306a0ac4b [core] Simplify force rewrite files in Compact Task (#5751)
new 9b4791014a [flink] Make PostponeBucketSink no state and no intended
failure (#5746)
new c2bdec5f6f [core] Introduce auto-buffer-spill for postpone bucket
write (#5744)
new b99bf0b623 [flink] Introduce 'memory' type for 'lookup.cache' to
in-memory cache (#5748)
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/content/flink/sql-query.md | 17 ++
docs/content/maintenance/filesystems.md | 5 +-
.../shortcodes/generated/core_configuration.html | 8 +-
.../generated/flink_connector_configuration.html | 20 +-
.../main/java/org/apache/paimon/CoreOptions.java | 13 +-
.../java/org/apache/paimon/rest/HttpClient.java | 1 +
.../org/apache/paimon/utils/ThreadPoolUtils.java | 16 +-
.../paimon/utils/SemaphoredDelegatingExecutor.java | 184 ++++++++++++++++
.../java/org/apache/paimon/KeyValueFileStore.java | 2 +
.../org/apache/paimon/append/AppendOnlyWriter.java | 216 +++----------------
.../append/BucketedAppendCompactManager.java | 19 +-
.../org/apache/paimon/compact/CompactUnit.java | 43 ++--
.../paimon/crosspartition/GlobalIndexAssigner.java | 16 +-
.../java/org/apache/paimon/lookup/BulkLoader.java | 101 +--------
.../LevelSortedRun.java => lookup/ByteArray.java} | 39 ++--
.../org/apache/paimon/lookup/ListBulkLoader.java | 11 +-
.../java/org/apache/paimon/lookup/ListState.java | 14 +-
.../java/org/apache/paimon/lookup/SetState.java | 17 +-
.../main/java/org/apache/paimon/lookup/State.java | 15 +-
.../org/apache/paimon/lookup/StateFactory.java | 31 ++-
.../org/apache/paimon/lookup/ValueBulkLoader.java | 10 +-
.../java/org/apache/paimon/lookup/ValueState.java | 14 +-
.../paimon/lookup/memory/InMemoryListState.java | 77 +++++++
.../paimon/lookup/memory/InMemorySetState.java | 69 ++++++
.../apache/paimon/lookup/memory/InMemoryState.java | 64 ++++++
.../paimon/lookup/memory/InMemoryStateFactory.java | 66 ++++++
.../paimon/lookup/memory/InMemoryValueState.java | 76 +++++++
.../RocksDBBulkLoader.java} | 33 ++-
.../lookup/{ => rocksdb}/RocksDBListState.java | 17 +-
.../lookup/{ => rocksdb}/RocksDBOptions.java | 2 +-
.../lookup/{ => rocksdb}/RocksDBSetState.java | 10 +-
.../paimon/lookup/{ => rocksdb}/RocksDBState.java | 52 ++---
.../lookup/{ => rocksdb}/RocksDBStateFactory.java | 14 +-
.../lookup/{ => rocksdb}/RocksDBValueState.java | 21 +-
.../paimon/mergetree/compact/CompactStrategy.java | 39 ++--
.../mergetree/compact/FileRewriteCompactTask.java | 67 ++++++
.../mergetree/compact/MergeTreeCompactManager.java | 47 +++--
.../mergetree/compact/MergeTreeCompactTask.java | 56 ++---
.../paimon/operation/AbstractFileStoreWrite.java | 2 +-
.../operation/BucketedAppendFileStoreWrite.java | 1 +
.../paimon/operation/KeyValueFileStoreWrite.java | 3 +-
.../postpone/PostponeBucketFileStoreWrite.java | 71 ++++++-
.../paimon/postpone/PostponeBucketWriter.java | 141 +++++++++++--
.../paimon/utils/ManifestReadThreadPool.java | 27 ++-
.../java/org/apache/paimon/utils/SinkWriter.java | 234 +++++++++++++++++++++
.../apache/paimon/append/AppendOnlyWriterTest.java | 1 +
.../append/BucketedAppendCompactManagerTest.java | 1 +
.../apache/paimon/append/FullCompactTaskTest.java | 2 +-
.../apache/paimon/format/FileFormatSuffixTest.java | 2 +-
.../apache/paimon/mergetree/MergeTreeTestBase.java | 6 +-
.../compact/MergeTreeCompactManagerTest.java | 9 +-
.../paimon/table/PrimaryKeySimpleTableTest.java | 38 ++++
.../configuration/ConfigOptionsDocGenerator.java | 2 +-
.../java/org/apache/paimon/cosn/COSNLoader.java | 12 ++
.../cdc/format/debezium/DebeziumSchemaUtils.java | 1 +
.../paimon/flink/sink/cdc/CdcAppendTableSink.java | 7 +
.../sink/cdc/CdcAppendTableWriteOperator.java | 13 ++
.../flink/sink/cdc/FlinkCdcMultiTableSink.java | 2 +-
.../format/debezium/DebeziumSchemaUtilsTest.java | 153 ++++++++++++++
.../apache/paimon/flink/FlinkConnectorOptions.java | 20 +-
.../flink/lookup/FileStoreLookupFunction.java | 4 +-
.../paimon/flink/lookup/FullCacheLookupTable.java | 38 +++-
.../flink/lookup/NoPrimaryKeyLookupTable.java | 19 +-
.../paimon/flink/lookup/PrimaryKeyLookupTable.java | 11 +-
.../flink/lookup/SecondaryIndexLookupTable.java | 4 +-
.../postpone/PostponeBucketCompactSplitSource.java | 2 +-
.../apache/paimon/flink/sink/AppendTableSink.java | 2 +
.../apache/paimon/flink/sink/FlinkSinkBuilder.java | 13 +-
.../apache/paimon/flink/sink/FlinkWriteSink.java | 50 +++++
...ixedBucketSink.java => PostponeBucketSink.java} | 20 +-
.../RestoreAndFailCommittableStateManager.java | 56 +----
...er.java => RestoreCommittableStateManager.java} | 28 +--
.../paimon/flink/sink/RowAppendTableSink.java | 40 +---
.../sink/RowDataHashPartitionChannelComputer.java | 10 +-
.../flink/source/ContinuousFileStoreSource.java | 2 +-
.../paimon/flink/source/FlinkSourceBuilder.java | 17 +-
.../align/AlignedContinuousFileStoreSource.java | 2 +-
.../flink/source/operator/MonitorSource.java | 38 +++-
.../paimon/flink/source/operator/ReadOperator.java | 25 ++-
.../apache/paimon/flink/utils/TableScanUtils.java | 16 +-
.../apache/paimon/flink/BatchFileStoreITCase.java | 38 ++++
.../org/apache/paimon/flink/LookupJoinITCase.java | 6 +-
.../paimon/flink/PostponeBucketTableITCase.java | 11 +-
.../flink/lookup/FileStoreLookupFunctionTest.java | 2 +-
.../paimon/flink/lookup/LookupTableTest.java | 90 +++++---
.../paimon/flink/lookup/RocksDBListStateTest.java | 4 +-
.../flink/procedure/CompactProcedureITCase.java | 67 ++++++
.../sink/BatchWriteGeneratorTagOperatorTest.java | 4 +-
.../paimon/flink/sink/CommitterOperatorTest.java | 2 +-
.../paimon/flink/sink/StoreMultiCommitterTest.java | 2 +-
.../flink/source/operator/OperatorSourceTest.java | 60 +++++-
.../java/org/apache/paimon/hive/HiveCatalog.java | 20 +-
.../paimon/spark/SparkCatalogWithHiveTest.java | 1 +
.../spark/SparkGenericCatalogWithHiveTest.java | 30 ++-
.../catalog/functions/BucketFunctionTest.java | 6 +
95 files changed, 2166 insertions(+), 844 deletions(-)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/utils/SemaphoredDelegatingExecutor.java
copy
paimon-core/src/main/java/org/apache/paimon/{mergetree/LevelSortedRun.java =>
lookup/ByteArray.java} (61%)
copy
paimon-api/src/main/java/org/apache/paimon/rest/responses/PagedResponse.java =>
paimon-core/src/main/java/org/apache/paimon/lookup/ListBulkLoader.java (78%)
copy
paimon-common/src/main/java/org/apache/paimon/lookup/LookupStoreWriter.java =>
paimon-core/src/main/java/org/apache/paimon/lookup/ListState.java (76%)
copy
paimon-common/src/main/java/org/apache/paimon/lookup/LookupStoreWriter.java =>
paimon-core/src/main/java/org/apache/paimon/lookup/SetState.java (71%)
copy
paimon-common/src/main/java/org/apache/paimon/lookup/LookupStoreReader.java =>
paimon-core/src/main/java/org/apache/paimon/lookup/State.java (76%)
copy
paimon-common/src/main/java/org/apache/paimon/lookup/LookupStoreReader.java =>
paimon-core/src/main/java/org/apache/paimon/lookup/StateFactory.java (52%)
copy paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFTokenLoader.java
=> paimon-core/src/main/java/org/apache/paimon/lookup/ValueBulkLoader.java (78%)
copy
paimon-common/src/main/java/org/apache/paimon/lookup/LookupStoreReader.java =>
paimon-core/src/main/java/org/apache/paimon/lookup/ValueState.java (74%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemoryListState.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemorySetState.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemoryState.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemoryStateFactory.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemoryValueState.java
copy paimon-core/src/main/java/org/apache/paimon/lookup/{BulkLoader.java =>
rocksdb/RocksDBBulkLoader.java} (82%)
rename paimon-core/src/main/java/org/apache/paimon/lookup/{ =>
rocksdb}/RocksDBListState.java (87%)
rename paimon-core/src/main/java/org/apache/paimon/lookup/{ =>
rocksdb}/RocksDBOptions.java (99%)
rename paimon-core/src/main/java/org/apache/paimon/lookup/{ =>
rocksdb}/RocksDBSetState.java (95%)
rename paimon-core/src/main/java/org/apache/paimon/lookup/{ =>
rocksdb}/RocksDBState.java (82%)
rename paimon-core/src/main/java/org/apache/paimon/lookup/{ =>
rocksdb}/RocksDBStateFactory.java (94%)
rename paimon-core/src/main/java/org/apache/paimon/lookup/{ =>
rocksdb}/RocksDBValueState.java (86%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/mergetree/compact/FileRewriteCompactTask.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/utils/SinkWriter.java
create mode 100644
paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumSchemaUtilsTest.java
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/{FixedBucketSink.java
=> PostponeBucketSink.java} (73%)
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/{RestoreAndFailCommittableStateManager.java
=> RestoreCommittableStateManager.java} (75%)
