This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 77078566bd [Python] fix oss file_io client_kwargs for pyarrow 16.0
(#6255)
new ade0600b3a [core] Modify all row lineage to row tracking in codes
(#6262)
new 6ffe696519 [core][flink] Remove withMemoryPool in TableWriteImpl
(#6265)
new b572f8173f [doc] Fix Python code syntax errors and typo in PVFS
documentation (#6269)
new 21328c6c3d [hotfix] Fix comments in MergeFileSplitRead and style
new 95c540a6e4 [append] Fix error of compaction append table with deletion
vectors (#6258)
new 4f2c4f4c7f [core] Primary key types should not be changed (#6264)
new d050f44aa2 [spark] Avoid unnecessary repeated resolve MergeIntoTable
(#6275)
new d1767f50f7 [iceberg] Enhance Iceberg timestamp type compatibility with
extended precision support. (#6278)
new 429827ea86 [core] Separate indexIncrement into dataIncrement and
compactIncrement (#6285)
new 3b1f7593cc [core] Fix the commit kind when performing row-level
changes on non-pk table (#6025)
new f3955cd5c9 [hotfix] Minor refactor for FileStoreCommitImpl
new abe882a5fa [spark] Resolve function after all args have been resolved
(#6292)
new db8610d00c [rest] Add tableType parameter support for listing tables
(#6295)
new 16f0ceadfc [cdc] Fix cannot be cast to float due to precision loss
(#6291)
new cee4a541a8 [lance] Remove shade for paimon-lance (#6310)
new 7b49031bf4 [core] nested-update supports limit the input (#6309)
new af25ba1223 [core] Cross partition can work with fixed bucket and
postpone bucket (#6307)
new b7f49f84aa [spark] System function max_pt should be used as
'sys.max_pt' (#6312)
new 909a165964 [python] Support reading data by splitting according to
rows (#6274)
new 0a5004c78d [python] Support ignore_if_exists param for database and
table (#6314)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/content/append-table/row-tracking.md | 8 +-
docs/content/concepts/rest/pvfs.md | 20 +-
docs/content/iceberg/overview.md | 44 +-
.../content/primary-key-table/data-distribution.md | 44 +-
.../primary-key-table/merge-engine/aggregation.md | 4 +
docs/content/spark/sql-functions.md | 9 +-
docs/static/rest-catalog-open-api.yaml | 5 +
.../main/java/org/apache/paimon/CoreOptions.java | 9 +
.../src/main/java/org/apache/paimon/Snapshot.java | 7 +-
.../main/java/org/apache/paimon/rest/RESTApi.java | 15 +-
.../org/apache/paimon/table/SpecialFields.java | 20 +-
.../apache/paimon/casting/FallbackMappingRow.java | 6 +-
.../paimon/data/columnar/ColumnarRowIterator.java | 2 +-
.../java/org/apache/paimon/table/BucketMode.java | 19 +-
.../java/org/apache/paimon/utils/TypeUtils.java | 7 +-
.../org/apache/paimon/utils/TypeUtilsTest.java | 4 +-
.../org/apache/paimon/AppendOnlyFileStore.java | 2 +-
.../java/org/apache/paimon/KeyValueFileStore.java | 4 +-
.../apache/paimon/append/AppendCompactTask.java | 32 +-
.../org/apache/paimon/catalog/AbstractCatalog.java | 25 +-
.../java/org/apache/paimon/catalog/Catalog.java | 12 +-
.../org/apache/paimon/catalog/CatalogUtils.java | 9 +
.../org/apache/paimon/catalog/DelegateCatalog.java | 23 +-
.../DeletionVectorIndexFileWriter.java | 1 -
.../paimon/iceberg/metadata/IcebergDataField.java | 12 +-
.../org/apache/paimon/io/CompactIncrement.java | 42 +-
.../org/apache/paimon/io/DataFileRecordReader.java | 30 +-
.../java/org/apache/paimon/io/DataIncrement.java | 48 ++-
.../java/org/apache/paimon/io/IndexIncrement.java | 88 ----
.../org/apache/paimon/io/PojoDataFileMeta.java | 2 +-
.../compact/aggregate/FieldNestedUpdateAgg.java | 31 +-
.../factory/FieldNestedUpdateAggFactory.java | 9 +-
.../paimon/operation/AbstractFileStoreWrite.java | 20 +-
.../paimon/operation/DataEvolutionSplitRead.java | 8 +-
.../apache/paimon/operation/FileStoreCommit.java | 2 +-
.../paimon/operation/FileStoreCommitImpl.java | 192 +++++----
.../apache/paimon/operation/FileStoreWrite.java | 10 -
.../paimon/operation/MergeFileSplitRead.java | 8 +-
.../apache/paimon/operation/RawFileSplitRead.java | 4 +-
.../java/org/apache/paimon/rest/RESTCatalog.java | 17 +-
.../org/apache/paimon/schema/SchemaManager.java | 33 +-
.../org/apache/paimon/schema/SchemaValidation.java | 12 +-
.../paimon/table/AbstractFileStoreTable.java | 2 +-
.../paimon/table/sink/CommitMessageImpl.java | 40 +-
.../sink/CommitMessageLegacyV2Serializer.java | 21 +-
.../paimon/table/sink/CommitMessageSerializer.java | 68 ++-
.../apache/paimon/table/sink/TableCommitImpl.java | 13 +-
.../org/apache/paimon/table/sink/TableWrite.java | 7 +-
.../apache/paimon/table/sink/TableWriteImpl.java | 6 -
...{RowLineageTable.java => RowTrackingTable.java} | 16 +-
.../paimon/table/system/SystemTableLoader.java | 4 +-
.../org/apache/paimon/TestAppendFileStore.java | 25 +-
.../test/java/org/apache/paimon/TestFileStore.java | 11 +-
.../paimon/append/AppendCompactTaskTest.java | 212 +++++++++
.../org/apache/paimon/catalog/CatalogTestBase.java | 31 +-
.../deletionvectors/BucketedDvMaintainerTest.java | 37 +-
.../append/AppendDeletionFileMaintainerTest.java | 8 +-
.../iceberg/metadata/IcebergDataFieldTest.java | 70 ++-
.../index/DynamicBucketIndexMaintainerTest.java | 2 +-
.../paimon/index/HashBucketAssignerTest.java | 10 +-
...festCommittableSerializerCompatibilityTest.java | 175 +++++++-
.../compact/aggregate/FieldAggregatorTest.java | 41 +-
.../apache/paimon/operation/FileDeletionTest.java | 4 +-
.../paimon/operation/FileStoreCommitTest.java | 4 +-
.../apache/paimon/operation/TestCommitThread.java | 2 +-
.../apache/paimon/rest/MockRESTCatalogTest.java | 8 +-
.../org/apache/paimon/rest/RESTCatalogServer.java | 37 ++
.../org/apache/paimon/rest/RESTCatalogTest.java | 324 ++++++++++++--
.../org/apache/paimon/schema/TableSchemaTest.java | 7 -
.../paimon/table/DynamicBucketTableTest.java | 2 +-
.../apache/paimon/table/SchemaEvolutionTest.java | 16 +
.../table/sink/CommitMessageSerializerTest.java | 26 +-
.../apache/paimon/utils/CompatibilityUtils.java} | 23 +-
.../compatibility/manifest-committable-v10 | Bin 0 -> 3978 bytes
.../sink/cdc/CdcRecordStoreMultiWriteOperator.java | 24 +-
.../flink/sink/cdc/FlinkCdcMultiTableSink.java | 7 +-
.../sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java | 2 +-
.../cdc/CdcRecordStoreMultiWriteOperatorTest.java | 5 +-
.../sink/cdc/CdcRecordStoreWriteOperatorTest.java | 4 +-
.../AppendPreCommitCompactCoordinatorOperator.java | 7 +-
.../AppendPreCommitCompactWorkerOperator.java | 2 +-
.../paimon/flink/compact/AppendTableCompactor.java | 2 +-
.../ChangelogCompactCoordinateOperator.java | 9 +-
.../changelog/ChangelogCompactSortOperator.java | 14 +-
.../PostponeBucketCommittableRewriter.java | 13 +-
.../apache/paimon/flink/sink/FlinkSinkBuilder.java | 2 +-
.../flink/sink/GlobalFullCompactionSinkWrite.java | 6 +-
.../apache/paimon/flink/sink/LookupSinkWrite.java | 8 +-
.../sink/MultiTablesStoreCompactOperator.java | 2 +-
.../paimon/flink/sink/PrepareCommitOperator.java | 13 +-
.../paimon/flink/sink/StoreCompactOperator.java | 2 +-
.../apache/paimon/flink/sink/StoreSinkWrite.java | 32 +-
.../paimon/flink/sink/StoreSinkWriteImpl.java | 77 +---
.../paimon/flink/sink/TableWriteOperator.java | 2 +-
.../sink/listener/PartitionMarkDoneListener.java | 8 +-
.../org/apache/paimon/flink/AppendTableITCase.java | 56 +--
.../org/apache/paimon/flink/CatalogITCaseBase.java | 12 +-
...eITCase.java => CrossPartitionTableITCase.java} | 35 +-
.../FilterPushdownWithSchemaChangeITCase.java | 5 +-
.../paimon/flink/FlinkJobRecoveryITCase.java | 2 +-
...endPreCommitCompactCoordinatorOperatorTest.java | 3 +-
.../paimon/flink/sink/CompactorSinkITCase.java | 106 -----
.../flink/sink/StoreCompactOperatorTest.java | 2 +-
.../paimon/flink/sink/WriterOperatorTest.java | 8 +-
.../flink/sink/listener/ListenerTestUtils.java | 7 +-
paimon-lance/pom.xml | 87 +---
paimon-lance/src/main/resources/META-INF/NOTICE | 10 -
.../pypaimon/catalog/rest/rest_catalog.py | 35 +-
.../pypaimon/manifest/manifest_file_manager.py | 4 +-
paimon-python/pypaimon/read/plan.py | 6 +-
.../pypaimon/read/reader/concat_batch_reader.py | 27 ++
paimon-python/pypaimon/read/split.py | 2 +
paimon-python/pypaimon/read/split_read.py | 7 +-
paimon-python/pypaimon/read/table_scan.py | 181 ++++++--
paimon-python/pypaimon/tests/predicates_test.py | 66 +--
paimon-python/pypaimon/tests/pvfs_test.py | 14 -
.../pypaimon/tests/py36/ao_simple_test.py | 387 +++++++++++++++++
paimon-python/pypaimon/tests/rest/rest_server.py | 21 +-
.../pypaimon/tests/rest/rest_simple_test.py | 474 +++++++++++++++++++--
paimon-python/pypaimon/tests/serializable_test.py | 2 +-
paimon-python/pypaimon/write/file_store_commit.py | 2 +-
.../paimon/spark/sql/MergeIntoTableTest.scala | 2 +
...{RowLineageTest.scala => RowTrackingTest.scala} | 2 +-
.../paimon/spark/sql/MergeIntoTableTest.scala | 2 +
.../apache/paimon/spark/sql/RowLineageTest.scala | 21 -
.../apache/paimon/spark/sql/RowTrackingTest.scala} | 2 +-
.../paimon/spark/sql/MergeIntoTableTest.scala | 2 +
.../apache/paimon/spark/sql/RowLineageTest.scala | 21 -
.../apache/paimon/spark/sql/RowTrackingTest.scala} | 2 +-
.../paimon/spark/sql/MergeIntoTableTest.scala | 2 +
.../apache/paimon/spark/sql/RowLineageTest.scala | 21 -
.../apache/paimon/spark/sql/RowTrackingTest.scala} | 2 +-
.../paimon/spark/sql/MergeIntoTableTest.scala | 2 +
.../apache/paimon/spark/sql/RowLineageTest.scala | 21 -
.../apache/paimon/spark/sql/RowTrackingTest.scala} | 2 +-
.../java/org/apache/paimon/spark/SparkCatalog.java | 42 +-
.../paimon/spark/catalog/SparkBaseCatalog.java | 10 +-
.../paimon/spark/procedure/CompactProcedure.java | 2 +-
.../paimon/spark/ColumnPruningAndPushDown.scala | 2 +-
.../org/apache/paimon/spark/SparkTableWrite.scala | 4 +-
.../spark/catalyst/analysis/PaimonAnalysis.scala | 3 +-
.../catalyst/analysis/PaimonFunctionResolver.scala | 2 +-
.../commands/DeleteFromPaimonTableCommand.scala | 4 +-
.../spark/commands/MergeIntoPaimonTable.scala | 32 +-
.../spark/commands/PaimonRowLevelCommand.scala | 6 +-
.../paimon/spark/commands/PaimonSparkWriter.scala | 28 +-
.../spark/commands/UpdatePaimonTableCommand.scala | 2 +-
.../paimon/spark/schema/PaimonMetadataColumn.scala | 2 +-
.../apache/paimon/spark/util/ScanPlanHelper.scala | 6 +-
.../paimon/spark/SparkCatalogWithRestTest.java | 3 +
.../catalog/functions/BucketFunctionTest.java | 4 +-
.../spark/benchmark/BucketFunctionBenchmark.scala | 44 --
.../paimon/spark/sql/DeleteFromTableTestBase.scala | 38 +-
.../paimon/spark/sql/MergeIntoTableTestBase.scala | 91 +++-
.../paimon/spark/sql/PaimonFunctionTest.scala | 6 +-
.../spark/sql/PaimonV1FunctionTestBase.scala | 19 +-
...ageTestBase.scala => RowTrackingTestBase.scala} | 22 +-
.../paimon/spark/sql/UpdateTableTestBase.scala | 38 +-
.../paimon/spark/util/ScanPlanHelperTest.scala | 14 +-
159 files changed, 3107 insertions(+), 1374 deletions(-)
delete mode 100644
paimon-core/src/main/java/org/apache/paimon/io/IndexIncrement.java
rename
paimon-core/src/main/java/org/apache/paimon/table/system/{RowLineageTable.java
=> RowTrackingTable.java} (90%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/append/AppendCompactTaskTest.java
copy paimon-core/src/{main/java/org/apache/paimon/utils/SnapshotLoader.java =>
test/java/org/apache/paimon/utils/CompatibilityUtils.java} (61%)
create mode 100644
paimon-core/src/test/resources/compatibility/manifest-committable-v10
rename
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/{GlobalDynamicBucketTableITCase.java
=> CrossPartitionTableITCase.java} (84%)
delete mode 100644 paimon-lance/src/main/resources/META-INF/NOTICE
create mode 100644 paimon-python/pypaimon/tests/py36/ao_simple_test.py
rename
paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/{RowLineageTest.scala
=> RowTrackingTest.scala} (94%)
delete mode 100644
paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/RowLineageTest.scala
copy
paimon-spark/{paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/DDLTest.scala
=>
paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTest.scala}
(94%)
delete mode 100644
paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/RowLineageTest.scala
copy
paimon-spark/{paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/DDLTest.scala
=>
paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTest.scala}
(94%)
delete mode 100644
paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/RowLineageTest.scala
copy
paimon-spark/{paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/DDLTest.scala
=>
paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTest.scala}
(94%)
delete mode 100644
paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/RowLineageTest.scala
copy
paimon-spark/{paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/DDLTest.scala
=>
paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTest.scala}
(94%)
delete mode 100644
paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/benchmark/BucketFunctionBenchmark.scala
rename
paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/{RowLineageTestBase.scala
=> RowTrackingTestBase.scala} (95%)
