This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from a5f3dd4b99 [rest] Add semantic view definition management to REST
Catalog (#9848)
new f3862f6b4e [python] Cover native dynamic PK events and compacted
first-row scans (#9854)
new c1f92f1a41 [python] Speed up nested VARIANT reads by vectorizing
offsets (#9847)
new 8a429cfc36 [python] Expand native planning across resolved tables and
scan modes (#9873)
new 72e4a32b56 [core] Execute complete filters with unprojected fields
(#9872)
new 03d68bfec7 [python] Speed up LeRobot video window reads (#9852)
new 478a7201c4 [python] Support writing shared-shredding MAPs in Parquet
(#9837)
new b6d91b6fd8 [core] Add standalone manifest sidecar format and utilities
(#9845)
new 301713fe6c [core] Manage manifest extra file lifecycle (#9886)
new 32f0a9261e [python] Use conditional OSS puts for atomic metadata
writes (#9715)
new 9686b2a1c2 [core] Generate sidecars for newly written manifests (#9889)
new 3f0d716f97 [python] Reduce memory usage for Parquet row-id updates
(#9840)
new ce81e89b9d [spark][core] Support more expressions in V2 filter
pushdown (#9891)
new 084cde4772 [core] Prune manifest blocks with sidecars during scans
(#9897)
new a5fb087bde [spark][core] Support more scalar transforms in V2 filter
pushdown (#9905)
new 4151992545 [python] Resolve value stats columns using file schema
(#9906)
new d6644bbc97 [python] Make BLOB I/O merge parameters configurable (#9890)
new f951059590 [format] Support file.block-size for Avro (#9901)
new 394cbaaa11 Revert "[format] Support file.block-size for Avro (#9901)"
new 2e8803c498 [python] Honor snapshot.ignore-empty-commit for empty
appends (#9878)
new 1fc0f17ad2 [python] Extend ContiguousWindowDataset for frame-level
training (#9875)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci-python.yml | 1 +
docs/docs/concepts/spec/manifest.md | 247 +++++
docs/docs/pypaimon/multimodal-reading.md | 119 ++-
docs/docs/pypaimon/pytorch.md | 55 +-
docs/generated/catalog_configuration.html | 6 +
docs/generated/core_configuration.html | 6 +
.../main/java/org/apache/paimon/CoreOptions.java | 11 +
.../org/apache/paimon/options/CatalogOptions.java | 10 +
.../benchmark/ManifestFileSorterBenchmark.java | 4 +-
.../apache/paimon/fileindex/FileIndexReader.java | 20 +
.../paimon/globalindex/GlobalIndexReader.java | 24 +
...FieldTransform.java => BitLengthTransform.java} | 57 +-
.../java/org/apache/paimon/predicate/Contains.java | 2 +-
.../{FieldTransform.java => DateAddTransform.java} | 58 +-
...{FieldTransform.java => DateDiffTransform.java} | 58 +-
.../paimon/predicate/DateExtractTransform.java | 147 +++
.../paimon/predicate/DateTruncTransform.java | 135 +++
...oncatTransform.java => DayOfWeekTransform.java} | 32 +-
...oncatTransform.java => DayOfYearTransform.java} | 32 +-
.../{ConcatTransform.java => DayTransform.java} | 31 +-
.../java/org/apache/paimon/predicate/EndsWith.java | 2 +-
.../apache/paimon/predicate/FunctionVisitor.java | 16 +
.../{ConcatTransform.java => HourTransform.java} | 31 +-
...atTransform.java => IsoDayOfWeekTransform.java} | 32 +-
.../org/apache/paimon/predicate/LeafFunction.java | 4 +
.../apache/paimon/predicate/LengthTransform.java | 129 +++
.../java/org/apache/paimon/predicate/Like.java | 2 +-
.../{ConcatTransform.java => MinuteTransform.java} | 31 +-
.../{ConcatTransform.java => MonthTransform.java} | 31 +-
.../predicate/{Contains.java => NotContains.java} | 20 +-
.../predicate/{Contains.java => NotEndsWith.java} | 20 +-
.../predicate/{LessThan.java => NotLike.java} | 22 +-
.../{Contains.java => NotStartsWith.java} | 20 +-
.../predicate/OnlyPartitionKeyEqualVisitor.java | 20 +
.../apache/paimon/predicate/OverlayTransform.java | 124 +++
.../org/apache/paimon/predicate/PadTransform.java | 145 +++
.../apache/paimon/predicate/PredicateBuilder.java | 8 +
.../apache/paimon/predicate/PredicateRemapper.java | 87 ++
...{ConcatTransform.java => QuarterTransform.java} | 31 +-
.../{ConcatTransform.java => SecondTransform.java} | 31 +-
.../org/apache/paimon/predicate/StartsWith.java | 2 +-
.../org/apache/paimon/predicate/Transform.java | 21 +
.../paimon/predicate/TransformInputUtils.java | 77 ++
.../paimon/predicate/TranslateTransform.java | 95 ++
.../{ConcatTransform.java => WeekTransform.java} | 32 +-
...{ConcatTransform.java => WeekdayTransform.java} | 31 +-
...ncatTransform.java => YearOfWeekTransform.java} | 33 +-
.../{ConcatTransform.java => YearTransform.java} | 31 +-
.../org/apache/paimon/utils/DeltaVarintCodec.java | 126 +++
.../java/org/apache/paimon/utils/TypeUtils.java | 19 +
.../org/apache/paimon/utils/VarLengthIntUtils.java | 29 +
.../paimon/predicate/DateExtractTransformTest.java | 150 +++
.../paimon/predicate/LengthTransformTest.java | 82 ++
.../paimon/predicate/PredicateJsonSerdeTest.java | 31 +-
.../paimon/predicate/PredicateRemapperTest.java | 144 +++
.../org/apache/paimon/predicate/PredicateTest.java | 60 ++
.../paimon/predicate/ScalarTransformTest.java | 171 +++
.../paimon/predicate/TransformJsonSerdeTest.java | 118 +-
.../apache/paimon/utils/DeltaVarintCodecTest.java | 225 ++++
.../apache/paimon/utils/VarLengthIntUtilsTest.java | 101 ++
.../java/org/apache/paimon/AbstractFileStore.java | 10 +-
.../src/main/java/org/apache/paimon/FileStore.java | 2 +
.../org/apache/paimon/catalog/CachingCatalog.java | 21 +
.../paimon/catalog/TableQueryAuthResult.java | 133 +--
.../org/apache/paimon/manifest/BucketFilter.java | 16 +
.../apache/paimon/manifest/ManifestAvroWriter.java | 52 +-
.../org/apache/paimon/manifest/ManifestFile.java | 113 +-
.../apache/paimon/manifest/ManifestSidecar.java | 942 ++++++++++++++++
.../paimon/operation/AbstractFileStoreScan.java | 17 +-
.../apache/paimon/operation/ChangelogDeletion.java | 13 +-
.../operation/DataEvolutionFileStoreScan.java | 17 +-
.../apache/paimon/operation/FileDeletionBase.java | 20 +-
.../paimon/operation/ManifestFileMerger.java | 2 +-
.../paimon/operation/commit/CommitCleaner.java | 6 +-
.../paimon/table/AbstractFileStoreTable.java | 16 +
.../paimon/table/DelegatedFileStoreTable.java | 11 +
.../paimon/table/FallbackReadFileStoreTable.java | 6 +
.../org/apache/paimon/table/FileStoreTable.java | 5 +
.../paimon/table/format/FormatReadBuilder.java | 18 +-
.../paimon/table/format/FormatTableRead.java | 40 +-
.../paimon/table/source/AbstractDataTableRead.java | 199 +---
.../paimon/table/source/AbstractDataTableScan.java | 56 +-
.../apache/paimon/table/source/ReadTransform.java | 178 ++++
.../org/apache/paimon/table/source/TableRead.java | 12 +
.../java/org/apache/paimon/utils/ObjectsCache.java | 6 +
.../java/org/apache/paimon/CoreOptionsTest.java | 18 +
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 61 ++
.../apache/paimon/catalog/CachingCatalogTest.java | 142 +++
.../paimon/manifest/ManifestBlockIndexTest.java | 891 ++++++++++++++++
.../manifest/ManifestFileMetaSerializerTest.java | 3 +-
.../paimon/manifest/ManifestFileMetaTest.java | 8 +-
.../paimon/manifest/ManifestFileMetaTestBase.java | 4 +-
.../apache/paimon/manifest/ManifestFileTest.java | 688 +++++++++++-
.../paimon/manifest/ManifestIndexTestUtils.java | 100 ++
.../paimon/manifest/ManifestSidecarTest.java | 1123 ++++++++++++++++++++
.../paimon/manifest/ManifestSidecarWriteTest.java | 348 ++++++
.../paimon/operation/ExpireSnapshotsTest.java | 72 ++
.../operation/LocalOrphanFilesCleanTest.java | 35 +
.../operation/ManifestRewriteCleanupTest.java | 4 +-
.../operation/ManifestSidecarLifecycleTest.java | 106 ++
.../org/apache/paimon/rest/RESTCatalogTest.java | 87 ++
.../paimon/table/AppendOnlySimpleTableTest.java | 10 +-
.../paimon/table/DataEvolutionFileIndexTest.java | 44 +-
.../paimon/table/PrimaryKeySimpleTableTest.java | 26 +
.../paimon/table/format/FormatTableReadTest.java | 109 ++
.../table/source/AbstractDataTableReadTest.java | 369 ++++++-
.../resources/compatibility/manifest-sidecar-v1 | Bin 0 -> 2107 bytes
.../orc/filter/OrcPredicateFunctionVisitor.java | 22 +
.../format/orc/filter/OrcFilterConverterTest.java | 15 +
.../paimon/format/parquet/ParquetFiltersTest.java | 15 +
paimon-python/README.md | 128 ++-
paimon-python/dev/requirements-dev.txt | 1 +
paimon-python/dev/run_mixed_tests.sh | 15 +
paimon-python/pypaimon/benchmark/act/paimon.py | 11 +-
.../pypaimon/catalog/rest/rest_token_file_io.py | 47 +-
paimon-python/pypaimon/common/file_io.py | 30 +-
paimon-python/pypaimon/common/json_util.py | 2 +
paimon-python/pypaimon/common/options/config.py | 32 +
.../pypaimon/common/options/core_options.py | 57 +
.../pypaimon/data/map_shared_shredding.py | 51 +-
.../apply_deletion_vector_reader.py | 13 +-
paimon-python/pypaimon/filesystem/oss_file_io.py | 157 +++
.../pypaimon/filesystem/resolving_file_io.py | 3 +
.../pypaimon/manifest/manifest_file_manager.py | 3 +-
.../pypaimon/multimodal/lerobot/dataset.py | 144 ++-
paimon-python/pypaimon/multimodal/query.py | 51 +-
.../pypaimon/multimodal/window_dataset.py | 292 +++--
.../pypaimon/multimodal/window_transforms.py | 88 ++
paimon-python/pypaimon/read/native_plan.py | 84 +-
.../pypaimon/read/reader/format_pyarrow_reader.py | 19 +-
.../read/scanner/chunk_shuffle_split_generator.py | 13 +-
paimon-python/pypaimon/read/split_read.py | 18 +-
paimon-python/pypaimon/read/table_read.py | 6 +
paimon-python/pypaimon/read/table_scan.py | 150 ++-
.../rest_catalog_blob_as_descriptor_sample.py | 23 +-
paimon-python/pypaimon/table/file_store_table.py | 5 +-
paimon-python/pypaimon/tests/act_runner_test.py | 20 +-
paimon-python/pypaimon/tests/blob_test.py | 132 ++-
.../tests/contiguous_window_dataset_test.py | 236 ++++
.../pypaimon/tests/e2e/java_py_read_write_test.py | 46 +
paimon-python/pypaimon/tests/file_io_test.py | 7 +-
.../tests/format_pyarrow_variant_row_group_test.py | 32 +-
paimon-python/pypaimon/tests/lance_utils_test.py | 6 +-
.../tests/map_shared_shredding_write_test.py | 429 ++++++++
.../pypaimon/tests/multimodal_lerobot_test.py | 203 ++++
.../tests/native_plan_capabilities_test.py | 36 +
.../tests/native_plan_chunk_shuffle_test.py | 172 +++
.../tests/native_plan_dynamic_bucket_test.py | 33 +
.../pypaimon/tests/native_plan_integration_test.py | 13 +-
.../tests/native_plan_materialized_pk_test.py | 186 ++++
.../tests/native_plan_resolved_schema_test.py | 184 ++++
.../pypaimon/tests/native_plan_rest_test.py | 151 +++
paimon-python/pypaimon/tests/native_plan_test.py | 57 +-
.../pypaimon/tests/oss_atomic_write_test.py | 308 ++++++
paimon-python/pypaimon/tests/oss_file_io_test.py | 12 +-
.../pypaimon/tests/oss_legacy_mode_test.py | 19 +-
.../pypaimon/tests/py36/ao_simple_test.py | 8 +-
paimon-python/pypaimon/tests/reader_base_test.py | 23 +
.../pypaimon/tests/reader_predicate_test.py | 78 ++
.../pypaimon/tests/resolving_file_io_test.py | 21 +
.../pypaimon/tests/rest/rest_token_file_io_test.py | 3 +
.../pypaimon/tests/schema_evolution_read_test.py | 53 +-
.../pypaimon/tests/snapshot_manager_test.py | 24 +
paimon-python/pypaimon/tests/table_commit_test.py | 47 +
.../tests/table_update_by_row_id_chunked_test.py | 112 +-
.../pypaimon/tests/table_upsert_by_key_test.py | 250 +++++
.../pypaimon/tests/window_transforms_test.py | 111 ++
paimon-python/pypaimon/write/file_store_commit.py | 16 +-
.../pypaimon/write/map_shared_shredding_writer.py | 573 ++++++++++
paimon-python/pypaimon/write/table_commit.py | 3 +-
paimon-python/pypaimon/write/table_update.py | 1 +
.../pypaimon/write/table_update_by_row_id.py | 227 +++-
.../pypaimon/write/writer/data_vector_writer.py | 8 +-
paimon-python/pypaimon/write/writer/data_writer.py | 75 +-
.../write/writer/dedicated_format_writer.py | 8 +-
.../pypaimon/write/writer/single_file_writer.py | 106 ++
paimon-python/setup.py | 4 +
.../paimon/spark/SparkV2FilterConverter.scala | 9 +
.../spark/util/SparkExpressionConverter.scala | 121 ++-
.../paimon/spark/SparkFilterConverterTest.java | 17 +-
.../spark/procedure/PermissionProcedureTest.scala | 4 +-
.../spark/sql/SparkV2FilterConverterTestBase.scala | 284 ++++-
182 files changed, 14369 insertions(+), 1192 deletions(-)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{FieldTransform.java =>
BitLengthTransform.java} (55%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{FieldTransform.java =>
DateAddTransform.java} (53%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{FieldTransform.java =>
DateDiffTransform.java} (53%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/DateTruncTransform.java
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> DayOfWeekTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> DayOfYearTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> DayTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> HourTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> IsoDayOfWeekTransform.java} (61%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/LengthTransform.java
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> MinuteTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> MonthTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{Contains.java =>
NotContains.java} (75%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{Contains.java =>
NotEndsWith.java} (75%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{LessThan.java =>
NotLike.java} (70%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{Contains.java =>
NotStartsWith.java} (74%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/OverlayTransform.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/PadTransform.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/PredicateRemapper.java
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> QuarterTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> SecondTransform.java} (61%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/TransformInputUtils.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/predicate/TranslateTransform.java
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> WeekTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> WeekdayTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> YearOfWeekTransform.java} (61%)
copy
paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java
=> YearTransform.java} (61%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/utils/DeltaVarintCodec.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/predicate/LengthTransformTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/predicate/PredicateRemapperTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/predicate/ScalarTransformTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/utils/DeltaVarintCodecTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/utils/VarLengthIntUtilsTest.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/ReadTransform.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/operation/ManifestSidecarLifecycleTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableReadTest.java
create mode 100644
paimon-core/src/test/resources/compatibility/manifest-sidecar-v1
create mode 100644 paimon-python/pypaimon/filesystem/oss_file_io.py
create mode 100644 paimon-python/pypaimon/multimodal/window_transforms.py
create mode 100644
paimon-python/pypaimon/tests/map_shared_shredding_write_test.py
create mode 100644
paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py
create mode 100644
paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py
create mode 100644
paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py
create mode 100644 paimon-python/pypaimon/tests/native_plan_rest_test.py
create mode 100644 paimon-python/pypaimon/tests/oss_atomic_write_test.py
create mode 100644 paimon-python/pypaimon/tests/window_transforms_test.py
create mode 100644 paimon-python/pypaimon/write/map_shared_shredding_writer.py
create mode 100644 paimon-python/pypaimon/write/writer/single_file_writer.py
