This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 08f3fa8bdf [core] Add interrupt coverage for snapshot existence
retries (#9745)
new bbb6a0bb32 [python] Support linear interpolation in temporal alignment
(#9747)
new 7aff485263 [core] Improve the error when listing a Format Table custom
partition location fails (#9748)
new af1f81043b [python] Close vector readers when search raises
synchronously (#9755)
new 0c89f40a84 [test] Switch minio test image to quay.io (#9761)
new 30f051abec [python] Batch vector writes when building global indexes
(#9752)
new 7edc7072a5 [python] Share raw vector scans across batch queries (#9753)
new 82950d63ea [python] Read native vector index ranges with bounded
concurrency (#9756)
new 4c16937235 [python] Use persisted vector metrics for raw search and
refinement (#9754)
new 148fef0d57 [python] Read LeRobot video datasets for training (#9757)
new f208702603 [python] Support window joins in temporal alignment (#9760)
new 228a1dbc17 [python] Support reading shared-shredding maps (#9763)
new 8993507fa7 [ci] Run affected test suites through reusable workflows
(#9787)
new b775139941 [core] Support bucket pruning for manifest files (#9791)
new 0cb4b98259 [core] Simplify ManifestFileMeta constructors and field
order
new dc3d49b09b [python] Vectorize raw vector scoring and refinement in
bounded blocks (#9759)
new 9028c5278b [python] Stream batches when building generic global
indexes (#9751)
new 86b5641e17 [lumina] Release the vector file when FileBackedDataset
fails to construct (#9481)
new baa455e71a [core] Support bucket-first sorting for manifest files
(#9792)
new 81e9d87a37 [core] Fix bucket-first manifest sorting for postpone tables
new 2a5cbe906c [common] Fix JSON round-trip of temporal and decimal
predicate literals (#9640)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/CI.md | 83 ++
.github/workflows/{docs-tests.yml => ci-docs.yml} | 15 +-
.github/workflows/ci-java.yml | 73 ++
.../{check-licensing.yml => ci-licensing.yml} | 31 +-
.../{paimon-python-checks.yml => ci-python.yml} | 43 +-
.github/workflows/ci.yml | 112 +++
.github/workflows/e2e-tests-flink-1.x.yml | 80 --
.github/workflows/e2e-tests-flink-2.x-jdk11.yml | 81 --
.github/workflows/file-size-check.yml | 48 --
.../{stale-pr.yml => maintenance-stale-pr.yml} | 2 +-
...pshot-jdk17.yml => publish-snapshot-spark4.yml} | 14 +-
.../{publish_snapshot.yml => publish-snapshot.yml} | 14 +-
.github/workflows/release-java.yml | 2 +-
.github/workflows/release-python-publish.yml | 2 +-
.github/workflows/release-python.yml | 2 +-
.github/workflows/utcase-eslib.yml | 73 --
.github/workflows/utcase-full-text.yml | 61 --
.github/workflows/utitcase-flink-1.x-common.yml | 75 --
.github/workflows/utitcase-flink-1.x-others.yml | 75 --
.github/workflows/utitcase-flink-2.x-jdk11.yml | 76 --
.github/workflows/utitcase-iceberg-ga.yml | 66 --
.github/workflows/utitcase-jdk11.yml | 72 --
.github/workflows/utitcase-spark-3.x.yml | 79 --
.github/workflows/utitcase-spark-4.x.yml | 76 --
.github/workflows/utitcase.yml | 78 --
docs/docs/project/verifying-a-release-candidate.md | 2 +-
docs/docs/pypaimon/lerobot.md | 19 +-
docs/docs/pypaimon/multimodal-reading.md | 36 +
.../requests/ListPartitionsByFilterRequest.java | 10 +-
.../org/apache/paimon/predicate/LeafPredicate.java | 55 +-
.../paimon/predicate/PredicateJsonSerdeTest.java | 44 +
.../org/apache/paimon/manifest/BucketFilter.java | 25 +
.../apache/paimon/manifest/ManifestAvroWriter.java | 33 +-
.../paimon/manifest/ManifestBucketFilter.java | 11 +-
.../apache/paimon/manifest/ManifestFileMeta.java | 44 +-
.../manifest/ManifestFileMetaSerializer.java | 4 +-
.../paimon/manifest/ProjectedManifestEntry.java | 1 +
.../paimon/operation/AbstractFileStoreScan.java | 2 +
.../apache/paimon/operation/BucketSelector.java | 32 +-
.../paimon/operation/ManifestCompactDryRun.java | 4 +-
.../paimon/operation/ManifestEntryRunMerge.java | 25 +-
.../operation/ManifestEntryRunMergePlan.java | 2 +
.../paimon/operation/ManifestFileBlockMerger.java | 21 +
.../paimon/operation/ManifestFileSorter.java | 139 ++-
.../apache/paimon/operation/ManifestsReader.java | 40 +-
.../table/format/CatalogSplitEnumerator.java | 44 +-
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 53 ++
.../DataEvolutionCompactCoordinatorTest.java | 8 +-
.../DataEvolutionCompactRangePlannerTest.java | 2 +
.../DataEvolutionRowIdReassignerTest.java | 4 +-
.../LegacyManifestFileMetaSerializerPaimon10.java | 2 +
.../manifest/ManifestFileMetaSerializerTest.java | 1 +
.../paimon/manifest/ManifestFileMetaTest.java | 160 +++-
.../apache/paimon/manifest/ManifestFileTest.java | 38 +-
.../apache/paimon/manifest/ManifestListTest.java | 17 +-
.../paimon/manifest/ManifestTestDataGenerator.java | 16 +
.../paimon/operation/BucketSelectorTest.java | 51 ++
.../paimon/operation/ExpireSnapshotsTest.java | 3 +
.../operation/LocalOrphanFilesCleanTest.java | 1 +
.../operation/ManifestEntryRunMergeTest.java | 81 +-
.../paimon/operation/ManifestFileMergerTest.java | 4 +-
.../operation/ManifestRewriteCleanupTest.java | 2 +-
.../org/apache/paimon/rest/RESTApiJsonTest.java | 52 ++
.../format/CatalogManagedPartitionScanTest.java | 48 ++
.../index/LuminaVectorGlobalIndexWriter.java | 28 +-
.../index/LuminaFileBackedDatasetCloseTest.java | 69 ++
paimon-python/README.md | 29 +
paimon-python/dev/run_mixed_tests.sh | 38 +-
.../pypaimon/data/map_shared_shredding.py | 347 ++++++++
.../pypaimon/globalindex/create_global_index.py | 67 +-
.../vindex/vindex_vector_global_index_reader.py | 110 ++-
.../vindex/vindex_vector_index_writer.py | 66 ++
paimon-python/pypaimon/multimodal/__init__.py | 8 +-
.../pypaimon/multimodal/lerobot/dataset.py | 293 ++++++-
paimon-python/pypaimon/multimodal/temporal.py | 608 ++++++++++++-
.../pypaimon/read/reader/format_pyarrow_reader.py | 111 ++-
.../table/source/primary_key_vector_read.py | 27 +-
.../pypaimon/table/source/vector_search_read.py | 338 ++++++--
.../pypaimon/tests/batch_vector_raw_scan_test.py | 289 +++++++
.../pypaimon/tests/e2e/java_py_read_write_test.py | 28 +
.../format_pyarrow_shared_shredding_map_test.py | 402 +++++++++
.../pypaimon/tests/global_index_build_test.py | 189 ++++-
.../pypaimon/tests/multimodal_lerobot_test.py | 281 ++++++
.../pypaimon/tests/multimodal_temporal_test.py | 941 ++++++++++++++++++++-
.../tests/primary_key_global_index_golden_test.py | 93 ++
.../tests/vector_metric_consistency_test.py | 210 +++++
.../pypaimon/tests/vector_scoring_test.py | 121 +++
.../pypaimon/tests/vector_search_filter_test.py | 75 +-
.../pypaimon/tests/vindex_batch_write_test.py | 198 +++++
paimon-python/pypaimon/tests/vindex_input_test.py | 233 +++++
.../testutils/junit/DockerImageVersions.java | 2 +-
tools/ci/plan.py | 180 ++++
tools/ci/run-java-tests.sh | 129 +++
tools/ci/test_java_commands.py | 112 +++
tools/ci/test_plan.py | 238 ++++++
tools/ci/test_result.py | 56 ++
{.github/workflows => tools/ci}/utils.sh | 0
97 files changed, 7053 insertions(+), 1332 deletions(-)
create mode 100644 .github/CI.md
rename .github/workflows/{docs-tests.yml => ci-docs.yml} (84%)
create mode 100644 .github/workflows/ci-java.yml
rename .github/workflows/{check-licensing.yml => ci-licensing.yml} (82%)
rename .github/workflows/{paimon-python-checks.yml => ci-python.yml} (95%)
mode change 100755 => 100644
create mode 100644 .github/workflows/ci.yml
delete mode 100644 .github/workflows/e2e-tests-flink-1.x.yml
delete mode 100644 .github/workflows/e2e-tests-flink-2.x-jdk11.yml
delete mode 100644 .github/workflows/file-size-check.yml
rename .github/workflows/{stale-pr.yml => maintenance-stale-pr.yml} (98%)
rename .github/workflows/{publish_snapshot-jdk17.yml =>
publish-snapshot-spark4.yml} (89%)
rename .github/workflows/{publish_snapshot.yml => publish-snapshot.yml} (89%)
delete mode 100644 .github/workflows/utcase-eslib.yml
delete mode 100644 .github/workflows/utcase-full-text.yml
delete mode 100644 .github/workflows/utitcase-flink-1.x-common.yml
delete mode 100644 .github/workflows/utitcase-flink-1.x-others.yml
delete mode 100644 .github/workflows/utitcase-flink-2.x-jdk11.yml
delete mode 100644 .github/workflows/utitcase-iceberg-ga.yml
delete mode 100644 .github/workflows/utitcase-jdk11.yml
delete mode 100644 .github/workflows/utitcase-spark-3.x.yml
delete mode 100644 .github/workflows/utitcase-spark-4.x.yml
delete mode 100644 .github/workflows/utitcase.yml
copy paimon-common/src/main/java/org/apache/paimon/codegen/Projection.java =>
paimon-core/src/main/java/org/apache/paimon/manifest/ManifestBucketFilter.java
(68%)
create mode 100644
paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java
create mode 100644 paimon-python/pypaimon/data/map_shared_shredding.py
create mode 100644 paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py
create mode 100644
paimon-python/pypaimon/tests/format_pyarrow_shared_shredding_map_test.py
create mode 100644
paimon-python/pypaimon/tests/vector_metric_consistency_test.py
create mode 100644 paimon-python/pypaimon/tests/vector_scoring_test.py
create mode 100644 paimon-python/pypaimon/tests/vindex_batch_write_test.py
create mode 100644 paimon-python/pypaimon/tests/vindex_input_test.py
create mode 100644 tools/ci/plan.py
create mode 100644 tools/ci/run-java-tests.sh
create mode 100644 tools/ci/test_java_commands.py
create mode 100644 tools/ci/test_plan.py
create mode 100644 tools/ci/test_result.py
rename {.github/workflows => tools/ci}/utils.sh (100%)
