This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5a8238936d [python] Reject fixed-bucket tables for all global index
types (#10537)
5a8238936d is described below
commit 5a8238936d28caaeb6a60bf7e2a386909d30a039
Author: zhuanshenbsj1 <[email protected]>
AuthorDate: Wed Oct 14 09:56:45 2026 +0800
[python] Reject fixed-bucket tables for all global index types (#10537)
---
.../pypaimon/globalindex/create_global_index.py | 12 +++++--
.../pypaimon/tests/global_index_build_test.py | 41 ++++++++++++++++++++++
2 files changed, 50 insertions(+), 3 deletions(-)
diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py
b/paimon-python/pypaimon/globalindex/create_global_index.py
index 4ff50f8507..98da19c5a9 100644
--- a/paimon-python/pypaimon/globalindex/create_global_index.py
+++ b/paimon-python/pypaimon/globalindex/create_global_index.py
@@ -161,6 +161,7 @@ class GlobalIndexBuilder:
"Column '%s' does not exist in table '%s'."
% (column, self._table.identifier)
)
+ self._validate_unaware_bucket_table()
if self._index_type in _GENERIC_INDEX_IDENTIFIERS:
self._validate_generic_index_table()
@@ -504,14 +505,19 @@ class GlobalIndexBuilder:
)
raise ValueError("Unsupported generic global index type: %s" %
self._index_type)
- def _validate_generic_index_table(self) -> None:
- bucket = self._core_options.bucket()
+ def _validate_unaware_bucket_table(self) -> None:
+ # Global index files live in bucket 0 and are keyed by the table-wide
+ # _ROW_ID, so every index type needs an unaware-bucket table. Check the
+ # table options: caller options only tune the index build.
+ bucket = self._table.options.bucket()
if bucket != -1:
raise ValueError(
- "Generic global index only supports unaware-bucket tables "
+ "Global index only supports unaware-bucket tables "
"(bucket = -1), but table '%s' has bucket = %s."
% (self._table.identifier, bucket)
)
+
+ def _validate_generic_index_table(self) -> None:
if (self._core_options.deletion_vectors_enabled()
and not (self._index_type in VINDEX_IDENTIFIERS
and self._table.options.data_evolution_enabled())):
diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py
b/paimon-python/pypaimon/tests/global_index_build_test.py
index a430b19e9c..cd6c21bb55 100644
--- a/paimon-python/pypaimon/tests/global_index_build_test.py
+++ b/paimon-python/pypaimon/tests/global_index_build_test.py
@@ -34,8 +34,12 @@ from pypaimon.globalindex.build_plan import (
split_by_global_index_shard as _split_by_global_index_shard,
split_one_by_contiguous_row_range as _split_one_by_contiguous_row_range,
)
+from pypaimon import Schema
+from pypaimon.common.json_util import JSON
+from pypaimon.globalindex.bitmap.bitmap_index_writer import BITMAP_IDENTIFIER
from pypaimon.globalindex.create_global_index import GlobalIndexBuilder
from pypaimon.globalindex.btree.btree_file_footer import BTreeFileFooter
+from pypaimon.globalindex.btree.btree_index_writer import BTREE_IDENTIFIER
from pypaimon.globalindex.key_serializer import create_serializer
from pypaimon.globalindex.full_text.native_full_text_global_index_reader
import (
FULL_TEXT_IDENTIFIER,
@@ -1029,6 +1033,43 @@ class GlobalIndexBuildTest(
index_type=FULL_TEXT_IDENTIFIER,
)
+ def _create_legacy_fixed_bucket_table(self, bucket):
+ # Older pypaimon versions skipped row tracking schema validation, so a
+ # row tracking table with a fixed bucket may already exist on disk.
+ # Simulate it by rewriting the committed schema file directly.
+ name = self._unique_name()
+ self.catalog.create_table(name, Schema.from_pyarrow_schema(
+ self.pa_schema, options=self.table_options), False)
+ table = self.catalog.get_table(name)
+ legacy_options = dict(table.table_schema.options)
+ legacy_options['bucket'] = str(bucket)
+ legacy_schema = table.table_schema.copy(new_options=legacy_options)
+ schema_file = table.schema_manager._to_schema_path(legacy_schema.id)
+ table.file_io.overwrite_file_utf8(
+ schema_file, JSON.to_json(legacy_schema, indent=2))
+ table.schema_manager.schema_cache.pop(legacy_schema.id, None)
+
+ legacy_table = self.catalog.get_table(name)
+ self.assertEqual(bucket, legacy_table.options.bucket())
+ self.assertTrue(legacy_table.options.row_tracking_enabled())
+ return legacy_table
+
+ def
test_create_sorted_global_index_rejects_legacy_fixed_bucket_table(self):
+ table = self._create_legacy_fixed_bucket_table(bucket=4)
+ # A caller provided bucket only tunes the index build, so it must not
be
+ # able to pass the validation on behalf of the table.
+ for index_type in (BTREE_IDENTIFIER, BITMAP_IDENTIFIER):
+ for options in (None, {'bucket': '-1'}):
+ with self.subTest(index_type=index_type, options=options):
+ with self.assertRaisesRegex(
+ ValueError,
+ 'Global index only supports unaware-bucket tables '
+ r'\(bucket = -1\), but table .* has bucket = 4'):
+ table.create_global_index(
+ 'id', index_type=index_type, options=options)
+ # Rejected before building, so no index snapshot is committed.
+ self.assertIsNone(table.snapshot_manager().get_latest_snapshot())
+
def
test_create_vindex_global_index_rejects_generic_unsupported_tables(self):
schema = pa.schema([
('id', pa.int32()),
