This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 0139f7a0 Upgrade paimon-vindex-core to 0.5.0 (#879)
0139f7a0 is described below
commit 0139f7a064c286bc6f03e3e33e4c91a0b003f62d
Author: jerry <[email protected]>
AuthorDate: Sun Sep 20 13:17:10 2026 +0800
Upgrade paimon-vindex-core to 0.5.0 (#879)
---
Cargo.lock | 4 +-
DEPENDENCIES.rust.tsv | 2 +-
benchmarks/tpcds/DEPENDENCIES.rust.tsv | 2 +-
bindings/c/Cargo.toml | 2 +-
bindings/c/DEPENDENCIES.rust.tsv | 2 +-
bindings/go/DEPENDENCIES.rust.tsv | 2 +-
bindings/python/DEPENDENCIES.rust.tsv | 2 +-
crates/integration_tests/DEPENDENCIES.rust.tsv | 2 +-
.../integrations/datafusion/DEPENDENCIES.rust.tsv | 2 +-
crates/paimon-rest-server/DEPENDENCIES.rust.tsv | 2 +-
crates/paimon/Cargo.toml | 2 +-
crates/paimon/DEPENDENCIES.rust.tsv | 2 +-
crates/paimon/src/vindex/mod.rs | 105 +++++++++++++++++++++
docs/src/sql.md | 8 +-
14 files changed, 125 insertions(+), 14 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 6b3bc679..3bcd32ed 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4843,9 +4843,9 @@ dependencies = [
[[package]]
name = "paimon-vindex-core"
-version = "0.4.0"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0d3073770e80f3d82edf50a11ac4ff1f50133d2828d74532c9021c5eb3c2439c"
+checksum = "9e7541c9afa7804c8e68c0b7795f05cefa41a6403548a8b86f79088f7045301a"
dependencies = [
"half",
"matrixmultiply",
diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv
index 0e823a2b..d009b597 100644
--- a/DEPENDENCIES.rust.tsv
+++ b/DEPENDENCIES.rust.tsv
@@ -408,7 +408,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
diff --git a/benchmarks/tpcds/DEPENDENCIES.rust.tsv
b/benchmarks/tpcds/DEPENDENCIES.rust.tsv
index 15d9276a..c7f97537 100644
--- a/benchmarks/tpcds/DEPENDENCIES.rust.tsv
+++ b/benchmarks/tpcds/DEPENDENCIES.rust.tsv
@@ -274,7 +274,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml
index 949044a0..916605b7 100644
--- a/bindings/c/Cargo.toml
+++ b/bindings/c/Cargo.toml
@@ -50,6 +50,6 @@ bytes = "1.7.1"
[dev-dependencies]
# Test-only: the vector-search integration tests build a real primary-key
vindex
# IVF-flat ANN segment fixture in-process. Versions match crates/paimon.
-paimon-vindex-core = "0.4.0"
+paimon-vindex-core = "0.5.0"
tempfile = "3"
url = "2.5.2"
diff --git a/bindings/c/DEPENDENCIES.rust.tsv b/bindings/c/DEPENDENCIES.rust.tsv
index ba261e8e..d5b254c3 100644
--- a/bindings/c/DEPENDENCIES.rust.tsv
+++ b/bindings/c/DEPENDENCIES.rust.tsv
@@ -224,7 +224,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/bindings/go/DEPENDENCIES.rust.tsv
b/bindings/go/DEPENDENCIES.rust.tsv
index ba261e8e..d5b254c3 100644
--- a/bindings/go/DEPENDENCIES.rust.tsv
+++ b/bindings/go/DEPENDENCIES.rust.tsv
@@ -224,7 +224,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/bindings/python/DEPENDENCIES.rust.tsv
b/bindings/python/DEPENDENCIES.rust.tsv
index 341eda3a..23cd1edf 100644
--- a/bindings/python/DEPENDENCIES.rust.tsv
+++ b/bindings/python/DEPENDENCIES.rust.tsv
@@ -339,7 +339,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv
b/crates/integration_tests/DEPENDENCIES.rust.tsv
index 81971be1..0589ce32 100644
--- a/crates/integration_tests/DEPENDENCIES.rust.tsv
+++ b/crates/integration_tests/DEPENDENCIES.rust.tsv
@@ -207,7 +207,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
index 4d4a003d..88b5fee6 100644
--- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
+++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
@@ -348,7 +348,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
diff --git a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
index 8c60ff42..129e88ee 100644
--- a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
+++ b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
@@ -211,7 +211,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml
index a3d116fb..c970799d 100644
--- a/crates/paimon/Cargo.toml
+++ b/crates/paimon/Cargo.toml
@@ -134,7 +134,7 @@ uuid = { version = "1", features = ["v4"] }
paimon-mosaic-core = "0.2.0"
paimon-ftindex-core = { version = "0.1.0", optional = true }
tempfile = "3"
-paimon-vindex-core = "0.4.0"
+paimon-vindex-core = "0.5.0"
vortex = { version = "0.75.0", features = ["tokio"], optional = true }
libloading = "0.9"
log = "0.4"
diff --git a/crates/paimon/DEPENDENCIES.rust.tsv
b/crates/paimon/DEPENDENCIES.rust.tsv
index b4befc39..c52babe2 100644
--- a/crates/paimon/DEPENDENCIES.rust.tsv
+++ b/crates/paimon/DEPENDENCIES.rust.tsv
@@ -332,7 +332,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs
index 8ec93a13..d3c807f0 100644
--- a/crates/paimon/src/vindex/mod.rs
+++ b/crates/paimon/src/vindex/mod.rs
@@ -219,6 +219,25 @@ impl VindexVectorIndexOptions {
}
}
}
+ for key in [
+ "ivf.coarse-assignment",
+ "ivf.pq-encoding",
+ "ivf.train.max-points-per-centroid",
+ "pq.train.max-points-per-centroid",
+ ] {
+ if is_allowed_native_key(key, index_type) {
+ if let Some(value) = optional_value(
+ table_options,
+ user_options,
+ field.name(),
+ index_type,
+ key,
+ key,
+ ) {
+ native_options.insert(key.to_string(), value);
+ }
+ }
+ }
let config =
VectorIndexConfig::from_options(&native_options).map_err(|e| {
crate::Error::DataInvalid {
@@ -317,6 +336,13 @@ fn is_allowed_native_key(key: &str, index_type: &str) ->
bool {
matches!(index_type, IVF_RQ_IDENTIFIER | DISKANN_IDENTIFIER)
}
"pq.m" if index_type == IVF_PQ_IDENTIFIER => true,
+ "ivf.coarse-assignment" | "ivf.train.max-points-per-centroid" => {
+ index_type != DISKANN_IDENTIFIER
+ }
+ "ivf.pq-encoding" => index_type == IVF_PQ_IDENTIFIER,
+ "pq.train.max-points-per-centroid" => {
+ matches!(index_type, IVF_PQ_IDENTIFIER | DISKANN_IDENTIFIER)
+ }
_ => {
index_type == DISKANN_IDENTIFIER
&& DISKANN_OPTION_KEYS
@@ -337,6 +363,10 @@ fn is_allowed_paimon_suffix(suffix: &str, index_type:
&str) -> bool {
matches!(index_type, IVF_RQ_IDENTIFIER | DISKANN_IDENTIFIER)
}
"pq.m" if index_type == IVF_PQ_IDENTIFIER => true,
+ "ivf.coarse-assignment"
+ | "ivf.pq-encoding"
+ | "ivf.train.max-points-per-centroid"
+ | "pq.train.max-points-per-centroid" => is_allowed_native_key(suffix,
index_type),
_ => {
index_type == DISKANN_IDENTIFIER
&& DISKANN_OPTION_KEYS
@@ -557,6 +587,71 @@ mod tests {
);
}
+ #[test]
+ fn test_vindex_options_map_build_options() {
+ let user_options = HashMap::from([
+ (
+ "ivf-pq.ivf.coarse-assignment".to_string(),
+ "exact".to_string(),
+ ),
+ (
+ "fields.embedding.ivf.pq-encoding".to_string(),
+ "canonical".to_string(),
+ ),
+ (
+ "ivf-pq.ivf.train.max-points-per-centroid".to_string(),
+ "32".to_string(),
+ ),
+ (
+
"fields.embedding.pq.train.max-points-per-centroid".to_string(),
+ "64".to_string(),
+ ),
+ ]);
+
+ let options = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &user_options,
+ IVF_PQ_IDENTIFIER,
+ &array_float_field(),
+ )
+ .unwrap();
+
+ assert_eq!(
+ options
+ .native_options
+ .get("ivf.coarse-assignment")
+ .map(String::as_str),
+ Some("exact")
+ );
+ assert_eq!(
+ options
+ .native_options
+ .get("ivf.pq-encoding")
+ .map(String::as_str),
+ Some("canonical")
+ );
+ let resolved = options.config.resolved();
+ assert!(!resolved.use_approximate_coarse_assignment);
+ assert!(resolved.canonical_pq_encoding);
+ assert_eq!(resolved.ivf_train_max_points_per_centroid, Some(32));
+ assert_eq!(resolved.pq_train_max_points_per_centroid, Some(64));
+
+ let diskann = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &HashMap::from([(
+ "diskann.pq.train.max-points-per-centroid".to_string(),
+ "16".to_string(),
+ )]),
+ DISKANN_IDENTIFIER,
+ &array_float_field(),
+ )
+ .unwrap();
+ assert_eq!(
+ diskann.config.resolved().pq_train_max_points_per_centroid,
+ Some(16)
+ );
+ }
+
#[test]
fn test_vindex_options_map_ivf_rq_bits() {
let user_options = HashMap::from([
@@ -947,8 +1042,18 @@ mod tests {
fn test_vindex_options_reject_non_applicable_user_options() {
for (index_type, key) in [
(IVF_FLAT_IDENTIFIER, "ivf-flat.pq.m"),
+ (IVF_FLAT_IDENTIFIER, "ivf-flat.ivf.pq-encoding"),
+ (
+ IVF_FLAT_IDENTIFIER,
+ "ivf-flat.pq.train.max-points-per-centroid",
+ ),
(IVF_FLAT_IDENTIFIER, "diskann.max-degree"),
(DISKANN_IDENTIFIER, "diskann.nlist"),
+ (DISKANN_IDENTIFIER, "diskann.ivf.coarse-assignment"),
+ (
+ DISKANN_IDENTIFIER,
+ "diskann.ivf.train.max-points-per-centroid",
+ ),
] {
let user_options = HashMap::from([(key.to_string(),
"2".to_string())]);
let err = VindexVectorIndexOptions::new(
diff --git a/docs/src/sql.md b/docs/src/sql.md
index f35345dd..28c292bb 100644
--- a/docs/src/sql.md
+++ b/docs/src/sql.md
@@ -1244,9 +1244,13 @@ Supported vindex options:
| `<index-type>.dimension` | `128` | all vindex types | Vector dimension for
`ARRAY<FLOAT>` columns. Existing `VECTOR<FLOAT,N>` columns use `N` from the
type. |
| `<index-type>.distance.metric` | `inner_product` | all vindex types |
Distance metric: `inner_product`, `cosine`, or `l2`. |
| `<index-type>.nlist` | `256` | all IVF types | Number of IVF lists. DiskANN
rejects this option. |
+| `<index-type>.ivf.coarse-assignment` | `auto` | all IVF types | Build-time
list assignment mode: `auto` uses Vamana for large centroid matrices; `exact`
always uses exact assignment. |
+| `<index-type>.ivf.train.max-points-per-centroid` | `256` | all IVF types |
Positive limit on coarse K-means training data: at most `nlist × value`
vectors. |
| `<index-type>.train.sample-ratio` or `fields.<field>.train.sample-ratio` |
`1.0` | all vindex types | Fraction of shard rows selected evenly for training.
Must be in `(0, 1]`; all rows are still added to the index. The field-specific
option takes precedence. |
| `<index-type>.pq.m` | `16` | `ivf-pq` | Number of product-quantization
sub-vectors. The dimension must be divisible by this value. |
| `<index-type>.pq.use-opq` | `false` | `ivf-pq` | Whether to enable OPQ
before PQ encoding. |
+| `ivf-pq.ivf.pq-encoding` | `auto` | `ivf-pq` | Build-time PQ encoding mode:
`auto` selects an accelerated backend when supported; `canonical` uses the
canonical encoder. |
+| `<index-type>.pq.train.max-points-per-centroid` | `256` | `ivf-pq`,
`diskann` | Positive limit on PQ training data: at most `2^pq.bits × value`
vectors per subquantizer. |
| `ivf-rq.rq.bits` | `4`, or inferred | `ivf-rq` | Residual-quantization width
in the range `1` to `8`. When omitted, `ivf-rq.max-bytes-per-vector` can select
it. |
| `ivf-rq.max-bytes-per-vector` | unset | `ivf-rq` | Optional positive
persisted-code budget used to infer `rq.bits`. |
@@ -1274,7 +1278,9 @@ dimension. DiskANN accepts these build options:
For procedure calls, prefer the index-prefixed option names shown above. Native
vindex aliases are also accepted in the `options` string: `dimension`,
`metric`,
`nlist`, `pq.m`, `use-opq`, `rq.bits`, `max-bytes-per-vector`,
-`deployment-profile`, `target-recall`, `pq.code-ratio`, `pq.bits`, and the
+`deployment-profile`, `target-recall`, `pq.code-ratio`, `pq.bits`,
+`ivf.coarse-assignment`, `ivf.pq-encoding`,
+`ivf.train.max-points-per-centroid`, `pq.train.max-points-per-centroid`, and
the
`diskann.*` build keys listed in the table. Build options for another index
family are rejected rather than ignored.
