This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0b800e8288 [lumina] Use the persisted index metric for raw vector
search and refinement (#10562)
0b800e8288 is described below
commit 0b800e828834e069fd37a095ae20dd5b8c52fdd2
Author: Eunbin Son <[email protected]>
AuthorDate: Fri Oct 16 09:51:25 2026 +0900
[lumina] Use the persisted index metric for raw vector search and
refinement (#10562)
---
.../lumina/index/LuminaVectorGlobalIndexer.java | 39 ++++++++++-
.../LuminaVectorGlobalIndexerFactoryTest.java | 78 +++++++++++++++++++++-
2 files changed, 114 insertions(+), 3 deletions(-)
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexer.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexer.java
index c6a71d5e3f..8f62a4f8ff 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexer.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexer.java
@@ -28,6 +28,7 @@ import org.apache.paimon.options.Options;
import org.apache.paimon.types.DataType;
import org.apache.paimon.utils.Range;
+import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -37,6 +38,10 @@ public class LuminaVectorGlobalIndexer implements
VectorGlobalIndexer {
private final DataType fieldType;
private final LuminaVectorIndexOptions options;
+ // Metric persisted in the index files read by this indexer, guarded by
this.
+ private volatile String persistedMetric;
+ private boolean persistedMetricConflict;
+
public LuminaVectorGlobalIndexer(DataType fieldType, Options options) {
this.fieldType = fieldType;
this.options = new LuminaVectorIndexOptions(options);
@@ -54,11 +59,41 @@ public class LuminaVectorGlobalIndexer implements
VectorGlobalIndexer {
long totalRowCount,
List<Range> rowRanges,
ExecutorService executor) {
- return new LuminaVectorGlobalIndexReader(fileReader, files, fieldType,
options, executor);
+ LuminaVectorGlobalIndexReader reader =
+ new LuminaVectorGlobalIndexReader(fileReader, files,
fieldType, options, executor);
+ recordPersistedMetric(files.get(0).metadata());
+ return reader;
}
+ /**
+ * Returns the metric persisted in the index files read so far, so that
raw search and
+ * refinement score rows in the same metric as the index. Falls back to
the configured metric
+ * before any reader is created.
+ */
@Override
public String metric() {
- return options.metric().getLuminaName();
+ String metric = persistedMetric;
+ return metric == null ? options.metric().getLuminaName() : metric;
+ }
+
+ private synchronized void recordPersistedMetric(byte[] metadata) {
+ if (persistedMetricConflict) {
+ return;
+ }
+ String metric;
+ try {
+ metric =
LuminaIndexMeta.deserialize(metadata).metric().getLuminaName();
+ } catch (IOException | IllegalArgumentException e) {
+ // The reader reports invalid metadata when it loads the index.
+ return;
+ }
+ if (persistedMetric == null) {
+ persistedMetric = metric;
+ } else if (!persistedMetric.equals(metric)) {
+ // Shards built with different metrics have no common metric. Keep
using the
+ // configured metric, as before, instead of failing the search.
+ persistedMetricConflict = true;
+ persistedMetric = null;
+ }
}
}
diff --git
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactoryTest.java
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactoryTest.java
index fe0d5f8158..60853265b8 100644
---
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactoryTest.java
+++
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactoryTest.java
@@ -18,13 +18,24 @@
package org.apache.paimon.lumina.index;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.globalindex.GlobalIndexIOMeta;
+import org.apache.paimon.globalindex.GlobalIndexReader;
import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils;
+import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.types.DataTypes;
import org.junit.jupiter.api.Test;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
import static org.assertj.core.api.Assertions.assertThat;
-/** Tests for Lumina global indexer factory identifiers. */
+/** Tests for Lumina global indexer factory identifiers and the indexer
metric. */
public class LuminaVectorGlobalIndexerFactoryTest {
@Test
@@ -41,4 +52,69 @@ public class LuminaVectorGlobalIndexerFactoryTest {
assertThat(GlobalIndexerFactoryUtils.load("lumina-vector-ann"))
.isExactlyInstanceOf(LegacyLuminaVectorGlobalIndexerFactory.class);
}
+
+ @Test
+ public void testMetricBeforeReaderUsesConfiguration() {
+ LuminaVectorGlobalIndexer indexer = newIndexer();
+ assertThat(indexer.metric()).isEqualTo(configuredMetric());
+ }
+
+ @Test
+ public void testMetricFollowsPersistedIndexMeta() throws Exception {
+ LuminaVectorGlobalIndexer indexer = newIndexer();
+ readShard(indexer, metaBytes(LuminaVectorMetric.L2));
+
assertThat(indexer.metric()).isEqualTo(LuminaVectorMetric.L2.getLuminaName());
+ }
+
+ @Test
+ public void testConflictingShardMetricsKeepConfiguredMetric() throws
Exception {
+ LuminaVectorGlobalIndexer indexer = newIndexer();
+ readShard(indexer, metaBytes(LuminaVectorMetric.L2));
+ readShard(indexer, metaBytes(LuminaVectorMetric.COSINE));
+ assertThat(indexer.metric()).isEqualTo(configuredMetric());
+
+ readShard(indexer, metaBytes(LuminaVectorMetric.L2));
+ assertThat(indexer.metric()).isEqualTo(configuredMetric());
+ }
+
+ @Test
+ public void testCorruptMetaKeepsConfiguredMetric() throws Exception {
+ LuminaVectorGlobalIndexer indexer = newIndexer();
+ readShard(indexer, "not json".getBytes(StandardCharsets.UTF_8));
+ assertThat(indexer.metric()).isEqualTo(configuredMetric());
+
+ Map<String, String> unknownMetric = new HashMap<>();
+ unknownMetric.put("index.dimension", "2");
+ unknownMetric.put("distance.metric", "unknown");
+ readShard(indexer, new LuminaIndexMeta(unknownMetric).serialize());
+ assertThat(indexer.metric()).isEqualTo(configuredMetric());
+ }
+
+ private static LuminaVectorGlobalIndexer newIndexer() {
+ return new
LuminaVectorGlobalIndexer(DataTypes.ARRAY(DataTypes.FLOAT()), new Options());
+ }
+
+ private static String configuredMetric() {
+ return LuminaVectorMetric.INNER_PRODUCT.getLuminaName();
+ }
+
+ private static byte[] metaBytes(LuminaVectorMetric metric) throws
Exception {
+ Map<String, String> meta = new HashMap<>();
+ meta.put("index.dimension", "2");
+ meta.put("distance.metric", metric.getLuminaName());
+ return new LuminaIndexMeta(meta).serialize();
+ }
+
+ private static void readShard(LuminaVectorGlobalIndexer indexer, byte[]
metadata)
+ throws Exception {
+ GlobalIndexFileReader fileReader =
+ meta -> {
+ throw new AssertionError("Index file should not be
opened");
+ };
+ GlobalIndexIOMeta file = new GlobalIndexIOMeta(new
Path("/shard.index"), 1L, metadata);
+ try (GlobalIndexReader reader =
+ indexer.createReader(fileReader,
Collections.singletonList(file), 1L, null, null)) {
+
assertThat(reader).isInstanceOf(LuminaVectorGlobalIndexReader.class);
+ }
+ }
}
