alessandrobenedetti commented on code in PR #5328:
URL: https://github.com/apache/solr/pull/5328#discussion_r4616107294
##########
solr/core/src/java/org/apache/solr/schema/DenseVectorField.java:
##########
@@ -317,29 +317,27 @@ public void checkSchemaField(final SchemaField field)
throws SolrException {
getClass().getSimpleName() + " fields can not have docValues: " +
field.getName());
}
+ if (field.stored() && field.useRawVectorsAsStored()) {
+ log.warn(
+ "You set both \"stored\"=\"true\" and
\"useRawVectorsAsStored\"=\"true\" on field '{}', there is no benefit in this. "
+ + "Stored values are not used for retrieval because
\"useRawVectorsAsStored\"=true; consider stored=false to save space. "
+ + "To choose the best option for you, please refer to the
reference guide",
+ field.getName());
+ }
+
switch (vectorEncoding) {
Review Comment:
let's remove the switch and just add the warning if dimension exceeds the max
##########
changelog/unreleased/SOLR-18207_avoid_duplicate_vector_storage.yml:
##########
@@ -0,0 +1,12 @@
+title: >
+ Add opt-in useRawVectorsAsStored support for dense vector fields,
+ allowing retrieval without duplicate stored-field storage when
+ stored=false. Supports FLOAT32 and BYTE encodings, single-valued
+ and multivalued fields, /select, RealTime Get, and preservation
Review Comment:
rather than /select let's use the generic component (as the URI name can
change in the solrconfig)
##########
solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java:
##########
@@ -358,45 +358,56 @@ private void addFlatMultiValuedVectorsToParent(
VectorEncoding encoding) {
for (String multiValuedVectorField : multiValuedVectorFields) {
List<SolrDocument> solrDocuments = children.get(multiValuedVectorField);
+ if (solrDocuments == null || solrDocuments.isEmpty()) {
+ continue;
+ }
+
List<List<Number>> multiValuedVectors = new
ArrayList<>(solrDocuments.size());
- for (SolrDocument singleVector : solrDocuments) {
- List<Number> extractedVectors;
- switch (encoding) {
- case FLOAT32:
- extractedVectors =
-
this.extractFloatVector(singleVector.getFieldValues(multiValuedVectorField));
- break;
- case BYTE:
- extractedVectors =
-
this.extractByteVector(singleVector.getFieldValues(multiValuedVectorField));
- break;
- default:
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "Unsupported vector
encoding: " + encoding);
+ for (SolrDocument singleVectorDocument : solrDocuments) {
+ var values =
singleVectorDocument.getFieldValues(multiValuedVectorField);
+ if (values == null || values.isEmpty()) {
+ continue;
}
- multiValuedVectors.add(extractedVectors);
+
+ multiValuedVectors.add(extractVector(encoding, values));
}
parent.setField(multiValuedVectorField, multiValuedVectors);
}
}
+ private List<Number> extractVector(VectorEncoding encoding,
Collection<Object> values) {
+ return switch (encoding) {
+ case FLOAT32 -> extractFloatVector(values);
+ case BYTE -> extractByteVector(values);
+ default ->
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Unsupported vector
encoding: " + encoding);
+ };
+ }
+
private List<Number> extractFloatVector(Collection<Object> fieldValues) {
- List<Number> vector = new ArrayList<>(fieldValues.size());
- for (Object fieldValue : fieldValues) {
- StoredField storedVectorValue = (StoredField) fieldValue;
- vector.add(storedVectorValue.numericValue());
- }
- return vector;
+ return fieldValues.stream()
+ // If field values are Number instances that means they come from the
vector values
+ .map(
+ value ->
+ value instanceof StoredField storedField
+ ? storedField.numericValue()
+ : (Number) value)
+ .toList();
}
private List<Number> extractByteVector(Collection<Object> singleVector) {
- StoredField vector = (StoredField) singleVector.iterator().next();
- BytesRef byteVector = vector.binaryValue();
- List<Number> extractedVector = new ArrayList<>(byteVector.length);
- for (Byte element : byteVector.bytes) {
- extractedVector.add(element.byteValue());
+ if (singleVector.iterator().next() instanceof StoredField storedVector) {
+ BytesRef byteVector = storedVector.binaryValue();
+ List<Number> extractedVector = new ArrayList<>(byteVector.length);
+ for (int i = byteVector.offset; i < byteVector.offset +
byteVector.length; i++) {
+ extractedVector.add(byteVector.bytes[i]);
+ }
+ return extractedVector;
}
- return extractedVector;
+
Review Comment:
let's add 'else' as it's an alternative
##########
solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java:
##########
@@ -769,8 +788,62 @@ public Set<String> getNonStoredDVsWithoutCopyTargets() {
return nonStoredDVsWithoutCopyTargets;
}
+ public Set<String> calcRawVectorFieldsForReturn(ReturnFields returnFields) {
Review Comment:
maybe better getRawVectorFieldsFrom
##########
solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java:
##########
@@ -769,8 +788,62 @@ public Set<String> getNonStoredDVsWithoutCopyTargets() {
return nonStoredDVsWithoutCopyTargets;
}
+ public Set<String> calcRawVectorFieldsForReturn(ReturnFields returnFields) {
+ if (derivedStoredVectorFields.isEmpty()) {
+ return Set.of();
+ }
+
+ if (returnFields.returnsAllFields()) {
+ return new HashSet<>(derivedStoredVectorFields);
+ }
+
+ if (returnFields.hasPatternMatching()) {
+ return
derivedStoredVectorFields.stream().filter(returnFields::loadsField).collect(toSet());
+ } else {
+ return ofNullable(returnFields.getLuceneFieldNames())
+ .map(fields ->
fields.stream().filter(derivedStoredVectorFields::contains))
+ .orElseGet(() ->
derivedStoredVectorFields.stream().filter(returnFields::loadsField))
+ .collect(Collectors.toSet());
+ }
+ }
+
+ public void decorateWithRawVectorFields(
+ SolrDocument sdoc, int luceneDocId, Set<String> vectorFieldNames) throws
IOException {
Review Comment:
sdoc -> toDecorate
##########
solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java:
##########
@@ -931,4 +1003,29 @@ private SolrDocument getSolrDoc(int luceneDocId) {
return sdoc;
}
}
+
+ private static List<Number> vectorValueAsNumbers(KnnVectorValues
vectorValues, int ord)
Review Comment:
getRawVector(allRawVectorsInSegment, int ord)
##########
solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java:
##########
@@ -155,6 +163,18 @@ private SolrInputDocument mergeDocHavingSameId(
final SolrInputDocument fromDoc, SolrInputDocument toDoc) {
for (SolrInputField sif : fromDoc.values()) {
Object val = sif.getValue();
+ SchemaField field = schema.getFieldOrNull(sif.getName());
Review Comment:
mmmm better a different pull request, @agazzarini to check fi there's any
priority order in merging
##########
solr/core/src/test-files/solr/collection1/conf/schema-densevector-raw-rtg.xml:
##########
@@ -0,0 +1,79 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="densevector-raw-rtg" version="1.6">
Review Comment:
rtg -> expand to real time get
##########
solr/core/src/test/org/apache/solr/schema/DenseVectorFieldRetrievalTest.java:
##########
@@ -0,0 +1,2155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.util.EmbeddedSolrServerTestRule;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class DenseVectorFieldRetrievalTest extends SolrTestCase {
+
+ private static final String SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_single_value_float_encoding";
+ private static final String SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_single_value_byte_encoding";
+
+ private static final String MULTI_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_multi_value_float_encoding";
+ private static final String MULTI_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_multi_value_byte_encoding";
+
+ private static final String STRING_FIELD_NAME = "a_string_field";
+
+ private static final List<Float> A_FLOAT_VECTOR = List.of(1.25f, 2.5f,
3.75f, 4.0f);
+ private static final List<Integer> A_BYTE_VECTOR = List.of(-128, -1, 0, 127);
+
+ private static final List<Float> ANOTHER_FLOAT_VECTOR = List.of(4.0f, 3.75f,
2.5f, 1.25f);
+ private static final List<Integer> ANOTHER_BYTE_VECTOR = List.of(127, 0, -1,
-128);
+
+ private final boolean indexed;
+ private final boolean stored;
+
+ @Rule public final EmbeddedSolrServerTestRule solrTestRule = new
EmbeddedSolrServerTestRule();
+
+ private SolrClient client;
+
+ public DenseVectorFieldRetrievalTest(
Review Comment:
add a bit of explanation that all tests are repeated 4 times based on the
combination, results must be the same
##########
solr/core/src/test/org/apache/solr/schema/DenseVectorFieldRetrievalTest.java:
##########
@@ -0,0 +1,2155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.util.EmbeddedSolrServerTestRule;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class DenseVectorFieldRetrievalTest extends SolrTestCase {
+
+ private static final String SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_single_value_float_encoding";
+ private static final String SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_single_value_byte_encoding";
+
+ private static final String MULTI_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_multi_value_float_encoding";
+ private static final String MULTI_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_multi_value_byte_encoding";
+
+ private static final String STRING_FIELD_NAME = "a_string_field";
+
+ private static final List<Float> A_FLOAT_VECTOR = List.of(1.25f, 2.5f,
3.75f, 4.0f);
+ private static final List<Integer> A_BYTE_VECTOR = List.of(-128, -1, 0, 127);
+
+ private static final List<Float> ANOTHER_FLOAT_VECTOR = List.of(4.0f, 3.75f,
2.5f, 1.25f);
+ private static final List<Integer> ANOTHER_BYTE_VECTOR = List.of(127, 0, -1,
-128);
+
+ private final boolean indexed;
+ private final boolean stored;
+
+ @Rule public final EmbeddedSolrServerTestRule solrTestRule = new
EmbeddedSolrServerTestRule();
+
+ private SolrClient client;
+
+ public DenseVectorFieldRetrievalTest(
+ @Name("indexed") boolean indexed, @Name("stored") boolean stored) {
+ this.indexed = indexed;
+ this.stored = stored;
+ }
+
+ @ParametersFactory
+ public static Iterable<Object[]> parameters() {
+ return List.of(
+ new Object[] {false, false},
+ new Object[] {false, true},
+ new Object[] {true, false},
+ new Object[] {true, true});
+ }
+
+ @Before
+ public void setUpCore() throws Exception {
+ SolrTestCaseJ4.newRandomConfig();
+ System.setProperty("solr.index.updatelog.enabled", "true");
+ System.setProperty("tests.rawvectors.indexed", Boolean.toString(indexed));
+ System.setProperty("tests.rawvectors.stored", Boolean.toString(stored));
+
+ solrTestRule.startSolr();
+ solrTestRule
+ .newCollection()
+ .withConfigSet(
+
ExternalPaths.SOURCE_HOME.resolve("core/src/test-files/solr/collection1/conf"))
+ .withConfigFile("solrconfig.xml")
+ .withSchemaFile("schema-densevector-raw-rtg.xml")
+ .create();
+
+ client = solrTestRule.getSolrClient();
+
+ client.add(newDocumentWithVectors("0", A_FLOAT_VECTOR, A_BYTE_VECTOR));
+ client.add(newDocumentWithVectors("1", ANOTHER_FLOAT_VECTOR,
ANOTHER_BYTE_VECTOR));
+
+ SolrInputDocument withoutVectors = new SolrInputDocument();
+ withoutVectors.setField(CommonParams.ID, "2");
+ client.add(withoutVectors);
+
+ client.commit();
+ }
+
+ @Test
+ public void selectWithFieldPattern_shouldReturnAllMatchingVectorFields()
throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID, "vector*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ var doc = docs.getFirst();
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithReturnAllFields_shouldReturnAllVectorFields() throws
Exception {
+ var query = new SolrQuery("id:0").setFields("*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ // Search must include raw vectors when no field list is provided.
+ var query = new SolrQuery("id:0");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
selectGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID);
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ assertEquals(Set.of(CommonParams.ID), docs.getFirst().keySet());
+ assertEquals("0", docs.getFirst().getFieldValue(CommonParams.ID));
+ }
+
+ @Test
+ public void
realtimeGetWithMultipleIds_shouldReturnMatchingDocumentsWithVectors()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var params =
+ new ModifiableSolrParams()
+ .set("ids", "1,2,0")
+ .set(CommonParams.FL, String.join(",", fieldList));
+ var docs = (SolrDocumentList) client.request(new QueryRequest("/get",
params)).get("response");
+
+ assertNotNull(docs);
+ assertEquals(3L, docs.getNumFound());
+ assertEquals(3, docs.size());
+
+ var firstDoc = docs.getFirst();
+ assertEquals(fieldList, firstDoc.keySet());
+ assertEquals("1", firstDoc.getFieldValue(CommonParams.ID));
+ assertEquals(
+ ANOTHER_FLOAT_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var secondDoc = docs.get(1);
+ assertEquals(Set.of(CommonParams.ID), secondDoc.keySet());
+ assertEquals("2", secondDoc.getFieldValue(CommonParams.ID));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var last = docs.get(2);
+ assertEquals(fieldList, last.keySet());
+ assertEquals("0", last.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
last.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
last.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnVectorsForRequestedDocument()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("1", of(fieldList));
+
+ assertEquals(fieldList, doc.keySet());
+ assertEquals("1", doc.getFieldValue(CommonParams.ID));
+ assertEquals(ANOTHER_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedFloatVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedByteVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnAllMatchingVectorFields() throws
Exception {
+ var fieldList = Set.of(CommonParams.ID, "vector*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithReturnAllFields_shouldReturnAllVectorFields()
throws Exception {
+ var fieldList = Set.of("*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ var doc = realTimeGet("0", empty());
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGet_shouldOmitMissingVectorsEvenIfRequested() throws
Exception {
+ var missingVectorsDocumentId = "2";
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+
+ var expectedFieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet(missingVectorsDocumentId, of(fieldList));
+
+ assertEquals(expectedFieldList, doc.keySet());
+ assertEquals(missingVectorsDocumentId, doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithExplicitFieldList_shouldReturnVectorFields() throws
Exception {
Review Comment:
to check as out of place
##########
solr/core/src/test/org/apache/solr/schema/DenseVectorFieldRetrievalTest.java:
##########
@@ -0,0 +1,2155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.util.EmbeddedSolrServerTestRule;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class DenseVectorFieldRetrievalTest extends SolrTestCase {
+
+ private static final String SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_single_value_float_encoding";
+ private static final String SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_single_value_byte_encoding";
+
+ private static final String MULTI_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_multi_value_float_encoding";
+ private static final String MULTI_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_multi_value_byte_encoding";
+
+ private static final String STRING_FIELD_NAME = "a_string_field";
+
+ private static final List<Float> A_FLOAT_VECTOR = List.of(1.25f, 2.5f,
3.75f, 4.0f);
+ private static final List<Integer> A_BYTE_VECTOR = List.of(-128, -1, 0, 127);
+
+ private static final List<Float> ANOTHER_FLOAT_VECTOR = List.of(4.0f, 3.75f,
2.5f, 1.25f);
+ private static final List<Integer> ANOTHER_BYTE_VECTOR = List.of(127, 0, -1,
-128);
+
+ private final boolean indexed;
+ private final boolean stored;
+
+ @Rule public final EmbeddedSolrServerTestRule solrTestRule = new
EmbeddedSolrServerTestRule();
+
+ private SolrClient client;
+
+ public DenseVectorFieldRetrievalTest(
+ @Name("indexed") boolean indexed, @Name("stored") boolean stored) {
+ this.indexed = indexed;
+ this.stored = stored;
+ }
+
+ @ParametersFactory
+ public static Iterable<Object[]> parameters() {
+ return List.of(
+ new Object[] {false, false},
+ new Object[] {false, true},
+ new Object[] {true, false},
+ new Object[] {true, true});
+ }
+
+ @Before
+ public void setUpCore() throws Exception {
+ SolrTestCaseJ4.newRandomConfig();
+ System.setProperty("solr.index.updatelog.enabled", "true");
+ System.setProperty("tests.rawvectors.indexed", Boolean.toString(indexed));
+ System.setProperty("tests.rawvectors.stored", Boolean.toString(stored));
+
+ solrTestRule.startSolr();
+ solrTestRule
+ .newCollection()
+ .withConfigSet(
+
ExternalPaths.SOURCE_HOME.resolve("core/src/test-files/solr/collection1/conf"))
+ .withConfigFile("solrconfig.xml")
+ .withSchemaFile("schema-densevector-raw-rtg.xml")
+ .create();
+
+ client = solrTestRule.getSolrClient();
+
+ client.add(newDocumentWithVectors("0", A_FLOAT_VECTOR, A_BYTE_VECTOR));
+ client.add(newDocumentWithVectors("1", ANOTHER_FLOAT_VECTOR,
ANOTHER_BYTE_VECTOR));
+
+ SolrInputDocument withoutVectors = new SolrInputDocument();
+ withoutVectors.setField(CommonParams.ID, "2");
+ client.add(withoutVectors);
+
+ client.commit();
+ }
+
+ @Test
+ public void selectWithFieldPattern_shouldReturnAllMatchingVectorFields()
throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID, "vector*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ var doc = docs.getFirst();
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithReturnAllFields_shouldReturnAllVectorFields() throws
Exception {
+ var query = new SolrQuery("id:0").setFields("*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ // Search must include raw vectors when no field list is provided.
+ var query = new SolrQuery("id:0");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
selectGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID);
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ assertEquals(Set.of(CommonParams.ID), docs.getFirst().keySet());
+ assertEquals("0", docs.getFirst().getFieldValue(CommonParams.ID));
+ }
+
+ @Test
+ public void
realtimeGetWithMultipleIds_shouldReturnMatchingDocumentsWithVectors()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var params =
+ new ModifiableSolrParams()
+ .set("ids", "1,2,0")
+ .set(CommonParams.FL, String.join(",", fieldList));
+ var docs = (SolrDocumentList) client.request(new QueryRequest("/get",
params)).get("response");
+
+ assertNotNull(docs);
+ assertEquals(3L, docs.getNumFound());
+ assertEquals(3, docs.size());
+
+ var firstDoc = docs.getFirst();
+ assertEquals(fieldList, firstDoc.keySet());
+ assertEquals("1", firstDoc.getFieldValue(CommonParams.ID));
+ assertEquals(
+ ANOTHER_FLOAT_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var secondDoc = docs.get(1);
+ assertEquals(Set.of(CommonParams.ID), secondDoc.keySet());
+ assertEquals("2", secondDoc.getFieldValue(CommonParams.ID));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var last = docs.get(2);
+ assertEquals(fieldList, last.keySet());
+ assertEquals("0", last.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
last.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
last.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnVectorsForRequestedDocument()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("1", of(fieldList));
+
+ assertEquals(fieldList, doc.keySet());
+ assertEquals("1", doc.getFieldValue(CommonParams.ID));
+ assertEquals(ANOTHER_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedFloatVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedByteVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnAllMatchingVectorFields() throws
Exception {
+ var fieldList = Set.of(CommonParams.ID, "vector*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithReturnAllFields_shouldReturnAllVectorFields()
throws Exception {
+ var fieldList = Set.of("*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ var doc = realTimeGet("0", empty());
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGet_shouldOmitMissingVectorsEvenIfRequested() throws
Exception {
+ var missingVectorsDocumentId = "2";
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+
+ var expectedFieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet(missingVectorsDocumentId, of(fieldList));
+
+ assertEquals(expectedFieldList, doc.keySet());
+ assertEquals(missingVectorsDocumentId, doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithExplicitFieldList_shouldReturnVectorFields() throws
Exception {
+ var query =
+ new SolrQuery("id:0")
+ .setFields(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ assertEquals("0", docs.getFirst().getFieldValue(CommonParams.ID));
+ assertEquals(
+ A_FLOAT_VECTOR,
docs.getFirst().getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(
+ A_BYTE_VECTOR,
docs.getFirst().getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnMatchingFloatVectorField() throws
Exception {
+ var doc = realTimeGet("0", of(Set.of(CommonParams.ID,
"vector_single_value_float*")));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
Review Comment:
assert null, for the other field
##########
solr/core/src/test/org/apache/solr/schema/DenseVectorFieldRetrievalTest.java:
##########
@@ -0,0 +1,2155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.util.EmbeddedSolrServerTestRule;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class DenseVectorFieldRetrievalTest extends SolrTestCase {
+
+ private static final String SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_single_value_float_encoding";
+ private static final String SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_single_value_byte_encoding";
+
+ private static final String MULTI_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_multi_value_float_encoding";
+ private static final String MULTI_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_multi_value_byte_encoding";
+
+ private static final String STRING_FIELD_NAME = "a_string_field";
+
+ private static final List<Float> A_FLOAT_VECTOR = List.of(1.25f, 2.5f,
3.75f, 4.0f);
+ private static final List<Integer> A_BYTE_VECTOR = List.of(-128, -1, 0, 127);
+
+ private static final List<Float> ANOTHER_FLOAT_VECTOR = List.of(4.0f, 3.75f,
2.5f, 1.25f);
+ private static final List<Integer> ANOTHER_BYTE_VECTOR = List.of(127, 0, -1,
-128);
+
+ private final boolean indexed;
+ private final boolean stored;
+
+ @Rule public final EmbeddedSolrServerTestRule solrTestRule = new
EmbeddedSolrServerTestRule();
+
+ private SolrClient client;
+
+ public DenseVectorFieldRetrievalTest(
+ @Name("indexed") boolean indexed, @Name("stored") boolean stored) {
+ this.indexed = indexed;
+ this.stored = stored;
+ }
+
+ @ParametersFactory
+ public static Iterable<Object[]> parameters() {
+ return List.of(
+ new Object[] {false, false},
+ new Object[] {false, true},
+ new Object[] {true, false},
+ new Object[] {true, true});
+ }
+
+ @Before
+ public void setUpCore() throws Exception {
+ SolrTestCaseJ4.newRandomConfig();
+ System.setProperty("solr.index.updatelog.enabled", "true");
+ System.setProperty("tests.rawvectors.indexed", Boolean.toString(indexed));
+ System.setProperty("tests.rawvectors.stored", Boolean.toString(stored));
+
+ solrTestRule.startSolr();
+ solrTestRule
+ .newCollection()
+ .withConfigSet(
+
ExternalPaths.SOURCE_HOME.resolve("core/src/test-files/solr/collection1/conf"))
+ .withConfigFile("solrconfig.xml")
+ .withSchemaFile("schema-densevector-raw-rtg.xml")
+ .create();
+
+ client = solrTestRule.getSolrClient();
+
+ client.add(newDocumentWithVectors("0", A_FLOAT_VECTOR, A_BYTE_VECTOR));
+ client.add(newDocumentWithVectors("1", ANOTHER_FLOAT_VECTOR,
ANOTHER_BYTE_VECTOR));
+
+ SolrInputDocument withoutVectors = new SolrInputDocument();
+ withoutVectors.setField(CommonParams.ID, "2");
+ client.add(withoutVectors);
+
+ client.commit();
+ }
+
+ @Test
+ public void selectWithFieldPattern_shouldReturnAllMatchingVectorFields()
throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID, "vector*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ var doc = docs.getFirst();
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithReturnAllFields_shouldReturnAllVectorFields() throws
Exception {
+ var query = new SolrQuery("id:0").setFields("*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ // Search must include raw vectors when no field list is provided.
+ var query = new SolrQuery("id:0");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
selectGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID);
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ assertEquals(Set.of(CommonParams.ID), docs.getFirst().keySet());
+ assertEquals("0", docs.getFirst().getFieldValue(CommonParams.ID));
+ }
+
+ @Test
+ public void
realtimeGetWithMultipleIds_shouldReturnMatchingDocumentsWithVectors()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var params =
+ new ModifiableSolrParams()
+ .set("ids", "1,2,0")
+ .set(CommonParams.FL, String.join(",", fieldList));
+ var docs = (SolrDocumentList) client.request(new QueryRequest("/get",
params)).get("response");
+
+ assertNotNull(docs);
+ assertEquals(3L, docs.getNumFound());
+ assertEquals(3, docs.size());
+
+ var firstDoc = docs.getFirst();
+ assertEquals(fieldList, firstDoc.keySet());
+ assertEquals("1", firstDoc.getFieldValue(CommonParams.ID));
+ assertEquals(
+ ANOTHER_FLOAT_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var secondDoc = docs.get(1);
+ assertEquals(Set.of(CommonParams.ID), secondDoc.keySet());
+ assertEquals("2", secondDoc.getFieldValue(CommonParams.ID));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var last = docs.get(2);
+ assertEquals(fieldList, last.keySet());
+ assertEquals("0", last.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
last.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
last.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnVectorsForRequestedDocument()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("1", of(fieldList));
+
+ assertEquals(fieldList, doc.keySet());
+ assertEquals("1", doc.getFieldValue(CommonParams.ID));
+ assertEquals(ANOTHER_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedFloatVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedByteVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnAllMatchingVectorFields() throws
Exception {
+ var fieldList = Set.of(CommonParams.ID, "vector*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithReturnAllFields_shouldReturnAllVectorFields()
throws Exception {
+ var fieldList = Set.of("*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ var doc = realTimeGet("0", empty());
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGet_shouldOmitMissingVectorsEvenIfRequested() throws
Exception {
+ var missingVectorsDocumentId = "2";
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+
+ var expectedFieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet(missingVectorsDocumentId, of(fieldList));
+
+ assertEquals(expectedFieldList, doc.keySet());
+ assertEquals(missingVectorsDocumentId, doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithExplicitFieldList_shouldReturnVectorFields() throws
Exception {
+ var query =
+ new SolrQuery("id:0")
+ .setFields(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ assertEquals("0", docs.getFirst().getFieldValue(CommonParams.ID));
+ assertEquals(
+ A_FLOAT_VECTOR,
docs.getFirst().getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(
+ A_BYTE_VECTOR,
docs.getFirst().getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnMatchingFloatVectorField() throws
Exception {
+ var doc = realTimeGet("0", of(Set.of(CommonParams.ID,
"vector_single_value_float*")));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnMatchingByteVectorField() throws
Exception {
+ // A byte-only pattern must exclude the float vector from the response.
+ var doc = realTimeGet("0", of(Set.of(CommonParams.ID,
"vector_single_value_byte*")));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void rawVectors_winOverStoredValues() throws Exception {
Review Comment:
add another test that check that in case of useRawVectors false, the stored
values are read
##########
solr/core/src/test/org/apache/solr/schema/DenseVectorFieldRetrievalTest.java:
##########
@@ -0,0 +1,2155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.util.EmbeddedSolrServerTestRule;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class DenseVectorFieldRetrievalTest extends SolrTestCase {
+
+ private static final String SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_single_value_float_encoding";
+ private static final String SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_single_value_byte_encoding";
+
+ private static final String MULTI_VALUE_FLOAT_ENCODED_FIELD_NAME =
+ "vector_multi_value_float_encoding";
+ private static final String MULTI_VALUE_BYTE_ENCODED_FIELD_NAME =
+ "vector_multi_value_byte_encoding";
+
+ private static final String STRING_FIELD_NAME = "a_string_field";
+
+ private static final List<Float> A_FLOAT_VECTOR = List.of(1.25f, 2.5f,
3.75f, 4.0f);
+ private static final List<Integer> A_BYTE_VECTOR = List.of(-128, -1, 0, 127);
+
+ private static final List<Float> ANOTHER_FLOAT_VECTOR = List.of(4.0f, 3.75f,
2.5f, 1.25f);
+ private static final List<Integer> ANOTHER_BYTE_VECTOR = List.of(127, 0, -1,
-128);
+
+ private final boolean indexed;
+ private final boolean stored;
+
+ @Rule public final EmbeddedSolrServerTestRule solrTestRule = new
EmbeddedSolrServerTestRule();
+
+ private SolrClient client;
+
+ public DenseVectorFieldRetrievalTest(
+ @Name("indexed") boolean indexed, @Name("stored") boolean stored) {
+ this.indexed = indexed;
+ this.stored = stored;
+ }
+
+ @ParametersFactory
+ public static Iterable<Object[]> parameters() {
+ return List.of(
+ new Object[] {false, false},
+ new Object[] {false, true},
+ new Object[] {true, false},
+ new Object[] {true, true});
+ }
+
+ @Before
+ public void setUpCore() throws Exception {
+ SolrTestCaseJ4.newRandomConfig();
+ System.setProperty("solr.index.updatelog.enabled", "true");
+ System.setProperty("tests.rawvectors.indexed", Boolean.toString(indexed));
+ System.setProperty("tests.rawvectors.stored", Boolean.toString(stored));
+
+ solrTestRule.startSolr();
+ solrTestRule
+ .newCollection()
+ .withConfigSet(
+
ExternalPaths.SOURCE_HOME.resolve("core/src/test-files/solr/collection1/conf"))
+ .withConfigFile("solrconfig.xml")
+ .withSchemaFile("schema-densevector-raw-rtg.xml")
+ .create();
+
+ client = solrTestRule.getSolrClient();
+
+ client.add(newDocumentWithVectors("0", A_FLOAT_VECTOR, A_BYTE_VECTOR));
+ client.add(newDocumentWithVectors("1", ANOTHER_FLOAT_VECTOR,
ANOTHER_BYTE_VECTOR));
+
+ SolrInputDocument withoutVectors = new SolrInputDocument();
+ withoutVectors.setField(CommonParams.ID, "2");
+ client.add(withoutVectors);
+
+ client.commit();
+ }
+
+ @Test
+ public void selectWithFieldPattern_shouldReturnAllMatchingVectorFields()
throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID, "vector*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ var doc = docs.getFirst();
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithReturnAllFields_shouldReturnAllVectorFields() throws
Exception {
+ var query = new SolrQuery("id:0").setFields("*");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ // Search must include raw vectors when no field list is provided.
+ var query = new SolrQuery("id:0");
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+
+ var doc = docs.getFirst();
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
selectGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var query = new SolrQuery("id:0").setFields(CommonParams.ID);
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ assertEquals(Set.of(CommonParams.ID), docs.getFirst().keySet());
+ assertEquals("0", docs.getFirst().getFieldValue(CommonParams.ID));
+ }
+
+ @Test
+ public void
realtimeGetWithMultipleIds_shouldReturnMatchingDocumentsWithVectors()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var params =
+ new ModifiableSolrParams()
+ .set("ids", "1,2,0")
+ .set(CommonParams.FL, String.join(",", fieldList));
+ var docs = (SolrDocumentList) client.request(new QueryRequest("/get",
params)).get("response");
+
+ assertNotNull(docs);
+ assertEquals(3L, docs.getNumFound());
+ assertEquals(3, docs.size());
+
+ var firstDoc = docs.getFirst();
+ assertEquals(fieldList, firstDoc.keySet());
+ assertEquals("1", firstDoc.getFieldValue(CommonParams.ID));
+ assertEquals(
+ ANOTHER_FLOAT_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
firstDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var secondDoc = docs.get(1);
+ assertEquals(Set.of(CommonParams.ID), secondDoc.keySet());
+ assertEquals("2", secondDoc.getFieldValue(CommonParams.ID));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(secondDoc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+
+ var last = docs.get(2);
+ assertEquals(fieldList, last.keySet());
+ assertEquals("0", last.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
last.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
last.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnVectorsForRequestedDocument()
+ throws Exception {
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("1", of(fieldList));
+
+ assertEquals(fieldList, doc.keySet());
+ assertEquals("1", doc.getFieldValue(CommonParams.ID));
+ assertEquals(ANOTHER_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(ANOTHER_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedFloatVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldReturnOnlyRequestedByteVectorField()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithExplicitFieldList_shouldExcludeUnrequestedVectorFields()
+ throws Exception {
+ var fieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(Set.of(CommonParams.ID), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnAllMatchingVectorFields() throws
Exception {
+ var fieldList = Set.of(CommonParams.ID, "vector*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals(
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME),
+ doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithReturnAllFields_shouldReturnAllVectorFields()
throws Exception {
+ var fieldList = Set.of("*");
+ var doc = realTimeGet("0", of(fieldList));
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGetWithNoFieldList_shouldReturnAllVectorFields() throws
Exception {
+ var doc = realTimeGet("0", empty());
+
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void realtimeGet_shouldOmitMissingVectorsEvenIfRequested() throws
Exception {
+ var missingVectorsDocumentId = "2";
+ var fieldList =
+ Set.of(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+
+ var expectedFieldList = Set.of(CommonParams.ID);
+ var doc = realTimeGet(missingVectorsDocumentId, of(fieldList));
+
+ assertEquals(expectedFieldList, doc.keySet());
+ assertEquals(missingVectorsDocumentId, doc.getFieldValue(CommonParams.ID));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertNull(doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void selectWithExplicitFieldList_shouldReturnVectorFields() throws
Exception {
+ var query =
+ new SolrQuery("id:0")
+ .setFields(
+ CommonParams.ID,
+ SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME,
+ SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME);
+
+ var docs = new QueryRequest("/select", query).process(client).getResults();
+
+ assertEquals(1L, docs.getNumFound());
+ assertEquals(1, docs.size());
+ assertEquals("0", docs.getFirst().getFieldValue(CommonParams.ID));
+ assertEquals(
+ A_FLOAT_VECTOR,
docs.getFirst().getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ assertEquals(
+ A_BYTE_VECTOR,
docs.getFirst().getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnMatchingFloatVectorField() throws
Exception {
+ var doc = realTimeGet("0", of(Set.of(CommonParams.ID,
"vector_single_value_float*")));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_FLOAT_VECTOR,
doc.getFieldValue(SINGLE_VALUE_FLOAT_ENCODED_FIELD_NAME));
+ }
+
+ @Test
+ public void
realtimeGetWithFieldPattern_shouldReturnMatchingByteVectorField() throws
Exception {
+ // A byte-only pattern must exclude the float vector from the response.
+ var doc = realTimeGet("0", of(Set.of(CommonParams.ID,
"vector_single_value_byte*")));
+
+ assertEquals(Set.of(CommonParams.ID,
SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME), doc.keySet());
+ assertEquals("0", doc.getFieldValue(CommonParams.ID));
+ assertEquals(A_BYTE_VECTOR,
doc.getFieldValue(SINGLE_VALUE_BYTE_ENCODED_FIELD_NAME));
Review Comment:
assert null for the other
##########
solr/core/src/java/org/apache/solr/schema/DenseVectorField.java:
##########
@@ -317,29 +317,27 @@ public void checkSchemaField(final SchemaField field)
throws SolrException {
getClass().getSimpleName() + " fields can not have docValues: " +
field.getName());
}
+ if (field.stored() && field.useRawVectorsAsStored()) {
+ log.warn(
+ "You set both \"stored\"=\"true\" and
\"useRawVectorsAsStored\"=\"true\" on field '{}', there is no benefit in this. "
+ + "Stored values are not used for retrieval because
\"useRawVectorsAsStored\"=true; consider stored=false to save space. "
+ + "To choose the best option for you, please refer to the
reference guide",
Review Comment:
after discussion, we agree to remove the last line as there's no other best
option to check in the reference guide
##########
solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java:
##########
@@ -95,6 +103,7 @@ public class SolrDocumentFetcher {
private final Set<String> allStored;
private final Set<String> dvsCanSubstituteStored;
+ private final Set<String> derivedStoredVectorFields;
Review Comment:
let's change the variable name 'rawVectorsFields'
##########
changelog/unreleased/SOLR-18207_avoid_duplicate_vector_storage.yml:
##########
@@ -0,0 +1,12 @@
+title: >
+ Add opt-in useRawVectorsAsStored support for dense vector fields,
+ allowing retrieval without duplicate stored-field storage when
+ stored=false. Supports FLOAT32 and BYTE encodings, single-valued
+ and multivalued fields, /select, RealTime Get, and preservation
+ during atomic updates.
+type: added
+authors:
+ - name: Andrea Gazzarini
Review Comment:
add also David and I (in general substantial reviewers/helper)
##########
solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java:
##########
@@ -769,8 +788,62 @@ public Set<String> getNonStoredDVsWithoutCopyTargets() {
return nonStoredDVsWithoutCopyTargets;
}
+ public Set<String> calcRawVectorFieldsForReturn(ReturnFields returnFields) {
+ if (derivedStoredVectorFields.isEmpty()) {
+ return Set.of();
+ }
+
+ if (returnFields.returnsAllFields()) {
+ return new HashSet<>(derivedStoredVectorFields);
+ }
+
+ if (returnFields.hasPatternMatching()) {
+ return
derivedStoredVectorFields.stream().filter(returnFields::loadsField).collect(toSet());
+ } else {
+ return ofNullable(returnFields.getLuceneFieldNames())
+ .map(fields ->
fields.stream().filter(derivedStoredVectorFields::contains))
+ .orElseGet(() ->
derivedStoredVectorFields.stream().filter(returnFields::loadsField))
+ .collect(Collectors.toSet());
+ }
+ }
+
+ public void decorateWithRawVectorFields(
+ SolrDocument sdoc, int luceneDocId, Set<String> vectorFieldNames) throws
IOException {
+ if (vectorFieldNames.isEmpty()) {
+ return;
+ }
+
+ final List<LeafReaderContext> leafContexts = searcher.getLeafContexts();
+ final int subIndex = ReaderUtil.subIndex(luceneDocId, leafContexts);
+ final LeafReaderContext leafReaderContext = leafContexts.get(subIndex);
+ final LeafReader leafReader = leafReaderContext.reader();
+ final int localId = luceneDocId - leafReaderContext.docBase;
+
+ for (String vectorFieldName : vectorFieldNames) {
+ SchemaField schemaField =
searcher.getSchema().getFieldOrNull(vectorFieldName);
+ if (schemaField == null || !(schemaField.getType() instanceof
DenseVectorField vectorField)) {
+ continue;
+ }
+
+ KnnVectorValues vectorValues =
Review Comment:
allRawVectorsInSegment
##########
solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java:
##########
@@ -769,8 +788,62 @@ public Set<String> getNonStoredDVsWithoutCopyTargets() {
return nonStoredDVsWithoutCopyTargets;
}
+ public Set<String> calcRawVectorFieldsForReturn(ReturnFields returnFields) {
+ if (derivedStoredVectorFields.isEmpty()) {
+ return Set.of();
+ }
+
+ if (returnFields.returnsAllFields()) {
+ return new HashSet<>(derivedStoredVectorFields);
+ }
+
+ if (returnFields.hasPatternMatching()) {
+ return
derivedStoredVectorFields.stream().filter(returnFields::loadsField).collect(toSet());
+ } else {
+ return ofNullable(returnFields.getLuceneFieldNames())
+ .map(fields ->
fields.stream().filter(derivedStoredVectorFields::contains))
+ .orElseGet(() ->
derivedStoredVectorFields.stream().filter(returnFields::loadsField))
+ .collect(Collectors.toSet());
+ }
+ }
+
+ public void decorateWithRawVectorFields(
+ SolrDocument sdoc, int luceneDocId, Set<String> vectorFieldNames) throws
IOException {
+ if (vectorFieldNames.isEmpty()) {
+ return;
+ }
+
+ final List<LeafReaderContext> leafContexts = searcher.getLeafContexts();
+ final int subIndex = ReaderUtil.subIndex(luceneDocId, leafContexts);
+ final LeafReaderContext leafReaderContext = leafContexts.get(subIndex);
+ final LeafReader leafReader = leafReaderContext.reader();
+ final int localId = luceneDocId - leafReaderContext.docBase;
+
+ for (String vectorFieldName : vectorFieldNames) {
Review Comment:
redundant probably
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]