This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit dac50a356c918011c2faf3755513563b4224ac9b Author: Kevin Liang <[email protected]> AuthorDate: Thu Sep 4 10:07:25 2025 -0400 SOLR-17780: Add support for scalar quantized dense vectors (#3385) * Code, tests, and documentation --------- Co-authored-by: kliang78 <[email protected]> Co-authored-by: Alessandro Benedetti <[email protected]> (cherry picked from commit 8d6b6d6f94255a724e27fa2c844138da32711127) --- .../org/apache/solr/core/SchemaCodecFactory.java | 13 +- .../org/apache/solr/schema/DenseVectorField.java | 5 + .../schema/ScalarQuantizedDenseVectorField.java | 135 +++++++++++++++++ .../conf/bad-schema-densevector-quantized-bits.xml | 64 ++++++++ .../bad-schema-densevector-quantized-compress.xml | 67 +++++++++ ...sevector-quantized-confidence-interval-high.xml | 66 +++++++++ ...nsevector-quantized-confidence-interval-low.xml | 66 +++++++++ .../conf/schema-densevector-quantized.xml | 86 +++++++++++ .../solr/core/TestSchemaCodecFactoryDefaults.java | 12 +- .../ScalarQuantizedDenseVectorFieldTest.java | 163 +++++++++++++++++++++ .../query-guide/pages/dense-vector-search.adoc | 76 ++++++++++ 11 files changed, 739 insertions(+), 14 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java index 76ce9322ec0..60c306ee8ef 100644 --- a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java +++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java @@ -127,17 +127,14 @@ public class SchemaCodecFactory extends CodecFactory implements SolrCoreAware { final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field); FieldType fieldType = (schemaField == null ? null : schemaField.getType()); if (fieldType instanceof DenseVectorField) { - DenseVectorField vectorType = (DenseVectorField) fieldType; - String knnAlgorithm = vectorType.getKnnAlgorithm(); - if (DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) { - int maxConn = vectorType.getHnswMaxConn(); - int beamWidth = vectorType.getHnswBeamWidth(); - var delegate = new Lucene99HnswVectorsFormat(maxConn, beamWidth); - return new SolrDelegatingKnnVectorsFormat(delegate, vectorType.getDimension()); - } else { + DenseVectorField vectorField = (DenseVectorField) fieldType; + final String knnAlgorithm = vectorField.getKnnAlgorithm(); + if (!DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) { throw new SolrException( ErrorCode.SERVER_ERROR, knnAlgorithm + " KNN algorithm is not supported"); } + return new SolrDelegatingKnnVectorsFormat( + vectorField.buildKnnVectorsFormat(), vectorField.getDimension()); } return super.getKnnVectorsFormatForField(field); } diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java index c6607d0556f..5e29d69b6a6 100644 --- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnFloatVectorField; @@ -342,6 +343,10 @@ public class DenseVectorField extends FloatPointField { } } + public KnnVectorsFormat buildKnnVectorsFormat() { + return new Lucene99HnswVectorsFormat(hnswMaxConn, hnswBeamWidth); + } + @Override public UninvertingReader.Type getUninversionType(SchemaField sf) { return null; diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java new file mode 100644 index 00000000000..885d09cea17 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import static java.util.Optional.ofNullable; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; + +import java.util.Map; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.solr.common.SolrException; + +public class ScalarQuantizedDenseVectorField extends DenseVectorField { + public static final String BITS_PARAM = "bits"; // + public static final String CONFIDENCE_INTERVAL_PARAM = "confidenceInterval"; + public static final String DYNAMIC_CONFIDENCE_INTERVAL_PARAM = "dynamicConfidenceInterval"; + public static final String COMPRESS_PARAM = + "compress"; // can only be enabled when bits = 4 per Lucene codec spec + + static final int DEFAULT_BITS = 7; // use signed byte as default when unspecified + static final Float DEFAULT_CONFIDENCE_INTERVAL = null; // use dimension scaled confidence interval + + /** + * Number of bits to use for storage Must be 4 (half-byte) or 7 (signed-byte) per Lucene codec + * spec + */ + private int bits; + + /** + * Confidence interval to use for scalar quantization Default is calculated as + * `1-1/(vector_dimensions + 1)` + */ + private Float confidenceInterval; + + /** + * When enabled, in conjunction with 4 bit size, will pair values into single bytes for 50% + * reduction in memory usage (comes at the cost of some decode speed penalty) + */ + private boolean compress; + + public ScalarQuantizedDenseVectorField() { + super(); + } + + public ScalarQuantizedDenseVectorField( + int dimension, + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding, + int bits, + Float confidenceInterval, + boolean compress) { + super(dimension, similarityFunction, vectorEncoding); + this.bits = bits; + this.confidenceInterval = confidenceInterval; + this.compress = compress; + } + + @Override + public void init(IndexSchema schema, Map<String, String> args) { + this.bits = ofNullable(args.remove(BITS_PARAM)).map(Integer::parseInt).orElse(DEFAULT_BITS); + + this.compress = + ofNullable(args.remove(COMPRESS_PARAM)).map(Boolean::parseBoolean).orElse(false); + + this.confidenceInterval = + ofNullable(args.remove(CONFIDENCE_INTERVAL_PARAM)) + .map(Float::parseFloat) + .orElse(DEFAULT_CONFIDENCE_INTERVAL); + + if (ofNullable(args.remove(DYNAMIC_CONFIDENCE_INTERVAL_PARAM)) + .map(Boolean::parseBoolean) + .orElse(false)) { + this.confidenceInterval = Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL; + } + + super.init(schema, args); + } + + @Override + public KnnVectorsFormat buildKnnVectorsFormat() { + return new Lucene99HnswScalarQuantizedVectorsFormat( + getHnswMaxConn(), + getHnswBeamWidth(), + DEFAULT_NUM_MERGE_WORKER, + getBits(), + useCompression(), + getConfidenceInterval(), + null); + } + + @Override + public void checkSchemaField(final SchemaField field) throws SolrException { + super.checkSchemaField(field); + + try { + // format does not expose any validation, however by constructing + // we will run lucene level argument validation + buildKnnVectorsFormat(); + } catch (IllegalArgumentException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + getClass().getSimpleName() + " " + e.getMessage() + ": " + field.getName(), + e); + } + } + + public int getBits() { + return bits; + } + + public boolean useCompression() { + return compress; + } + + public Float getConfidenceInterval() { + return confidenceInterval; + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml new file mode 100644 index 00000000000..c0a740a74d7 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-bits.xml @@ -0,0 +1,64 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- Test schema file for DenseVectorField types --> + +<schema name="bad-schema-densevector-quantized-bits" version="1.0"> + <fieldType name="string" class="solr.StrField" multiValued="true"/> + <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" /> + <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/> + <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/> + + <!-- Scalar Quantized vectors --> + <!-- use an invalid bit value --> + <fieldType name="knn_vector_scalar_quantized_bits" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + bits="6"/> + <field name="v_scalar_bits" type="knn_vector_scalar_quantized_bits" indexed="true" stored="true" /> + + <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + <field name="vector" type="knn_vector" indexed="true" stored="true"/> + <field name="vector2" type="knn_vector" indexed="true" stored="true"/> + <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" /> + <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" /> + <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" /> + + <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + + <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" /> + <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/> + <copyField source="*" dest="_text_"/> + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <uniqueKey>id</uniqueKey> +</schema> diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml new file mode 100644 index 00000000000..1fd4242390f --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-compress.xml @@ -0,0 +1,67 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- Test schema file for DenseVectorField types --> + +<schema name="bad-schema-densevector-quantized-compress" version="1.0"> + <fieldType name="string" class="solr.StrField" multiValued="true"/> + <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" /> + <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/> + <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/> + + <!-- Scalar Quantized vectors --> + <!-- attempt to use compression with an invalid bit size alignment --> + <fieldType name="knn_vector_scalar_quantized_compressed" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + bits="7" + compress="true"/> + <field name="v_scalar_compressed" type="knn_vector_scalar_quantized_compressed" indexed="true" stored="true" /> + + <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + <field name="vector" type="knn_vector" indexed="true" stored="true"/> + <field name="vector2" type="knn_vector" indexed="true" stored="true"/> + <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" /> + <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" /> + <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" /> + + + + <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + + <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" /> + <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/> + <copyField source="*" dest="_text_"/> + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <uniqueKey>id</uniqueKey> +</schema> diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml new file mode 100644 index 00000000000..4c6bbbb17aa --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-high.xml @@ -0,0 +1,66 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- Test schema file for DenseVectorField types --> + +<schema name="bad-schema-densevector-quantized-confidence-interval-high" version="1.0"> + <fieldType name="string" class="solr.StrField" multiValued="true"/> + <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" /> + <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/> + <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/> + + <!-- Scalar Quantized vectors --> + <!-- attempt to use compression with an invalid bit size alignment --> + <fieldType name="knn_vector_scalar_quantized_ci_high" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + confidenceInterval="1.5"/> + <field name="v_scalar_ci" type="knn_vector_scalar_quantized_ci_high" indexed="true" stored="true" /> + + <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + <field name="vector" type="knn_vector" indexed="true" stored="true"/> + <field name="vector2" type="knn_vector" indexed="true" stored="true"/> + <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" /> + <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" /> + <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" /> + + + + <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + + <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" /> + <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/> + <copyField source="*" dest="_text_"/> + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <uniqueKey>id</uniqueKey> +</schema> diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml new file mode 100644 index 00000000000..9de85797208 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-quantized-confidence-interval-low.xml @@ -0,0 +1,66 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- Test schema file for DenseVectorField types --> + +<schema name="bad-schema-densevector-quantized-confidence-interval-low" version="1.0"> + <fieldType name="string" class="solr.StrField" multiValued="true"/> + <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" /> + <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/> + <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/> + + <!-- Scalar Quantized vectors --> + <!-- attempt to use compression with an invalid bit size alignment --> + <fieldType name="knn_vector_scalar_quantized_ci_low" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + confidenceInterval="0.8"/> + <field name="v_scalar_ci" type="knn_vector_scalar_quantized_ci_low" indexed="true" stored="true" /> + + <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + <field name="vector" type="knn_vector" indexed="true" stored="true"/> + <field name="vector2" type="knn_vector" indexed="true" stored="true"/> + <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" /> + <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" /> + <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" /> + + + + <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + + <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" /> + <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/> + <copyField source="*" dest="_text_"/> + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <uniqueKey>id</uniqueKey> +</schema> diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml new file mode 100644 index 00000000000..b2e9b0d27f3 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-quantized.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- Test schema file for DenseVectorField types --> + +<schema name="schema-densevector-quantized" version="1.0"> + <fieldType name="string" class="solr.StrField" multiValued="true"/> + <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" /> + <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="high_dimensional_float_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="FLOAT32"/> + <fieldType name="high_dimensional_byte_knn_vector" class="solr.DenseVectorField" vectorDimension="2048" similarityFunction="cosine" vectorEncoding="BYTE"/> + <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/> + + <!-- Scalar Quantized vectors --> + <fieldType name="knn_vector_scalar_quantized_default" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + /> + <fieldType name="knn_vector_scalar_quantized_half_byte" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + bits="4"/> + <fieldType name="knn_vector_scalar_quantized_compressed" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + bits="4" + compress="true"/> + <fieldType name="knn_vector_scalar_quantized_confidence_interval" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + confidenceInterval="0.91"/> + <fieldType name="knn_vector_scalar_quantized_confidence_dynamic" class="solr.ScalarQuantizedDenseVectorField" + vectorDimension="4" + similarityFunction="cosine" + dynamicConfidenceInterval="true"/> + + + <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + <field name="vector" type="knn_vector" indexed="true" stored="true"/> + <field name="vector2" type="knn_vector" indexed="true" stored="true"/> + <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" /> + <field name="2048_byte_vector" type="high_dimensional_byte_knn_vector" indexed="true" stored="true" /> + <field name="2048_float_vector" type="high_dimensional_float_knn_vector" indexed="true" stored="true" /> + + <field name="v_scalar_default" type="knn_vector_scalar_quantized_default" indexed="true" stored="true" /> + <field name="v_scalar_half_byte" type="knn_vector_scalar_quantized_half_byte" indexed="true" stored="true" /> + <field name="v_scalar_compressed" type="knn_vector_scalar_quantized_compressed" indexed="true" stored="true" /> + <field name="v_scalar_confidence" type="knn_vector_scalar_quantized_confidence_interval" indexed="true" stored="true" /> + <field name="v_scalar_dynamic" type="knn_vector_scalar_quantized_confidence_dynamic" indexed="true" stored="true" /> + + <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/> + + <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" /> + <field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/> + <copyField source="*" dest="_text_"/> + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/> + <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <uniqueKey>id</uniqueKey> +</schema> diff --git a/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java b/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java index 8f27aa930c9..d8c5b60c821 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java +++ b/solr/core/src/test/org/apache/solr/core/TestSchemaCodecFactoryDefaults.java @@ -48,7 +48,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 { // TODO: replace with EmbeddedSolrServerTestRule (at a glance, I'm not sure how just yet) initCore("solrconfig-minimal.xml", "schema-minimal.xml"); - // Some basic assertions that are fundemental to the entire test... + // Some basic assertions that are fundamental to the entire test... assertNull( "Someone broke test config so it declares a CodecFactory, making this test useless", @@ -84,7 +84,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 { * Methods with these names must exist, take a single String arg, and return instances of the same * concrete class as the corrisponding method in the Lucene default codec * - * @see #testPerFieldMethodEquivilence + * @see #testPerFieldMethodEquivalence * @see #testCodecDeclaredMethodCoverage */ public static Set<String> PER_FIELD_FORMAT_METHOD_NAMES = @@ -97,10 +97,10 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 { * * <p>The primary purpose of this Set is to list the no-arg versions of per-field methods that are * already tested via in {@link #PER_FIELD_FORMAT_METHOD_NAMES} (Because the solr "per-field" - * wrapper wll be different then the Lucene "per-field" default impl of these methods) + * wrapper will be different from the Lucene "per-field" default impl of these methods) * * <p>The other reason to include methods in this set, is if they already have bespoke tests - * (perhaps because thye require arugments?) and can't be automatically tested via reflection + * (perhaps because they require arguments?) and can't be automatically tested via reflection * * @see #testCodecDeclaredMethodCoverage */ @@ -114,7 +114,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 { * * @see #PER_FIELD_FORMAT_METHOD_NAMES */ - public void testPerFieldMethodEquivilence() throws Exception { + public void testPerFieldMethodEquivalence() throws Exception { final String fieldNameShouldNotMatter = TestUtil.randomSimpleString(random(), 64); final Codec expectedCodec = LUCENE_DEFAULT_CODEC; @@ -138,7 +138,7 @@ public class TestSchemaCodecFactoryDefaults extends SolrTestCaseJ4 { * runtime type. * * <p>Any method found which requires arguments will cause a failure unless it is in {@link - * #PER_FIELD_FORMAT_METHOD_NAMES} (which are tested by {@link #testPerFieldMethodEquivilence}) or + * #PER_FIELD_FORMAT_METHOD_NAMES} (which are tested by {@link #testPerFieldMethodEquivalence}) or * {@link #IGNORED_CODEC_DECLARED_METHOD_NAMES} (Exempt from testing, or should have their own * bespoke test) */ diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java new file mode 100644 index 00000000000..70a239600f6 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import static org.hamcrest.core.Is.is; + +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.solr.core.AbstractBadConfigTestBase; +import org.junit.Before; +import org.junit.Test; + +public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBase { + @Before + public void init() {} + + @Test + public void fieldTypeDefinition_invalidBitSize_shouldThrowException() throws Exception { + assertConfigs( + "solrconfig-basic.xml", + "bad-schema-densevector-quantized-bits.xml", + "ScalarQuantizedDenseVectorField bits must be one of: 4, 7; bits=6: v_scalar_bits"); + } + + @Test + public void fieldTypeDefinition_improperCompressionUse_shouldThrowException() throws Exception { + assertConfigs( + "solrconfig-basic.xml", + "bad-schema-densevector-quantized-compress.xml", + "ScalarQuantizedDenseVectorField compress=true only applies when bits=4: v_scalar_compressed"); + } + + @Test + public void fieldTypeDefinition_confidenceIntervalTooLow_shouldThrowException() throws Exception { + assertConfigs( + "solrconfig-basic.xml", + "bad-schema-densevector-quantized-confidence-interval-low.xml", + "ScalarQuantizedDenseVectorField confidenceInterval must be between 0.9 and 1.0 or 0; confidenceInterval=0.8: v_scalar_ci"); + } + + @Test + public void fieldTypeDefinition_confidenceIntervalTooHigh_shouldThrowException() + throws Exception { + assertConfigs( + "solrconfig-basic.xml", + "bad-schema-densevector-quantized-confidence-interval-high.xml", + "ScalarQuantizedDenseVectorField confidenceInterval must be between 0.9 and 1.0 or 0; confidenceInterval=1.5: v_scalar_ci"); + } + + @Test + public void fieldDefinition_default_shouldLoadSchemaField() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); + + IndexSchema schema = h.getCore().getLatestSchema(); + + SchemaField defaultVectorField = schema.getField("v_scalar_default"); + assertNotNull(defaultVectorField); + + ScalarQuantizedDenseVectorField defaultVectorType = + (ScalarQuantizedDenseVectorField) defaultVectorField.getType(); + assertThat(defaultVectorType.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE)); + assertThat(defaultVectorType.getDimension(), is(4)); + assertThat(defaultVectorType.getKnnAlgorithm(), is("hnsw")); + assertThat(defaultVectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS)); + assertThat( + defaultVectorType.getConfidenceInterval(), + is(ScalarQuantizedDenseVectorField.DEFAULT_CONFIDENCE_INTERVAL)); + assertThat(defaultVectorType.useCompression(), is(false)); + } finally { + deleteCore(); + } + } + + @Test + public void fieldDefinition_halfByteSize_shouldLoadSchemaField() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); + + IndexSchema schema = h.getCore().getLatestSchema(); + + SchemaField vectorField = schema.getField("v_scalar_half_byte"); + assertNotNull(vectorField); + + ScalarQuantizedDenseVectorField vectorType = + (ScalarQuantizedDenseVectorField) vectorField.getType(); + assertThat(vectorType.getBits(), is(4)); + } finally { + deleteCore(); + } + } + + @Test + public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); + + IndexSchema schema = h.getCore().getLatestSchema(); + + SchemaField vectorField = schema.getField("v_scalar_compressed"); + assertNotNull(vectorField); + + ScalarQuantizedDenseVectorField vectorType = + (ScalarQuantizedDenseVectorField) vectorField.getType(); + assertThat(vectorType.getBits(), is(4)); + assertThat(vectorType.useCompression(), is(true)); + } finally { + deleteCore(); + } + } + + @Test + public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); + + IndexSchema schema = h.getCore().getLatestSchema(); + + SchemaField vectorField = schema.getField("v_scalar_confidence"); + assertNotNull(vectorField); + + ScalarQuantizedDenseVectorField vectorType = + (ScalarQuantizedDenseVectorField) vectorField.getType(); + assertThat(vectorType.getConfidenceInterval(), is(0.91F)); + } finally { + deleteCore(); + } + } + + @Test + public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); + + IndexSchema schema = h.getCore().getLatestSchema(); + + SchemaField vectorField = schema.getField("v_scalar_dynamic"); + assertNotNull(vectorField); + + ScalarQuantizedDenseVectorField vectorType = + (ScalarQuantizedDenseVectorField) vectorField.getType(); + assertThat( + vectorType.getConfidenceInterval(), + is(Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL)); + } finally { + deleteCore(); + } + } +} diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc index 27547b55532..edc25b39ab9 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc @@ -284,6 +284,82 @@ client.add(Arrays.asList(d1, d2)); ==== ====== +=== ScalarQuantizedDenseVectorField +Because dense vectors can have a costly size, it may be worthwhile to use a technique called "quantization" +which creates a compressed representation of the original vectors. This allows more of the index to be stored in faster memory +at the cost of some precision. + +This dense vector type uses a conversion that projects a 32 bit float precision feature down to an 8 bit int (or smaller) +by linearly mapping the float range of each dimension down to evenly sized "buckets" of values that fit into an int. For example: +with 8 bits we can store up to 256 discrete values, so a float dimension with values from 0.0 to 1.0 may be mapped as + +[0.0, 0.0039) => 0, [0.0039, 0.0078) => 1 ... etc + +As a specific type of DenseVectorField, this field type supports all the same configurable properties outlined above as well +as some additional ones. + +Here is how a ScalarQuantizedDenseVectorField can be defined in the schema: + +[source,xml] +<fieldType name="scalar_quantized_vector" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine"/> +<field name="vector" type="scalar_quantized_vector" indexed="true" stored="true"/> + +`bits`:: ++ +[%autowidth,frame=none] +|=== +s|Optional |Default: `7` +|=== ++ +The number of bits to use for each quantized dimension value ++ +Accepted values: 4 (half byte) or 7 (unsigned byte). + +`confidenceInterval`:: ++ +[%autowidth,frame=none] +|=== +s|Optional |Default: `dimension-scaled` +|=== ++ +Statistically, outlier values are rarely meaningfully relevant to searches, so to increase the size of each bucket for +quantization (and therefore information gain) we can scale the quantization intervals to the middle n % of values and place the remaining +outliers in the outermost intervals. ++ +For example: 0.9 means scale interval sizes to the middle 90% of values ++ +If this param is omitted a default is used; scaled to the number of dimensions according to `1-1/(vector_dimensions + 1)` ++ +Accepted values: `FLOAT32` (within 0.9 and 1.0) + +`dynamicConfidenceInterval`:: ++ +[%autowidth,frame=none] +|=== +s|Optional |Default: `false` +|=== ++ +If set to true, enables dynamically determining confidence interval (per dimension) by sampling values each time a merge occurs. ++ +`NOTE: when this is enabled, it will take precedence over any value configured for confidenceInterval` ++ +Accepted values: `BOOLEAN` + +`compress`:: ++ +[%autowidth,frame=none] +|=== +s|Optional |Default: `false` +|=== ++ +If set to true, this will further pack multiple dimension values within a one byte alignment. This further decreases the +quantized vector disk storage size by 50% at some decode penalty. This does not affect the raw vector which is always +preserved when `stored` is true. ++ +`NOTE: this can only be enabled when bits=4` ++ +Accepted values: `BOOLEAN` + == Query Time Apache Solr provides three query parsers that work with dense vector fields, that each support different ways of matching documents based on vector similarity: The `knn` query parser, the `vectorSimilarity` query parser and the `knn_text_to_vector` query parser.
