sonatype-lift[bot] commented on a change in pull request #629: URL: https://github.com/apache/lucene/pull/629#discussion_r794049842
########## File path: lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91Codec.java ########## @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene91; + +import java.util.Objects; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; +import org.apache.lucene.codecs.lucene90.Lucene90FieldInfosFormat; +import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90SegmentInfoFormat; +import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; + +/** + * Implements the Lucene 9.1 index format + * + * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}. + * + * @see org.apache.lucene.codecs.lucene91 package documentation for file format details. + * @lucene.experimental Review comment: *InvalidBlockTag:* Tag name `lucene.experimental` is unknown. If this is a commonly-used custom tag, please click 'not useful' and file a bug. [(details)](https://errorprone.info/bugpattern/InvalidBlockTag) (at-me [in a reply](https://help.sonatype.com/lift/talking-to-lift) with `help` or `ignore`) ########## File path: lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsFormat.java ########## @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.codecs.lucene91; + +import java.io.IOException; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.hnsw.HnswGraph; + +/** + * Lucene 9.0 vector format, which encodes numeric vector values and an optional associated graph + * connecting the documents having values. The graph is used to power HNSW search. The format + * consists of three files: + * + * <h2>.vec (vector data) file</h2> + * + * <p>This file stores all the floating-point vector data ordered by field, document ordinal, and + * vector dimension. The floats are stored in little-endian byte order. + * + * <h2>.vex (vector index)</h2> + * + * <p>Stores graphs connecting the documents for each field organized as a list of nodes' neighbours + * as following: + * + * <ul> + * <li>For each level: + * <ul> + * <li>For each node: + * <ul> + * <li><b>[int32]</b> the number of neighbor nodes + * <li><b>array[int32]</b> the neighbor ordinals + * <li><b>array[int32]</b> padding from empty integers if the number of neighbors less + * than the maximum number of connections (maxConn). Padding is equal to + * ((maxConn-the number of neighbours) * 4) bytes. + * </ul> + * </ul> + * </ul> + * + * <h2>.vem (vector metadata) file</h2> + * + * <p>For each field: + * + * <ul> + * <li><b>[int32]</b> field number + * <li><b>[int32]</b> vector similarity function ordinal + * <li><b>[vlong]</b> offset to this field's vectors in the .vec file + * <li><b>[vlong]</b> length of this field's vectors, in bytes + * <li><b>[vlong]</b> offset to this field's index in the .vex file + * <li><b>[vlong]</b> length of this field's index data, in bytes + * <li><b>[int]</b> dimension of this field's vectors + * <li><b>[int]</b> the number of documents having values for this field + * <li><b>array[vint]</b> the docids of documents having vectors, in order + * <li><b>[int]</b> the maximum number of connections (neigbours) that each node can have + * <li><b>[int]</b> number of levels in the graph + * <li>Graph nodes by level. For each level + * <ul> + * <li><b>[int]</b> the number of nodes on this level + * <li><b>array[vint]</b> for levels greater than 0 list of nodes on this level, stored as + * the the level 0th nodes ordinals. + * </ul> + * </ul> + * + * @lucene.experimental Review comment: *InvalidBlockTag:* Tag name `lucene.experimental` is unknown. If this is a commonly-used custom tag, please click 'not useful' and file a bug. [(details)](https://errorprone.info/bugpattern/InvalidBlockTag) (at-me [in a reply](https://help.sonatype.com/lift/talking-to-lift) with `help` or `ignore`) ########## File path: lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsReader.java ########## @@ -0,0 +1,554 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.codecs.lucene91; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.KnnGraphValues; +import org.apache.lucene.index.RandomAccessVectorValues; +import org.apache.lucene.index.RandomAccessVectorValuesProducer; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.index.VectorValues; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.hnsw.HnswGraph; +import org.apache.lucene.util.hnsw.NeighborQueue; + +/** + * Reads vectors from the index segments along with index data structures supporting KNN search. + * + * @lucene.experimental Review comment: *InvalidBlockTag:* Tag name `lucene.experimental` is unknown. If this is a commonly-used custom tag, please click 'not useful' and file a bug. [(details)](https://errorprone.info/bugpattern/InvalidBlockTag) (at-me [in a reply](https://help.sonatype.com/lift/talking-to-lift) with `help` or `ignore`) ########## File path: lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsWriter.java ########## @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.codecs.lucene91; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +import java.io.IOException; +import java.util.Arrays; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.KnnGraphValues.NodesIterator; +import org.apache.lucene.index.RandomAccessVectorValuesProducer; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.index.VectorValues; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.hnsw.HnswGraph; +import org.apache.lucene.util.hnsw.HnswGraphBuilder; +import org.apache.lucene.util.hnsw.NeighborArray; + +/** + * Writes vector values and knn graphs to index segments. + * + * @lucene.experimental Review comment: *InvalidBlockTag:* Tag name `lucene.experimental` is unknown. If this is a commonly-used custom tag, please click 'not useful' and file a bug. [(details)](https://errorprone.info/bugpattern/InvalidBlockTag) (at-me [in a reply](https://help.sonatype.com/lift/talking-to-lift) with `help` or `ignore`) ########## File path: lucene/core/src/java/org/apache/lucene/util/hnsw/BoundsChecker.java ########## @@ -17,62 +17,74 @@ package org.apache.lucene.util.hnsw; -abstract class BoundsChecker { +/** + * A helper class for an hnsw graph that serves as a comparator of the currently set bound value + * with a new value. + */ +public abstract class BoundsChecker { float bound; /** Update the bound if sample is better */ - abstract void update(float sample); + public abstract void update(float sample); /** Update the bound unconditionally */ - void set(float sample) { + public void set(float sample) { bound = sample; } /** @return whether the sample exceeds (is worse than) the bound */ Review comment: *MissingSummary:* A summary fragment is required; consider using the value of the @return block as a summary fragment instead. [(details)](https://google.github.io/styleguide/javaguide.html#s7.2-summary-fragment) (at-me [in a reply](https://help.sonatype.com/lift/talking-to-lift) with `help` or `ignore`) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For additional commands, e-mail: issues-h...@lucene.apache.org
