[GitHub] [lucene-solr] mikemccand commented on a change in pull request #2022: LUCENE-9004: KNN vector search using NSW graphs

[GitBox]

mikemccand commented on a change in pull request #2022:
URL: https://github.com/apache/lucene-solr/pull/2022#discussion_r519895947



##########
File path: 
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90VectorReader.java
##########
@@ -22,39 +22,60 @@
 import java.nio.FloatBuffer;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Random;
 
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.VectorReader;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.KnnGraphValues;
+import org.apache.lucene.index.RandomAccessVectorValues;
+import org.apache.lucene.index.RandomAccessVectorValuesProducer;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.VectorValues;
+import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.hnsw.HnswGraph;
+import org.apache.lucene.util.hnsw.Neighbor;
+import org.apache.lucene.util.hnsw.Neighbors;
+
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 /**
- * Reads vectors from the index segments.
+ * Reads vectors from the index segments along with index data structures 
supporting KNN search.
  * @lucene.experimental
  */
 public final class Lucene90VectorReader extends VectorReader {
 
   private final FieldInfos fieldInfos;
   private final Map<String, FieldEntry> fields = new HashMap<>();
   private final IndexInput vectorData;
-  private final int maxDoc;
+  private final IndexInput vectorIndex;
+  private final long checksumSeed;
 
   Lucene90VectorReader(SegmentReadState state) throws IOException {
     this.fieldInfos = state.fieldInfos;
-    this.maxDoc = state.segmentInfo.maxDoc();
 
-    String metaFileName = 
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, 
Lucene90VectorFormat.META_EXTENSION);
+    int versionMeta = readMetadata(state, Lucene90VectorFormat.META_EXTENSION);
+    long[] checksumRef = new long[1];
+    vectorData = openDataInput(state, versionMeta, 
Lucene90VectorFormat.VECTOR_DATA_EXTENSION, 
Lucene90VectorFormat.VECTOR_DATA_CODEC_NAME, checksumRef);
+    vectorIndex = openDataInput(state, versionMeta, 
Lucene90VectorFormat.VECTOR_INDEX_EXTENSION, 
Lucene90VectorFormat.VECTOR_INDEX_CODEC_NAME, checksumRef);
+    checksumSeed = checksumRef[0];
+  }
+
+  private int readMetadata(SegmentReadState state, String fileExtension) 
throws IOException {
+    String metaFileName = 
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, 
fileExtension);
     int versionMeta = -1;
+    long checksum = -1;

Review comment:
       Hmm is this unused?

##########
File path: 
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90VectorReader.java
##########
@@ -277,24 +351,46 @@ public long cost() {
     }
 
     @Override
-    public RandomAccess randomAccess() {
+    public RandomAccessVectorValues randomAccess() {
       return new OffHeapRandomAccess(dataIn.clone());
     }
 
+    @Override
+    public TopDocs search(float[] vector, int topK, int fanout) throws 
IOException {
+      // use a seed that is fixed for the index so we get reproducible results 
for the same query
+      final Random random = new Random(checksumSeed);

Review comment:
       Clever seed!




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org