mayya-sharipova commented on code in PR #992:
URL: https://github.com/apache/lucene/pull/992#discussion_r919343914
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene93/Lucene93HnswVectorsWriter.java:
##########
@@ -266,65 +470,128 @@ private void writeMeta(
}
}
- private OnHeapHnswGraph writeGraph(
- RandomAccessVectorValuesProducer vectorValues, VectorSimilarityFunction
similarityFunction)
+ /**
+ * Writes the vector values to the output and returns a set of documents
that contains vectors.
+ */
+ private static DocsWithFieldSet writeVectorData(IndexOutput output,
VectorValues vectors)
throws IOException {
+ DocsWithFieldSet docsWithField = new DocsWithFieldSet();
+ for (int docV = vectors.nextDoc(); docV != NO_MORE_DOCS; docV =
vectors.nextDoc()) {
+ // write vector
+ BytesRef binaryValue = vectors.binaryValue();
+ assert binaryValue.length == vectors.dimension() * Float.BYTES;
+ output.writeBytes(binaryValue.bytes, binaryValue.offset,
binaryValue.length);
+ docsWithField.add(docV);
+ }
+ return docsWithField;
+ }
- // build graph
- HnswGraphBuilder hnswGraphBuilder =
- new HnswGraphBuilder(
- vectorValues, similarityFunction, M, beamWidth,
HnswGraphBuilder.randSeed);
- hnswGraphBuilder.setInfoStream(segmentWriteState.infoStream);
- OnHeapHnswGraph graph =
hnswGraphBuilder.build(vectorValues.randomAccess());
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(meta, vectorData, vectorIndex);
+ }
- // write vectors' neighbours on each level into the vectorIndex file
- int countOnLevel0 = graph.size();
- for (int level = 0; level < graph.numLevels(); level++) {
- int maxConnOnLevel = level == 0 ? (M * 2) : M;
- NodesIterator nodesOnLevel = graph.getNodesOnLevel(level);
- while (nodesOnLevel.hasNext()) {
- int node = nodesOnLevel.nextInt();
- NeighborArray neighbors = graph.getNeighbors(level, node);
- int size = neighbors.size();
- vectorIndex.writeInt(size);
- // Destructively modify; it's ok we are discarding it after this
- int[] nnodes = neighbors.node();
- Arrays.sort(nnodes, 0, size);
- for (int i = 0; i < size; i++) {
- int nnode = nnodes[i];
- assert nnode < countOnLevel0 : "node too large: " + nnode + ">=" +
countOnLevel0;
- vectorIndex.writeInt(nnode);
- }
- // if number of connections < maxConn, add bogus values up to maxConn
to have predictable
- // offsets
- for (int i = size; i < maxConnOnLevel; i++) {
- vectorIndex.writeInt(0);
- }
+ private static class FieldData extends KnnFieldVectorsWriter {
Review Comment:
Nice suggestion! Addressed in b47ddc9e6834e2b0a838adf7fc1bed791b24ce2e
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
