john-mlika commented on code in PR #16481: URL: https://github.com/apache/lucene/pull/16481#discussion_r3717166231
########## lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestMergeReadAdviceRevert.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene99; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.DataAccessHint; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.FilterIndexInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; + +public class TestMergeReadAdviceRevert extends LuceneTestCase { + + private static final int DIM = 16; + + public void testSequentialAdviceIsRevertedAfterMerge() throws Exception { + Recorder recorder = new Recorder(); + try (Directory raw = new ByteBuffersDirectory(); + Directory dir = new RecordingDirectory(raw, recorder)) { + + IndexWriterConfig iwc = new IndexWriterConfig(); + iwc.setCodec(hnswFloatCodec()); + // Expose .vec files to RecordingDirectory. + iwc.setUseCompoundFile(false); + iwc.setMergePolicy(new TieredMergePolicy()); + + try (IndexWriter w = new IndexWriter(dir, iwc)) { + for (int seg = 0; seg < 2; seg++) { + for (int i = 0; i < 64; i++) { + Document doc = new Document(); + float[] v = new float[DIM]; + for (int d = 0; d < DIM; d++) { + v[d] = random().nextFloat(); + } + doc.add(new KnnFloatVectorField("field", v, VectorSimilarityFunction.DOT_PRODUCT)); + w.addDocument(doc); + } + w.commit(); + } + + // Keep the source SegmentReaders open so the merge reuses their vector inputs. + try (DirectoryReader nrt = DirectoryReader.open(w)) { + assertEquals(2, nrt.leaves().size()); + recorder.mark("--- forceMerge(1) start ---"); + w.forceMerge(1); + recorder.mark("--- forceMerge(1) done ---"); + + // Check before the source SegmentReaders are closed. + List<String> offenders = new ArrayList<>(); + List<String> sawSequential = new ArrayList<>(); + for (Map.Entry<String, List<String>> e : recorder.snapshot().entrySet()) { + String file = e.getKey(); + if (file.endsWith(".vec") == false) { + continue; + } + List<String> events = e.getValue(); + String lastAdvice = null; + for (String ev : events) { + if (ev.startsWith("ADVICE:")) { + lastAdvice = ev.substring("ADVICE:".length()); + } + } + if (events.contains("ADVICE:SEQUENTIAL")) { + sawSequential.add(file); + if ("SEQUENTIAL".equals(lastAdvice)) { + offenders.add(file + " " + events); + } + } + } + + assertFalse( + "no .vec input received SEQUENTIAL advice:\n" + recorder.dump(), + sawSequential.isEmpty()); + + assertTrue( + ".vec inputs still using SEQUENTIAL advice:\n " + + String.join("\n ", offenders) + + "\n\nEvents:\n" + + recorder.dump(), + offenders.isEmpty()); + } + } + } + } + + private static Codec hnswFloatCodec() { + Codec def = TestUtil.getDefaultCodec(); + final KnnVectorsFormat perField = + new PerFieldKnnVectorsFormat() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new Lucene99HnswVectorsFormat(); + } + }; + return new FilterCodec(def.getName(), def) { + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perField; + } + }; + } + + static final class Recorder { + private final Map<String, List<String>> events = new LinkedHashMap<>(); + private final List<String> timeline = new ArrayList<>(); + + synchronized void record(String file, String event) { + events.computeIfAbsent(file, k -> new ArrayList<>()).add(event); + timeline.add(file + " -> " + event); + } + + synchronized void mark(String note) { + timeline.add(note); + } + + synchronized Map<String, List<String>> snapshot() { + Map<String, List<String>> copy = new LinkedHashMap<>(); + for (Map.Entry<String, List<String>> e : events.entrySet()) { Review Comment: updated to return events directly and dropped the copy -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
