Unrelated to my previous mail to the list, but related to the same investigation...
The following test program just indexes a phrase of nonsense words using and then queries for one of the words using the same analyser. The same analyser is being used both for indexing and for querying, yet in the latter case, no search results come back. I added in debugging to print out the TermPositions for all terms and also the parsed query - lo and behold, two of the terms have disappeared on the way into the index. Where do the other two terms go? TX (PS doesn't this also mean that it's impossible to use the same analyser for both indexing and query? At least in some cases.) import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.ja.JapaneseAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermEnum; import org.apache.lucene.index.TermPositions; import org.apache.lucene.queryParser.standard.StandardQueryParser; import org.apache.lucene.queryParser.standard.config.StandardQueryConfigHandler; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Version; import org.junit.Test; import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; public class TestJapaneseAnalysis { @Test public void testJapaneseAnalysis() throws Exception { try (Directory directory = new RAMDirectory()) { Analyzer analyser = new JapaneseAnalyzer(Version.LUCENE_36); try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_36, analyser))) { Document document = new Document(); document.add(new Field("content", "blah blah commercial blah blah \u79CB\u8449\u539F blah blah", Field.Store.NO, Field.Index.ANALYZED)); writer.addDocument(document); } try (IndexReader reader = IndexReader.open(directory); TermEnum terms = reader.terms(new Term("content", "")); TermPositions termPositions = reader.termPositions()) { do { Term term = terms.term(); if (term.field() != "content") { break; } System.out.println(term); termPositions.seek(terms); while (termPositions.next()) { System.out.println(" " + termPositions.doc()); int freq = termPositions.freq(); for (int i = 0; i < freq; i++) { System.out.println(" " + termPositions.nextPosition()); } } } while (terms.next()); StandardQueryParser queryParser = new StandardQueryParser(analyser); queryParser.setDefaultOperator(StandardQueryConfigHandler.Operator.AND); // quoted to work around strange behaviour of StandardQueryParser treating this as a boolean query. Query query = queryParser.parse("\"\u79CB\u8449\u539F\"", "content"); System.out.println(query); TopDocs topDocs = new IndexSearcher(reader).search(query, 10); assertThat(topDocs.totalHits, is(1)); } } } } /* Output: content:blah 0 0 1 3 4 6 7 content:commercial 0 2 content:秋葉原 0 5 content:"(秋葉 秋葉原) 原" */ --------------------------------------------------------------------- To unsubscribe, e-mail: java-user-unsubscr...@lucene.apache.org For additional commands, e-mail: java-user-h...@lucene.apache.org