This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new bdb16c9706c SOLR-18362: replace org.apache.solr.spelling.Token with
SpellCheckToken (#4812)
bdb16c9706c is described below
commit bdb16c9706cf1cc94fba701c7e81aa89c19fd7a0
Author: Serhiy Bzhezytskyy <[email protected]>
AuthorDate: Fri Sep 11 04:12:54 2026 +0300
SOLR-18362: replace org.apache.solr.spelling.Token with SpellCheckToken
(#4812)
The spellchecker API no longer passes terms around as
org.apache.solr.spelling.Token, which is removed. Custom plugins in the
org.apache.solr.spelling package need updating: QueryConverter.convert(String)
now returns a Lucene TokenStream rather than a Collection<Token>,
SpellingOptions.tokens is a List<SpellCheckToken> rather than a
Collection<Token>, and SpellingResult is keyed by the new immutable record
SpellCheckToken wherever it previously took a Token. Behaviour of the shipped s
[...]
Co-authored-by: Eric Pugh <[email protected]>
---
.../SOLR-18362-remove-spelling-token.yml | 15 ++
.../handler/component/SpellCheckComponent.java | 65 +++-----
.../solr/spelling/AbstractLuceneSpellChecker.java | 8 +-
.../solr/spelling/ConjunctionSolrSpellChecker.java | 12 +-
.../solr/spelling/DirectSolrSpellChecker.java | 8 +-
.../apache/solr/spelling/PossibilityIterator.java | 7 +-
.../org/apache/solr/spelling/QueryConverter.java | 10 +-
.../solr/spelling/QueryWordsTokenStream.java | 149 ++++++++++++++++++
.../java/org/apache/solr/spelling/ResultEntry.java | 4 +-
.../org/apache/solr/spelling/SolrSpellChecker.java | 3 +-
.../apache/solr/spelling/SpellCheckCollator.java | 6 +-
.../apache/solr/spelling/SpellCheckCorrection.java | 6 +-
.../org/apache/solr/spelling/SpellCheckToken.java | 119 ++++++++++++++
.../org/apache/solr/spelling/SpellingOptions.java | 19 ++-
.../solr/spelling/SpellingQueryConverter.java | 78 +++-------
.../org/apache/solr/spelling/SpellingResult.java | 43 ++----
.../solr/spelling/SuggestQueryConverter.java | 18 +--
.../src/java/org/apache/solr/spelling/Token.java | 172 ---------------------
.../solr/spelling/WordBreakSolrSpellChecker.java | 22 +--
.../apache/solr/spelling/suggest/Suggester.java | 12 +-
.../solr/spelling/DirectSolrSpellCheckerTest.java | 25 +--
.../solr/spelling/FileBasedSpellCheckerTest.java | 36 +++--
.../solr/spelling/IndexBasedSpellCheckerTest.java | 58 ++++---
.../apache/solr/spelling/SimpleQueryConverter.java | 43 +-----
.../spelling/SpellPossibilityIteratorTest.java | 55 +++----
.../solr/spelling/SpellingQueryConverterTest.java | 138 +++++++++++------
.../spelling/TestSuggestSpellingConverter.java | 21 ++-
.../spelling/WordBreakSolrSpellCheckerTest.java | 12 +-
.../component/DummyCustomParamSpellChecker.java | 4 +-
29 files changed, 622 insertions(+), 546 deletions(-)
diff --git a/changelog/unreleased/SOLR-18362-remove-spelling-token.yml
b/changelog/unreleased/SOLR-18362-remove-spelling-token.yml
new file mode 100644
index 00000000000..468ed975392
--- /dev/null
+++ b/changelog/unreleased/SOLR-18362-remove-spelling-token.yml
@@ -0,0 +1,15 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: >
+ The spellchecker API no longer passes terms around as
`org.apache.solr.spelling.Token`, which is
+ removed. Custom plugins in the `org.apache.solr.spelling` package need
updating:
+ `QueryConverter.convert(String)` now returns a Lucene `TokenStream` rather
than a
+ `Collection<Token>`, `SpellingOptions.tokens` is a `List<SpellCheckToken>`
rather than a
+ `Collection<Token>`, and `SpellingResult` is keyed by the new immutable
record `SpellCheckToken`
+ wherever it previously took a `Token`. Behaviour of the shipped
spellcheckers and of the `/spell`
+ request handler is unchanged.
+type: removed
+authors:
+ - name: Serhiy Bzhezytskyy
+links:
+ - name: SOLR-18362
+ url: https://issues.apache.org/jira/browse/SOLR-18362
diff --git
a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
index 98af4ed46dc..0221f720f4d 100644
---
a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
+++
b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -32,12 +31,6 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.search.Query;
@@ -69,10 +62,10 @@ import org.apache.solr.spelling.QueryConverter;
import org.apache.solr.spelling.SolrSpellChecker;
import org.apache.solr.spelling.SpellCheckCollation;
import org.apache.solr.spelling.SpellCheckCollator;
+import org.apache.solr.spelling.SpellCheckToken;
import org.apache.solr.spelling.SpellingOptions;
import org.apache.solr.spelling.SpellingQueryConverter;
import org.apache.solr.spelling.SpellingResult;
-import org.apache.solr.spelling.Token;
import org.apache.solr.util.SolrResponseUtil;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
@@ -143,20 +136,23 @@ public class SpellCheckComponent extends SearchComponent
implements SolrCoreAwar
SolrSpellChecker spellChecker = getSpellChecker(params);
if (spellChecker != null) {
- Collection<Token> tokens;
String q = params.get(SPELLCHECK_Q);
+ // the query is analyzed once, here: every spellchecker reads every
token, and
+ // ConjunctionSolrSpellChecker hands the same options to each of its
children, so one list
+ // serves them all.
+ final List<SpellCheckToken> tokens;
if (q != null) {
// we have a spell check param, tokenize it with the query analyzer
applicable for this
// spellchecker
- tokens = getTokens(q, spellChecker.getQueryAnalyzer());
+ tokens = SpellCheckToken.drain(getTokens(q,
spellChecker.getQueryAnalyzer()));
} else {
q = rb.getQueryString();
if (q == null) {
q = params.get(CommonParams.Q);
}
- tokens = queryConverter.convert(q);
+ tokens = SpellCheckToken.drain(queryConverter.convert(q));
}
- if (tokens != null && tokens.isEmpty() == false) {
+ if (!tokens.isEmpty()) {
int count = params.getInt(SPELLCHECK_COUNT, 1);
boolean onlyMorePopular =
params.getBool(SPELLCHECK_ONLY_MORE_POPULAR,
DEFAULT_ONLY_MORE_POPULAR);
@@ -347,10 +343,10 @@ public class SpellCheckComponent extends SearchComponent
implements SolrCoreAwar
}
private void addOriginalTermsToResponse(
- NamedList<Object> response, Collection<Token> originalTerms) {
- List<String> originalTermStr = new ArrayList<String>();
- for (Token t : originalTerms) {
- originalTermStr.add(t.toString());
+ NamedList<Object> response, List<SpellCheckToken> originalTerms) {
+ List<String> originalTermStr = new ArrayList<>(originalTerms.size());
+ for (SpellCheckToken token : originalTerms) {
+ originalTermStr.add(token.text());
}
response.add("originalTerms", originalTermStr);
}
@@ -580,32 +576,9 @@ public class SpellCheckComponent extends SearchComponent
implements SolrCoreAwar
}
}
- private Collection<Token> getTokens(String q, Analyzer analyzer) throws
IOException {
- Collection<Token> result = new ArrayList<>();
+ private TokenStream getTokens(String q, Analyzer analyzer) {
assert analyzer != null;
- try (TokenStream ts = analyzer.tokenStream("", q)) {
- ts.reset();
- // TODO: support custom attributes
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);
- FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);
- PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
- PositionIncrementAttribute posIncAtt =
ts.addAttribute(PositionIncrementAttribute.class);
-
- while (ts.incrementToken()) {
- Token token = new Token();
- token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
- token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
- token.setType(typeAtt.type());
- token.setFlags(flagsAtt.getFlags());
- token.setPayload(payloadAtt.getPayload());
- token.setPositionIncrement(posIncAtt.getPositionIncrement());
- result.add(token);
- }
- ts.end();
- return result;
- }
+ return analyzer.tokenStream("", q);
}
protected SolrSpellChecker getSpellChecker(SolrParams params) {
@@ -658,13 +631,15 @@ public class SpellCheckComponent extends SearchComponent
implements SolrCoreAwar
String origQuery,
boolean extendedResults) {
NamedList<Object> result = new NamedList<>();
- Map<Token, LinkedHashMap<String, Integer>> suggestions =
spellingResult.getSuggestions();
+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions =
+ spellingResult.getSuggestions();
boolean hasFreqInfo = spellingResult.hasTokenFrequencyInfo();
boolean hasSuggestions = false;
boolean hasZeroFrequencyToken = false;
- for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry :
suggestions.entrySet()) {
- Token inputToken = entry.getKey();
- String tokenString = new String(inputToken.buffer(), 0,
inputToken.length());
+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :
+ suggestions.entrySet()) {
+ SpellCheckToken inputToken = entry.getKey();
+ String tokenString = inputToken.text();
Map<String, Integer> theSuggestions = new
LinkedHashMap<>(entry.getValue());
theSuggestions.keySet().removeIf(sug -> sug.equals(tokenString));
if (theSuggestions.size() > 0) {
diff --git
a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
index 25cb239b2e0..284df9a71e8 100644
---
a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
+++
b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
@@ -132,19 +132,19 @@ public abstract class AbstractLuceneSpellChecker extends
SolrSpellChecker {
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws
IOException {
- SpellingResult result = new SpellingResult(options.tokens);
+ SpellingResult result = new SpellingResult();
IndexReader reader = determineReader(options.reader);
Term term = field != null ? new Term(field, "") : null;
float theAccuracy =
(options.accuracy == Float.MIN_VALUE) ? spellChecker.getAccuracy() :
options.accuracy;
int count = Math.max(options.count,
AbstractLuceneSpellChecker.DEFAULT_SUGGESTION_COUNT);
- for (Token token : options.tokens) {
- if (token.length() == 0) {
+ for (SpellCheckToken token : options.tokens) {
+ String tokenText = token.text();
+ if (tokenText.isEmpty()) {
result.add(token, List.of());
continue;
}
- String tokenText = new String(token.buffer(), 0, token.length());
term = new Term(field, tokenText);
int docFreq = 0;
if (reader != null) {
diff --git
a/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java
b/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java
index dc67b47cb17..077464bd0aa 100644
---
a/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java
+++
b/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java
@@ -131,13 +131,14 @@ public class ConjunctionSolrSpellChecker extends
SolrSpellChecker {
// TODO: This just interleaves the results. In the future, we might want to
let users give each
// checker its own weight and use that in combination to score & frequency
to sort the results ?
private SpellingResult mergeCheckers(SpellingResult[] results, int numSug) {
- Map<Token, Integer> combinedTokenFrequency = new HashMap<>();
- Map<Token, List<LinkedHashMap<String, Integer>>> allSuggestions = new
LinkedHashMap<>();
+ Map<SpellCheckToken, Integer> combinedTokenFrequency = new HashMap<>();
+ Map<SpellCheckToken, List<LinkedHashMap<String, Integer>>> allSuggestions =
+ new LinkedHashMap<>();
for (SpellingResult result : results) {
if (result.getTokenFrequency() != null) {
combinedTokenFrequency.putAll(result.getTokenFrequency());
}
- for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry :
+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :
result.getSuggestions().entrySet()) {
List<LinkedHashMap<String, Integer>> allForThisToken =
allSuggestions.get(entry.getKey());
if (allForThisToken == null) {
@@ -148,8 +149,9 @@ public class ConjunctionSolrSpellChecker extends
SolrSpellChecker {
}
}
SpellingResult combinedResult = new SpellingResult();
- for (Map.Entry<Token, List<LinkedHashMap<String, Integer>>> entry :
allSuggestions.entrySet()) {
- Token original = entry.getKey();
+ for (Map.Entry<SpellCheckToken, List<LinkedHashMap<String, Integer>>>
entry :
+ allSuggestions.entrySet()) {
+ SpellCheckToken original = entry.getKey();
List<Iterator<Map.Entry<String, Integer>>> corrIters =
new ArrayList<>(entry.getValue().size());
for (LinkedHashMap<String, Integer> corrections : entry.getValue()) {
diff --git
a/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java
b/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java
index acbb8828316..09397ce39cf 100644
--- a/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java
+++ b/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java
@@ -183,18 +183,16 @@ public class DirectSolrSpellChecker extends
SolrSpellChecker {
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws
IOException {
- log.debug("getSuggestions: {}", options.tokens);
-
SpellingResult result = new SpellingResult();
float accuracy =
(options.accuracy == Float.MIN_VALUE) ? checker.getAccuracy() :
options.accuracy;
- for (Token token : options.tokens) {
- if (token.length() == 0) {
+ for (SpellCheckToken token : options.tokens) {
+ String tokenText = token.text();
+ if (tokenText.isEmpty()) {
result.add(token, List.of());
continue;
}
- String tokenText = token.toString();
Term term = new Term(field, tokenText);
int freq = options.reader.docFreq(term);
int count =
diff --git
a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java
b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java
index 821df88c4c9..2af1f069d0e 100644
--- a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java
+++ b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java
@@ -55,13 +55,14 @@ public class PossibilityIterator implements
Iterator<PossibilityIterator.RankedS
* Possible Correction".
*/
public PossibilityIterator(
- Map<Token, LinkedHashMap<String, Integer>> suggestions,
+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions,
int maximumRequiredSuggestions,
int maxEvaluations,
boolean overlap) {
this.suggestionsMayOverlap = overlap;
- for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry :
suggestions.entrySet()) {
- Token token = entry.getKey();
+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :
+ suggestions.entrySet()) {
+ SpellCheckToken token = entry.getKey();
if (entry.getValue().size() == 0) {
continue;
}
diff --git a/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java
b/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java
index 307e27471e9..55106c5c660 100644
--- a/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java
+++ b/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java
@@ -16,8 +16,8 @@
*/
package org.apache.solr.spelling;
-import java.util.Collection;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
/**
@@ -57,10 +57,12 @@ public abstract class QueryConverter implements
NamedListInitializedPlugin {
public static final int TERM_IN_BOOLEAN_QUERY_FLAG = 131072;
/**
- * Returns the Collection of {@link Token}s for the query. Offsets on the
Token should correspond
- * to the correct offset in the origQuery
+ * Returns a fresh {@link TokenStream} over the query's terms. Offsets
should correspond to the
+ * correct offset in the origQuery. The caller owns the returned stream's
lifecycle: {@link
+ * SpellCheckToken#drain(TokenStream)} resets it, reads it to the end and
closes it, and the
+ * spellcheckers read that list rather than this stream.
*/
- public abstract Collection<Token> convert(String original);
+ public abstract TokenStream convert(String original);
/** Set the analyzer to use. Must be set before any calls to convert. */
public void setAnalyzer(Analyzer analyzer) {
diff --git
a/solr/core/src/java/org/apache/solr/spelling/QueryWordsTokenStream.java
b/solr/core/src/java/org/apache/solr/spelling/QueryWordsTokenStream.java
new file mode 100644
index 00000000000..92524dba6ec
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/spelling/QueryWordsTokenStream.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.spelling;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.util.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Analyzes each parsed query word in turn and concatenates their analysis
into a single stream,
+ * shifting offsets to the original query string and setting {@link
FlagsAttribute} from the
+ * query-syntax parse ({@link QueryConverter#REQUIRED_TERM_FLAG} and friends).
Each word is only
+ * analyzed once this stream reaches it, not up front.
+ */
+final class QueryWordsTokenStream extends TokenStream {
+
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /** One query word as parsed from query syntax, prior to analysis. */
+ record ParsedWord(String text, int startIndex, int flags) {}
+
+ private final List<ParsedWord> words;
+ private final Analyzer analyzer;
+ private int nextWordIndex;
+ private int currentWordIndex;
+ private TokenStream current;
+ private CharTermAttribute currentTermAtt;
+ private OffsetAttribute currentOffsetAtt;
+ private TypeAttribute currentTypeAtt;
+ private PositionIncrementAttribute currentPosIncAtt;
+ private PayloadAttribute currentPayloadAtt;
+
+ private final CharTermAttribute termAtt =
addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt =
addAttribute(OffsetAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+ private final PositionIncrementAttribute posIncAtt =
+ addAttribute(PositionIncrementAttribute.class);
+ private final PayloadAttribute payloadAtt =
addAttribute(PayloadAttribute.class);
+ private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
+
+ QueryWordsTokenStream(List<ParsedWord> words, Analyzer analyzer) {
+ this.words = words;
+ this.analyzer = analyzer;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ while (true) {
+ // A word whose analysis fails is skipped and the query's other words
are still checked; an
+ // IOException escaping here would fail the whole spellcheck request
instead.
+ if (current != null) {
+ try {
+ if (current.incrementToken()) {
+ ParsedWord word = words.get(currentWordIndex);
+ clearAttributes();
+ termAtt.append(currentTermAtt);
+ offsetAtt.setOffset(
+ word.startIndex() + currentOffsetAtt.startOffset(),
+ word.startIndex() + currentOffsetAtt.endOffset());
+ typeAtt.setType(currentTypeAtt.type());
+
posIncAtt.setPositionIncrement(currentPosIncAtt.getPositionIncrement());
+ payloadAtt.setPayload(currentPayloadAtt.getPayload());
+ flagsAtt.setFlags(word.flags());
+ return true;
+ }
+ current.end();
+ } catch (IOException e) {
+ log.warn("Skipping the rest of query word '{}': its analysis
failed", currentWord(), e);
+ }
+ IOUtils.closeWhileHandlingException(current);
+ current = null;
+ }
+ if (nextWordIndex >= words.size()) {
+ return false;
+ }
+ currentWordIndex = nextWordIndex++;
+ try {
+ current = analyzer.tokenStream("", currentWord());
+ currentTermAtt = current.addAttribute(CharTermAttribute.class);
+ currentOffsetAtt = current.addAttribute(OffsetAttribute.class);
+ currentTypeAtt = current.addAttribute(TypeAttribute.class);
+ currentPosIncAtt =
current.addAttribute(PositionIncrementAttribute.class);
+ currentPayloadAtt = current.addAttribute(PayloadAttribute.class);
+ current.reset();
+ } catch (IOException e) {
+ log.warn("Skipping query word '{}': its analysis failed",
currentWord(), e);
+ IOUtils.closeWhileHandlingException(current);
+ current = null;
+ }
+ }
+ }
+
+ private String currentWord() {
+ return words.get(currentWordIndex).text();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ nextWordIndex = 0;
+ currentWordIndex = 0;
+ if (current != null) {
+ current.close();
+ current = null;
+ }
+ }
+
+ @Override
+ public void end() throws IOException {
+ super.end();
+ if (current != null) {
+ current.end();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (current != null) {
+ current.close();
+ current = null;
+ }
+ super.close();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java
b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java
index cadb3c76d6e..5f8ea9334ec 100644
--- a/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java
+++ b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java
@@ -19,11 +19,11 @@ package org.apache.solr.spelling;
import java.util.Objects;
public class ResultEntry {
- public Token token;
+ public SpellCheckToken token;
public String suggestion;
public int freq;
- ResultEntry(Token t, String s, int f) {
+ ResultEntry(SpellCheckToken t, String s, int f) {
token = t;
suggestion = s;
freq = f;
diff --git a/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java
b/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java
index f5a9989d9da..da0542243f6 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java
@@ -129,7 +129,8 @@ public abstract class SolrSpellChecker {
// create token
SpellCheckResponse.Suggestion suggestion =
mergeData.origVsSuggestion.get(original);
- Token token = new Token(original, suggestion.getStartOffset(),
suggestion.getEndOffset());
+ SpellCheckToken token =
+ new SpellCheckToken(original, suggestion.getStartOffset(),
suggestion.getEndOffset());
// get top 'count' suggestions out of 'sugQueue.size()' candidates
SuggestWord[] suggestions = new SuggestWord[Math.min(count,
sugQueue.size())];
diff --git
a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
index 344088c83ca..9faaa87a2d0 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
@@ -211,10 +211,10 @@ public class SpellCheckCollator {
String corr = "";
for (int i = 0; i < corrections.size(); i++) {
SpellCheckCorrection correction = corrections.get(i);
- Token tok = correction.getOriginal();
+ SpellCheckToken tok = correction.getOriginal();
// we are replacing the query in order, but injected terms might cause
// illegal offsets due to previous replacements.
- if (tok.getPositionIncrement() == 0) continue;
+ if (tok.positionIncrement() == 0) continue;
corr = correction.getCorrection();
boolean addParenthesis = false;
Character requiredOrProhibited = null;
@@ -233,7 +233,7 @@ public class SpellCheckCollator {
requiredOrProhibited = previousChar;
}
bump++;
- } else if ((tok.getFlags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)
+ } else if ((tok.flags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)
== QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG) {
addParenthesis = true;
corrSb.insert(indexOfSpace + bump, "AND ");
diff --git
a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java
b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java
index c040aca9b66..b8d465b5cbc 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java
@@ -17,12 +17,12 @@
package org.apache.solr.spelling;
public class SpellCheckCorrection {
- private Token original;
+ private SpellCheckToken original;
private String originalAsString = null;
private String correction;
private int numberOfOccurences;
- public Token getOriginal() {
+ public SpellCheckToken getOriginal() {
return original;
}
@@ -33,7 +33,7 @@ public class SpellCheckCorrection {
return originalAsString;
}
- public void setOriginal(Token original) {
+ public void setOriginal(SpellCheckToken original) {
this.original = original;
this.originalAsString = null;
}
diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckToken.java
b/solr/core/src/java/org/apache/solr/spelling/SpellCheckToken.java
new file mode 100644
index 00000000000..34f89a92768
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckToken.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.spelling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * One term occurrence carried through the spellchecker API, and the key type
of {@link
+ * SpellingResult}. Unlike the old {@code Token} it replaces, this is a plain,
immutable record --
+ * not a Lucene {@code AttributeImpl} subclass.
+ *
+ * <p>A value type is needed here, rather than reading terms off a stream,
because {@link
+ * SolrSpellChecker#mergeSuggestions} keys suggestions by (text, offset) pairs
deserialized from a
+ * remote shard's response, where there is no {@link
org.apache.lucene.analysis.TokenStream} to read
+ * from at all.
+ */
+public record SpellCheckToken(
+ String text,
+ int startOffset,
+ int endOffset,
+ String type,
+ int positionIncrement,
+ int flags,
+ BytesRef payload) {
+
+ public SpellCheckToken {
+ // PayloadAttribute hands out a buffer it refills for the next token, so a
reference to it is
+ // not a value. This record is a map key in SpellingResult and in
ConjunctionSolrSpellChecker,
+ // which need one whose bytes cannot change after it is stored.
+ payload = payload == null ? null : BytesRef.deepCopyOf(payload);
+ }
+
+ public SpellCheckToken(String text, int startOffset, int endOffset) {
+ this(text, startOffset, endOffset, "word", 1, 0, null);
+ }
+
+ /** The length of {@link #text()}, which the {@code Token} this replaced
also offered. */
+ public int length() {
+ return text.length();
+ }
+
+ @Override
+ public String toString() {
+ return text;
+ }
+
+ /**
+ * Reads {@code stream} to its end into a list, and closes it. This is the
single point where a
+ * {@link TokenStream} becomes values: every consumer of the spellcheck API
reads every token, so
+ * the stream's lifecycle need not reach any of them.
+ */
+ public static List<SpellCheckToken> drain(TokenStream stream) throws
IOException {
+ List<SpellCheckToken> tokens = new ArrayList<>();
+ try (stream) {
+ AttributeReader attrs = new AttributeReader(stream);
+ stream.reset();
+ while (stream.incrementToken()) {
+ tokens.add(attrs.current());
+ }
+ stream.end();
+ }
+ return tokens;
+ }
+
+ /** Registers the six attributes {@link #drain} reads, once for the whole
stream. */
+ private static final class AttributeReader {
+ private final CharTermAttribute termAtt;
+ private final OffsetAttribute offsetAtt;
+ private final TypeAttribute typeAtt;
+ private final PositionIncrementAttribute posIncAtt;
+ private final FlagsAttribute flagsAtt;
+ private final PayloadAttribute payloadAtt;
+
+ AttributeReader(TokenStream stream) {
+ termAtt = stream.addAttribute(CharTermAttribute.class);
+ offsetAtt = stream.addAttribute(OffsetAttribute.class);
+ typeAtt = stream.addAttribute(TypeAttribute.class);
+ posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);
+ flagsAtt = stream.addAttribute(FlagsAttribute.class);
+ payloadAtt = stream.addAttribute(PayloadAttribute.class);
+ }
+
+ /** Builds a {@link SpellCheckToken} from the stream's current position. */
+ SpellCheckToken current() {
+ return new SpellCheckToken(
+ termAtt.toString(),
+ offsetAtt.startOffset(),
+ offsetAtt.endOffset(),
+ typeAtt.type(),
+ posIncAtt.getPositionIncrement(),
+ flagsAtt.getFlags(),
+ payloadAtt.getPayload());
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java
b/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java
index 4975c1538ee..9f89c97a7f7 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java
@@ -16,7 +16,7 @@
*/
package org.apache.solr.spelling;
-import java.util.Collection;
+import java.util.List;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.spell.SuggestMode;
import org.apache.solr.common.params.SolrParams;
@@ -24,8 +24,11 @@ import org.apache.solr.common.params.SolrParams;
/** */
public class SpellingOptions {
- /** The tokens to spell check */
- public Collection<Token> tokens;
+ /**
+ * The terms to spell check, analyzed once. Several {@link
SolrSpellChecker}s may read them --
+ * e.g. via {@link ConjunctionSolrSpellChecker} -- and each reads this same
list.
+ */
+ public List<SpellCheckToken> tokens;
/** An optional {@link org.apache.lucene.index.IndexReader} */
public IndexReader reader;
@@ -52,24 +55,24 @@ public class SpellingOptions {
public SpellingOptions() {}
// A couple of convenience ones
- public SpellingOptions(Collection<Token> tokens, int count) {
+ public SpellingOptions(List<SpellCheckToken> tokens, int count) {
this.tokens = tokens;
this.count = count;
}
- public SpellingOptions(Collection<Token> tokens, IndexReader reader) {
+ public SpellingOptions(List<SpellCheckToken> tokens, IndexReader reader) {
this.tokens = tokens;
this.reader = reader;
}
- public SpellingOptions(Collection<Token> tokens, IndexReader reader, int
count) {
+ public SpellingOptions(List<SpellCheckToken> tokens, IndexReader reader, int
count) {
this.tokens = tokens;
this.reader = reader;
this.count = count;
}
public SpellingOptions(
- Collection<Token> tokens,
+ List<SpellCheckToken> tokens,
IndexReader reader,
int count,
SuggestMode suggestMode,
@@ -86,7 +89,7 @@ public class SpellingOptions {
}
public SpellingOptions(
- Collection<Token> tokens,
+ List<SpellCheckToken> tokens,
IndexReader reader,
int count,
int alternativeTermCount,
diff --git
a/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
b/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
index 6870459a377..9fda927588e 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
@@ -16,31 +16,24 @@
*/
package org.apache.solr.spelling;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
/**
- * Converts the query string to a Collection of Lucene tokens using a regular
expression. Boolean
- * operators AND, OR, NOT are skipped.
+ * Converts the query string to a TokenStream using a regular expression.
Boolean operators AND, OR,
+ * NOT are skipped.
*
* <p>Each term is checked to determine if it is optional, required or
prohibited. Required terms
- * output a {@link Token} with the {@link QueryConverter#REQUIRED_TERM_FLAG}
set. Prohibited terms
- * output a {@link Token} with the {@link QueryConverter#PROHIBITED_TERM_FLAG}
set. If the query
- * uses the plus (+) and minus (-) to denote required and prohibited, this
determination will be
- * accurate. In the case boolean AND/OR/NOTs are used, this converter makes an
uninformed guess as
- * to whether the term would likely behave as if it is Required or Prohibited
and sets the flags
- * accordingly. These flags are used downstream to generate collations for
{@link
- * WordBreakSolrSpellChecker}, in cases where an original term is split up
into multiple Tokens.
+ * output a token with the {@link QueryConverter#REQUIRED_TERM_FLAG} set.
Prohibited terms output a
+ * token with the {@link QueryConverter#PROHIBITED_TERM_FLAG} set. If the
query uses the plus (+)
+ * and minus (-) to denote required and prohibited, this determination will be
accurate. In the case
+ * boolean AND/OR/NOTs are used, this converter makes an uninformed guess as
to whether the term
+ * would likely behave as if it is Required or Prohibited and sets the flags
accordingly. These
+ * flags are used downstream to generate collations for {@link
WordBreakSolrSpellChecker}, in cases
+ * where an original term is split up into multiple tokens.
*
* @since solr 1.3
*/
@@ -95,20 +88,22 @@ public class SpellingQueryConverter extends QueryConverter {
protected Pattern QUERY_REGEX = Pattern.compile(PATTERN);
/**
- * Converts the original query string to a collection of Lucene Tokens.
+ * Parses the original query string into a {@link TokenStream}; each matched
query word is
+ * analyzed lazily as the stream is consumed.
*
* @param original the original query string
- * @return a Collection of Lucene Tokens
+ * @return a TokenStream over the query's terms, with {@code FlagsAttribute}
set per word from the
+ * query-syntax parse below
*/
@Override
- public Collection<Token> convert(String original) {
+ public TokenStream convert(String original) {
if (original == null) { // this can happen with q.alt = and no query
- return List.of();
+ return new QueryWordsTokenStream(List.of(), analyzer);
}
boolean mightContainRangeQuery =
(original.indexOf('[') != -1 || original.indexOf('{') != -1)
&& (original.indexOf(']') != -1 || original.indexOf('}') != -1);
- Collection<Token> result = new ArrayList<>();
+ List<QueryWordsTokenStream.ParsedWord> words = new ArrayList<>();
Matcher matcher = QUERY_REGEX.matcher(original);
String nextWord = null;
int nextStartIndex = 0;
@@ -161,42 +156,17 @@ public class SpellingQueryConverter extends
QueryConverter {
&& ("NOT".equals(nextWord))) {
flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;
}
- try {
- analyze(result, word, startIndex, flagValue);
- } catch (IOException e) {
- // TODO: shouldn't we log something?
- }
+ words.add(new QueryWordsTokenStream.ParsedWord(word, startIndex,
flagValue));
}
if (lastBooleanOp != null) {
- for (Token t : result) {
- int f = t.getFlags();
- t.setFlags(f |= QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG);
+ for (int i = 0; i < words.size(); i++) {
+ QueryWordsTokenStream.ParsedWord w = words.get(i);
+ words.set(
+ i,
+ new QueryWordsTokenStream.ParsedWord(
+ w.text(), w.startIndex(), w.flags() |
QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG));
}
}
- return result;
- }
-
- protected void analyze(Collection<Token> result, String text, int offset,
int flagsAttValue)
- throws IOException {
- TokenStream stream = analyzer.tokenStream("", text);
- // TODO: support custom attributes
- CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
- TypeAttribute typeAtt = stream.addAttribute(TypeAttribute.class);
- PayloadAttribute payloadAtt = stream.addAttribute(PayloadAttribute.class);
- PositionIncrementAttribute posIncAtt =
stream.addAttribute(PositionIncrementAttribute.class);
- OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
- stream.reset();
- while (stream.incrementToken()) {
- Token token = new Token();
- token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
- token.setOffset(offset + offsetAtt.startOffset(), offset +
offsetAtt.endOffset());
- token.setFlags(flagsAttValue); // overwriting any flags already set...
- token.setType(typeAtt.type());
- token.setPayload(payloadAtt.getPayload());
- token.setPositionIncrement(posIncAtt.getPositionIncrement());
- result.add(token);
- }
- stream.end();
- stream.close();
+ return new QueryWordsTokenStream(words, analyzer);
}
}
diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java
b/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java
index 8cf29aa148f..fdec078404a 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.spelling;
-import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -28,30 +27,25 @@ import java.util.Map;
* @since solr 1.3
*/
public class SpellingResult {
- private Collection<Token> tokens;
/**
* Key == token Value = Map -> key is the suggestion, value is the frequency
of the token in the
* collection
*/
- private Map<Token, LinkedHashMap<String, Integer>> suggestions = new
LinkedHashMap<>();
+ private Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions =
new LinkedHashMap<>();
- private Map<Token, Integer> tokenFrequency;
+ private Map<SpellCheckToken, Integer> tokenFrequency;
public static final int NO_FREQUENCY_INFO = -1;
public SpellingResult() {}
- public SpellingResult(Collection<Token> tokens) {
- this.tokens = tokens;
- }
-
/**
* Adds a whole bunch of suggestions, and does not worry about frequency.
*
* @param token The token to associate the suggestions with
* @param suggestions The suggestions
*/
- public void add(Token token, List<String> suggestions) {
+ public void add(SpellCheckToken token, List<String> suggestions) {
LinkedHashMap<String, Integer> map = this.suggestions.get(token);
if (map == null) {
map = new LinkedHashMap<>();
@@ -68,7 +62,7 @@ public class SpellingResult {
* @param token original token
* @param docFreq original token's document frequency
*/
- public void addFrequency(Token token, int docFreq) {
+ public void addFrequency(SpellCheckToken token, int docFreq) {
if (tokenFrequency == null) {
tokenFrequency = new LinkedHashMap<>();
}
@@ -78,11 +72,11 @@ public class SpellingResult {
/**
* Suggestions must be added with the best suggestion first. ORDER is
important.
*
- * @param token The {@link Token}
- * @param suggestion The suggestion for the Token
+ * @param token The {@link SpellCheckToken}
+ * @param suggestion The suggestion for the token
* @param docFreq The document frequency
*/
- public void add(Token token, String suggestion, int docFreq) {
+ public void add(SpellCheckToken token, String suggestion, int docFreq) {
LinkedHashMap<String, Integer> map = this.suggestions.get(token);
// Don't bother adding if we already have this token
if (map == null) {
@@ -95,13 +89,13 @@ public class SpellingResult {
/**
* Gets the suggestions for the given token.
*
- * @param token The {@link Token} to look up
+ * @param token The {@link SpellCheckToken} to look up
* @return A LinkedHashMap of the suggestions. Key is the suggestion, value
is the token frequency
* in the index, else {@link #NO_FREQUENCY_INFO}.
* <p>The suggestions are added in sorted order (i.e. best suggestion
first) then the iterator
* will return the suggestions in order
*/
- public LinkedHashMap<String, Integer> get(Token token) {
+ public LinkedHashMap<String, Integer> get(SpellCheckToken token) {
return suggestions.get(token);
}
@@ -111,7 +105,7 @@ public class SpellingResult {
* @param token The token
* @return The frequency or null
*/
- public Integer getTokenFrequency(Token token) {
+ public Integer getTokenFrequency(SpellCheckToken token) {
return tokenFrequency.get(token);
}
@@ -122,26 +116,15 @@ public class SpellingResult {
/**
* All the suggestions. The ordering of the inner LinkedHashMap is by best
suggestion first.
*
- * @return The Map of suggestions for each Token. Key is the token, value is
a LinkedHashMap whose
+ * @return The Map of suggestions for each token. Key is the token, value is
a LinkedHashMap whose
* key is the Suggestion and the value is the frequency or {@link
#NO_FREQUENCY_INFO} if
* frequency info is not available.
*/
- public Map<Token, LinkedHashMap<String, Integer>> getSuggestions() {
+ public Map<SpellCheckToken, LinkedHashMap<String, Integer>> getSuggestions()
{
return suggestions;
}
- public Map<Token, Integer> getTokenFrequency() {
+ public Map<SpellCheckToken, Integer> getTokenFrequency() {
return tokenFrequency;
}
-
- /**
- * @return The original tokens
- */
- public Collection<Token> getTokens() {
- return tokens;
- }
-
- public void setTokens(Collection<Token> tokens) {
- this.tokens = tokens;
- }
}
diff --git
a/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java
b/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java
index da0bcbf3a55..84f4d5d7541 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java
@@ -16,26 +16,18 @@
*/
package org.apache.solr.spelling;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
+import org.apache.lucene.analysis.TokenStream;
/** Passes the entire query string to the configured analyzer as-is. */
public class SuggestQueryConverter extends SpellingQueryConverter {
@Override
- public Collection<Token> convert(String original) {
+ public TokenStream convert(String original) {
if (original == null) { // this can happen with q.alt = and no query
- return List.of();
+ return new QueryWordsTokenStream(List.of(), analyzer);
}
-
- Collection<Token> result = new ArrayList<>();
- try {
- analyze(result, original, 0, 0);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return result;
+ return new QueryWordsTokenStream(
+ List.of(new QueryWordsTokenStream.ParsedWord(original, 0, 0)),
analyzer);
}
}
diff --git a/solr/core/src/java/org/apache/solr/spelling/Token.java
b/solr/core/src/java/org/apache/solr/spelling/Token.java
deleted file mode 100644
index c163784a8bd..00000000000
--- a/solr/core/src/java/org/apache/solr/spelling/Token.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.spelling;
-
-import java.util.Objects;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.util.AttributeImpl;
-import org.apache.lucene.util.AttributeReflector;
-import org.apache.lucene.util.BytesRef;
-
-/**
- * A Token is an occurrence of a term from the text of a field. It consists of
a term's text, the
- * start and end offset of the term in the text of the field, and a type
string.
- *
- * <p>The start and end offsets permit applications to re-associate a token
with its source text,
- * e.g., to display highlighted query terms in a document browser, or to show
matching text
- * fragments in a <a
href="http://en.wikipedia.org/wiki/Key_Word_in_Context";>KWIC</a> display, etc.
- *
- * <p>The type is a string, assigned by a lexical analyzer (a.k.a. tokenizer),
naming the lexical or
- * syntactic class that the token belongs to. For example an end of sentence
marker token might be
- * implemented with type "eos". The default token type is "word".
- *
- * <p>A Token can optionally have metadata (a.k.a. payload) in the form of a
variable length byte
- * array. Use {@link org.apache.lucene.index.PostingsEnum#getPayload()} to
retrieve the payloads
- * from the index.
- *
- * <p>A few things to note:
- *
- * <ul>
- * <li>clear() initializes all of the fields to default values. This was
changed in contrast to
- * Lucene 2.4, but should affect no one.
- * <li>Because <code>TokenStreams</code> can be chained, one cannot assume
that the <code>Token's
- * </code> current type is correct.
- * <li>The startOffset and endOffset represent the start and offset in the
source text, so be
- * careful in adjusting them.
- * <li>When caching a reusable token, clone it. When injecting a cached
token into a stream that
- * can be reset, clone it again.
- * </ul>
- */
-@Deprecated(since = "7.0")
-public class Token extends PackedTokenAttributeImpl implements FlagsAttribute,
PayloadAttribute {
-
- // TODO Refactor the spellchecker API to use TokenStreams properly, rather
than this hack
-
- private int flags;
- private BytesRef payload;
-
- /** Constructs a Token will null text. */
- public Token() {}
-
- /**
- * Constructs a Token with the given term text, start and end offsets. The
type defaults to
- * "word." <b>NOTE:</b> for better indexing speed you should instead use the
char[] termBuffer
- * methods to set the term text.
- *
- * @param text term text
- * @param start start offset in the source text
- * @param end end offset in the source text
- */
- public Token(CharSequence text, int start, int end) {
- append(text);
- setOffset(start, end);
- }
-
- /**
- * {@inheritDoc}
- *
- * @see FlagsAttribute
- */
- @Override
- public int getFlags() {
- return flags;
- }
-
- /**
- * {@inheritDoc}
- *
- * @see FlagsAttribute
- */
- @Override
- public void setFlags(int flags) {
- this.flags = flags;
- }
-
- /**
- * {@inheritDoc}
- *
- * @see PayloadAttribute
- */
- @Override
- public BytesRef getPayload() {
- return this.payload;
- }
-
- /**
- * {@inheritDoc}
- *
- * @see PayloadAttribute
- */
- @Override
- public void setPayload(BytesRef payload) {
- this.payload = payload;
- }
-
- /**
- * Resets the term text, payload, flags, positionIncrement, positionLength,
startOffset, endOffset
- * and token type to default.
- */
- @Override
- public void clear() {
- super.clear();
- flags = 0;
- payload = null;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) return true;
- if (!(obj instanceof Token other)) return false;
-
- return (flags == other.flags && (Objects.equals(payload, other.payload))
&& super.equals(obj));
- }
-
- @Override
- public int hashCode() {
- int code = super.hashCode();
- code = code * 31 + flags;
- if (payload != null) {
- code = code * 31 + payload.hashCode();
- }
- return code;
- }
-
- @Override
- public Token clone() {
- final Token t = (Token) super.clone();
- if (payload != null) {
- t.payload = BytesRef.deepCopyOf(payload);
- }
- return t;
- }
-
- @Override
- public void copyTo(AttributeImpl target) {
- super.copyTo(target);
- ((FlagsAttribute) target).setFlags(flags);
- ((PayloadAttribute) target).setPayload((payload == null) ? null :
BytesRef.deepCopyOf(payload));
- }
-
- @Override
- public void reflectWith(AttributeReflector reflector) {
- super.reflectWith(reflector);
- reflector.reflect(FlagsAttribute.class, "flags", flags);
- reflector.reflect(PayloadAttribute.class, "payload", payload);
- }
-}
diff --git
a/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java
b/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java
index 1095acc2bd4..8f573054e74 100644
--- a/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java
+++ b/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java
@@ -158,9 +158,11 @@ public class WordBreakSolrSpellChecker extends
SolrSpellChecker {
int numSuggestions = options.count;
StringBuilder sb = new StringBuilder();
- Token[] tokenArr = options.tokens.toArray(new Token[0]);
- List<Token> tokenArrWithSeparators = new ArrayList<>(options.tokens.size()
+ 2);
- List<Term> termArr = new ArrayList<>(options.tokens.size() + 2);
+ // an array because, unlike the other SolrSpellCheckers, this one reads
several positions at
+ // once to combine adjacent terms into one corrected phrase
+ SpellCheckToken[] tokenArr = options.tokens.toArray(new
SpellCheckToken[0]);
+ List<SpellCheckToken> tokenArrWithSeparators = new
ArrayList<>(tokenArr.length + 2);
+ List<Term> termArr = new ArrayList<>(tokenArr.length + 2);
List<ResultEntry> breakSuggestionList = new ArrayList<>();
List<ResultEntry> noBreakSuggestionList = new ArrayList<>();
boolean lastOneProhibited = false;
@@ -168,13 +170,13 @@ public class WordBreakSolrSpellChecker extends
SolrSpellChecker {
boolean lastOneprocedesNewBooleanOp = false;
for (int i = 0; i < tokenArr.length; i++) {
boolean prohibited =
- (tokenArr[i].getFlags() & QueryConverter.PROHIBITED_TERM_FLAG)
+ (tokenArr[i].flags() & QueryConverter.PROHIBITED_TERM_FLAG)
== QueryConverter.PROHIBITED_TERM_FLAG;
boolean required =
- (tokenArr[i].getFlags() & QueryConverter.REQUIRED_TERM_FLAG)
+ (tokenArr[i].flags() & QueryConverter.REQUIRED_TERM_FLAG)
== QueryConverter.REQUIRED_TERM_FLAG;
boolean procedesNewBooleanOp =
- (tokenArr[i].getFlags() &
QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)
+ (tokenArr[i].flags() &
QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)
== QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;
if (i > 0
&& (prohibited != lastOneProhibited
@@ -234,8 +236,8 @@ public class WordBreakSolrSpellChecker extends
SolrSpellChecker {
}
sb.append(tokenArrWithSeparators.get(i).toString());
}
- Token token =
- new Token(
+ SpellCheckToken token =
+ new SpellCheckToken(
sb.toString(),
tokenArrWithSeparators.get(firstTermIndex).startOffset(),
tokenArrWithSeparators.get(lastTermIndex).endOffset());
@@ -316,7 +318,7 @@ public class WordBreakSolrSpellChecker extends
SolrSpellChecker {
private void addToResult(
SpellingResult result,
- Token token,
+ SpellCheckToken token,
int tokenFrequency,
String suggestion,
int suggestionFrequency) {
@@ -329,7 +331,7 @@ public class WordBreakSolrSpellChecker extends
SolrSpellChecker {
}
}
- private int getCombineFrequency(IndexReader ir, Token token) throws
IOException {
+ private int getCombineFrequency(IndexReader ir, SpellCheckToken token)
throws IOException {
String[] words = spacePattern.split(token.toString());
int result = 0;
if (sortMethod ==
BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY) {
diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
index c2a5d374ba7..21ce05aed6b 100644
--- a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
+++ b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
@@ -35,15 +35,14 @@ import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester;
import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup;
-import org.apache.lucene.util.CharsRef;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CloseHook;
import org.apache.solr.core.SolrCore;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.spelling.SolrSpellChecker;
+import org.apache.solr.spelling.SpellCheckToken;
import org.apache.solr.spelling.SpellingOptions;
import org.apache.solr.spelling.SpellingResult;
-import org.apache.solr.spelling.Token;
import org.apache.solr.spelling.suggest.fst.FSTLookupFactory;
import org.apache.solr.spelling.suggest.tst.TSTLookupFactory;
import org.slf4j.Logger;
@@ -195,22 +194,17 @@ public class Suggester extends SolrSpellChecker {
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws
IOException {
- log.debug("getSuggestions: {}", options.tokens);
if (lookup == null) {
log.info("Lookup is null - invoke spellchecker.build first");
return EMPTY_RESULT;
}
SpellingResult res = new SpellingResult();
- CharsRef scratch = new CharsRef();
- for (Token t : options.tokens) {
- scratch.chars = t.buffer();
- scratch.offset = 0;
- scratch.length = t.length();
+ for (SpellCheckToken t : options.tokens) {
boolean onlyMorePopular =
(options.suggestMode == SuggestMode.SUGGEST_MORE_POPULAR)
&& !(lookup instanceof WFSTCompletionLookup)
&& !(lookup instanceof AnalyzingSuggester);
- List<LookupResult> suggestions = lookup.lookup(scratch, onlyMorePopular,
options.count);
+ List<LookupResult> suggestions = lookup.lookup(t.text(),
onlyMorePopular, options.count);
if (suggestions == null) {
continue;
}
diff --git
a/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java
b/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java
index 3a642df514f..0b17667ed01 100644
---
a/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java
+++
b/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java
@@ -16,9 +16,10 @@
*/
package org.apache.solr.spelling;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.SpellingParams;
@@ -51,6 +52,11 @@ public class DirectSolrSpellCheckerTest extends
SolrTestCaseJ4 {
queryConverter.init(new NamedList<>());
}
+ /** A stream that emits exactly one token whose term text is empty. */
+ private static TokenStream singleEmptyTermTokenStream() {
+ return new KeywordAnalyzer().tokenStream("", "");
+ }
+
@Test
public void test() throws Exception {
DirectSolrSpellChecker checker = new DirectSolrSpellChecker();
@@ -67,11 +73,11 @@ public class DirectSolrSpellCheckerTest extends
SolrTestCaseJ4 {
searcher -> {
// check that 'fob' is corrected to 'foo'
- Collection<Token> tokens = queryConverter.convert("fob");
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(queryConverter.convert("fob"));
SpellingOptions spellOpts = new SpellingOptions(tokens,
searcher.getIndexReader());
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
assertFalse("suggestions shouldn't be empty",
suggestions.isEmpty());
Map.Entry<String, Integer> entry =
suggestions.entrySet().iterator().next();
assertEquals("foo", entry.getKey());
@@ -81,18 +87,18 @@ public class DirectSolrSpellCheckerTest extends
SolrTestCaseJ4 {
(int) entry.getValue());
// check that 'super' is *not* corrected
- spellOpts.tokens = queryConverter.convert("super");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("super"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNotNull("suggestions shouldn't be null", suggestions);
assertTrue("suggestions should be empty", suggestions.isEmpty());
// Check empty token due to spellcheck.q = ""
- spellOpts.tokens = List.of(new Token("", 0, 0));
+ spellOpts.tokens =
SpellCheckToken.drain(singleEmptyTermTokenStream());
result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(new SpellCheckToken("", 0, 0));
assertNotNull("suggestions shouldn't be null", suggestions);
assertTrue("suggestions should be empty", suggestions.isEmpty());
return null;
@@ -143,11 +149,12 @@ public class DirectSolrSpellCheckerTest extends
SolrTestCaseJ4 {
h.getCore()
.withSearcher(
searcher -> {
- Collection<Token> tokens = queryConverter.convert("anothar");
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(queryConverter.convert("anothar"));
SpellingOptions spellOpts = new SpellingOptions(tokens,
searcher.getIndexReader());
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
assertNotNull("suggestions shouldn't be null", suggestions);
if (limitQueryLength) {
diff --git
a/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java
b/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java
index 5065e0c398a..5c90814ff9a 100644
--- a/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java
+++ b/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java
@@ -18,9 +18,10 @@ package org.apache.solr.spelling;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks;
import org.apache.solr.SolrTestCaseJ4;
@@ -59,6 +60,11 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
queryConverter = null;
}
+ /** A stream that emits exactly one token whose term text is empty. */
+ private static TokenStream singleEmptyTermTokenStream() {
+ return new KeywordAnalyzer().tokenStream("", "");
+ }
+
@Test
public void test() throws Exception {
FileBasedSpellChecker checker = new FileBasedSpellChecker();
@@ -79,11 +85,11 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
h.getCore()
.withSearcher(
searcher -> {
- Collection<Token> tokens = queryConverter.convert("fob");
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(queryConverter.convert("fob"));
SpellingOptions spellOpts = new SpellingOptions(tokens,
searcher.getIndexReader());
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
Map.Entry<String, Integer> entry =
suggestions.entrySet().iterator().next();
assertEquals(entry.getKey() + " is not equal to " + "foo",
"foo", entry.getKey());
assertEquals(
@@ -91,18 +97,18 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
SpellingResult.NO_FREQUENCY_INFO,
(int) entry.getValue());
- spellOpts.tokens = queryConverter.convert("super");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("super"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNotNull("suggestions shouldn't be null", suggestions);
assertTrue("suggestions should be empty", suggestions.isEmpty());
// Check empty token due to spellcheck.q = ""
- spellOpts.tokens = List.of(new Token("", 0, 0));
+ spellOpts.tokens =
SpellCheckToken.drain(singleEmptyTermTokenStream());
result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(new SpellCheckToken("", 0, 0));
assertNotNull("suggestions shouldn't be null", suggestions);
assertTrue("suggestions should be empty", suggestions.isEmpty());
return null;
@@ -127,7 +133,7 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
assertEquals(dictName + " is not equal to " + "external", "external",
dictName);
checker.build(core, null);
- Collection<Token> tokens = queryConverter.convert("Solar");
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(queryConverter.convert("Solar"));
h.getCore()
.withSearcher(
searcher -> {
@@ -135,7 +141,7 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
// should be lowercased, b/c we are using a lowercasing analyzer
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
assertEquals(
"suggestions Size: " + suggestions.size() + " is not: " + 1,
1,
@@ -148,10 +154,10 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
(int) entry.getValue());
// test something not in the spell checker
- spellOpts.tokens = queryConverter.convert("super");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("super"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNotNull("suggestions shouldn't be null", suggestions);
assertTrue("suggestions should be empty", suggestions.isEmpty());
return null;
@@ -180,12 +186,12 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
h.getCore()
.withSearcher(
searcher -> {
- Collection<Token> tokens = queryConverter.convert("solar");
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(queryConverter.convert("solar"));
SpellingOptions spellOpts = new SpellingOptions(tokens,
searcher.getIndexReader());
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
// should be lowercased, b/c we are using a lowercasing analyzer
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
assertEquals(
"suggestions Size: " + suggestions.size() + " is not: " + 1,
1,
@@ -197,10 +203,10 @@ public class FileBasedSpellCheckerTest extends
SolrTestCaseJ4 {
SpellingResult.NO_FREQUENCY_INFO,
(int) entry.getValue());
- spellOpts.tokens = queryConverter.convert("super");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("super"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result shouldn't be null", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNotNull("suggestions shouldn't be null", suggestions);
assertTrue("suggestions should be empty", suggestions.isEmpty());
return null;
diff --git
a/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java
b/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java
index 6ee537f814d..71bcf056be7 100644
---
a/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java
+++
b/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java
@@ -18,11 +18,12 @@ package org.apache.solr.spelling;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -126,12 +127,13 @@ public class IndexBasedSpellCheckerTest extends
SolrTestCaseJ4 {
checker.build(core, searcher);
IndexReader reader = searcher.getIndexReader();
- Collection<Token> tokens = queryConverter.convert("documemt");
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(queryConverter.convert("documemt"));
SpellingOptions spellOpts = new SpellingOptions(tokens, reader);
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
// should be lowercased, b/c we are using a lowercasing analyzer
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
assertNotNull("documemt is null and it shouldn't be",
suggestions);
assertEquals(
"documemt Size: " + suggestions.size() + " is not: " + 1, 1,
suggestions.size());
@@ -144,32 +146,32 @@ public class IndexBasedSpellCheckerTest extends
SolrTestCaseJ4 {
(int) entry.getValue());
// test something not in the spell checker
- spellOpts.tokens = queryConverter.convert("super");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("super"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertEquals("suggestions size should be 0", 0,
suggestions.size());
// test something that is spelled correctly
- spellOpts.tokens = queryConverter.convert("document");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("document"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNull("suggestions is null and it shouldn't be",
suggestions);
// Has multiple possibilities, but the exact exists, so that
should be returned
- spellOpts.tokens = queryConverter.convert("red");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("red"));
spellOpts.count = 2;
result = checker.getSuggestions(spellOpts);
assertNotNull(result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNull("suggestions is not null and it should be",
suggestions);
// Try out something which should have multiple suggestions
- spellOpts.tokens = queryConverter.convert("bug");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("bug"));
result = checker.getSuggestions(spellOpts);
assertNotNull(result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNotNull(suggestions);
assertEquals(
"suggestions Size: " + suggestions.size() + " is not: " + 2,
@@ -197,16 +199,21 @@ public class IndexBasedSpellCheckerTest extends
SolrTestCaseJ4 {
(int) entry.getValue());
// Check empty token due to spellcheck.q = ""
- spellOpts.tokens = List.of(new Token("", 0, 0));
+ spellOpts.tokens =
SpellCheckToken.drain(singleEmptyTermTokenStream());
result = checker.getSuggestions(spellOpts);
assertNotNull(result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(new SpellCheckToken("", 0, 0));
assertNotNull(suggestions);
assertTrue("suggestions should be empty", suggestions.isEmpty());
return null;
});
}
+ /** A stream that emits exactly one token whose term text is empty. */
+ private static TokenStream singleEmptyTermTokenStream() {
+ return new KeywordAnalyzer().tokenStream("", "");
+ }
+
@Test
public void testExtendedResults() throws Exception {
IndexBasedSpellChecker checker = new IndexBasedSpellChecker();
@@ -230,14 +237,15 @@ public class IndexBasedSpellCheckerTest extends
SolrTestCaseJ4 {
checker.build(core, searcher);
IndexReader reader = searcher.getIndexReader();
- Collection<Token> tokens = queryConverter.convert("documemt");
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(queryConverter.convert("documemt"));
SpellingOptions spellOpts =
new SpellingOptions(
tokens, reader, 1,
SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, true, 0.5f, null);
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
// should be lowercased, b/c we are using a lowercasing analyzer
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
assertNotNull("documemt is null and it shouldn't be",
suggestions);
assertEquals(
"documemt Size: " + suggestions.size() + " is not: " + 1, 1,
suggestions.size());
@@ -247,16 +255,16 @@ public class IndexBasedSpellCheckerTest extends
SolrTestCaseJ4 {
assertEquals(entry.getValue() + " does not equal: " + 2, 2,
(int) entry.getValue());
// test something not in the spell checker
- spellOpts.tokens = queryConverter.convert("super");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("super"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertEquals("suggestions size should be 0", 0,
suggestions.size());
- spellOpts.tokens = queryConverter.convert("document");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("document"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNull("suggestions is not null and it should be",
suggestions);
return null;
});
@@ -350,14 +358,14 @@ public class IndexBasedSpellCheckerTest extends
SolrTestCaseJ4 {
checker.build(core, searcher);
IndexReader reader = searcher.getIndexReader();
- Collection<Token> tokens = queryConverter.convert("flesh");
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(queryConverter.convert("flesh"));
SpellingOptions spellOpts =
new SpellingOptions(
tokens, reader, 1,
SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, true, 0.5f, null);
SpellingResult result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
// should be lowercased, b/c we are using a lowercasing analyzer
- Map<String, Integer> suggestions =
result.get(spellOpts.tokens.iterator().next());
+ Map<String, Integer> suggestions =
result.get(spellOpts.tokens.get(0));
assertNotNull("flesh is null and it shouldn't be", suggestions);
assertEquals(
"flesh Size: " + suggestions.size() + " is not: " + 1, 1,
suggestions.size());
@@ -366,16 +374,16 @@ public class IndexBasedSpellCheckerTest extends
SolrTestCaseJ4 {
assertEquals(entry.getValue() + " does not equal: " + 1, 1,
(int) entry.getValue());
// test something not in the spell checker
- spellOpts.tokens = queryConverter.convert("super");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("super"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertEquals("suggestions size should be 0", 0,
suggestions.size());
- spellOpts.tokens = queryConverter.convert("Caroline");
+ spellOpts.tokens =
SpellCheckToken.drain(queryConverter.convert("Caroline"));
result = checker.getSuggestions(spellOpts);
assertNotNull("result is null and it shouldn't be", result);
- suggestions = result.get(spellOpts.tokens.iterator().next());
+ suggestions = result.get(spellOpts.tokens.get(0));
assertNull("suggestions is not null and it should be",
suggestions);
return null;
});
diff --git
a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java
b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java
index 9cda54676b1..3342f6a5082 100644
--- a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java
+++ b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java
@@ -16,53 +16,18 @@
*/
package org.apache.solr.spelling;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
/**
* @since solr 1.3
*/
class SimpleQueryConverter extends SpellingQueryConverter {
- @Override
- public Collection<Token> convert(String origQuery) {
- Collection<Token> result = new HashSet<>();
-
- try (WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();
- TokenStream ts = analyzer.tokenStream("", origQuery)) {
- // TODO: support custom attributes
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);
- FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);
- PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
- PositionIncrementAttribute posIncAtt =
ts.addAttribute(PositionIncrementAttribute.class);
+ private static final WhitespaceAnalyzer ANALYZER = new WhitespaceAnalyzer();
- ts.reset();
-
- while (ts.incrementToken()) {
- Token tok = new Token();
- tok.copyBuffer(termAtt.buffer(), 0, termAtt.length());
- tok.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
- tok.setFlags(flagsAtt.getFlags());
- tok.setPayload(payloadAtt.getPayload());
- tok.setPositionIncrement(posIncAtt.getPositionIncrement());
- tok.setType(typeAtt.type());
- result.add(tok);
- }
- ts.end();
- return result;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ @Override
+ public TokenStream convert(String origQuery) {
+ return ANALYZER.tokenStream("", origQuery);
}
}
diff --git
a/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java
b/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java
index d402044b4ce..365744601c1 100644
---
a/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java
+++
b/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java
@@ -25,10 +25,10 @@ import org.junit.Before;
import org.junit.Test;
public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 {
- private static final Token TOKEN_AYE = new Token("AYE", 0, 3);
- private static final Token TOKEN_BEE = new Token("BEE", 4, 7);
- private static final Token TOKEN_AYE_BEE = new Token("AYE BEE", 0, 7);
- private static final Token TOKEN_CEE = new Token("CEE", 8, 11);
+ private static final SpellCheckToken TOKEN_AYE = new SpellCheckToken("AYE",
0, 3);
+ private static final SpellCheckToken TOKEN_BEE = new SpellCheckToken("BEE",
4, 7);
+ private static final SpellCheckToken TOKEN_AYE_BEE = new
SpellCheckToken("AYE BEE", 0, 7);
+ private static final SpellCheckToken TOKEN_CEE = new SpellCheckToken("CEE",
8, 11);
private LinkedHashMap<String, Integer> AYE;
private LinkedHashMap<String, Integer> BEE;
@@ -87,26 +87,26 @@ public class SpellPossibilityIteratorTest extends
SolrTestCaseJ4 {
@Test
public void testScalability() {
- Map<Token, LinkedHashMap<String, Integer>> lotsaSuggestions = new
LinkedHashMap<>();
+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> lotsaSuggestions =
new LinkedHashMap<>();
lotsaSuggestions.put(TOKEN_AYE, AYE);
lotsaSuggestions.put(TOKEN_BEE, BEE);
lotsaSuggestions.put(TOKEN_CEE, CEE);
- lotsaSuggestions.put(new Token("AYE1", 0, 3), AYE);
- lotsaSuggestions.put(new Token("BEE1", 4, 7), BEE);
- lotsaSuggestions.put(new Token("CEE1", 8, 11), CEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE1", 0, 3), AYE);
+ lotsaSuggestions.put(new SpellCheckToken("BEE1", 4, 7), BEE);
+ lotsaSuggestions.put(new SpellCheckToken("CEE1", 8, 11), CEE);
- lotsaSuggestions.put(new Token("AYE2", 0, 3), AYE);
- lotsaSuggestions.put(new Token("BEE2", 4, 7), BEE);
- lotsaSuggestions.put(new Token("CEE2", 8, 11), CEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE2", 0, 3), AYE);
+ lotsaSuggestions.put(new SpellCheckToken("BEE2", 4, 7), BEE);
+ lotsaSuggestions.put(new SpellCheckToken("CEE2", 8, 11), CEE);
- lotsaSuggestions.put(new Token("AYE3", 0, 3), AYE);
- lotsaSuggestions.put(new Token("BEE3", 4, 7), BEE);
- lotsaSuggestions.put(new Token("CEE3", 8, 11), CEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE3", 0, 3), AYE);
+ lotsaSuggestions.put(new SpellCheckToken("BEE3", 4, 7), BEE);
+ lotsaSuggestions.put(new SpellCheckToken("CEE3", 8, 11), CEE);
- lotsaSuggestions.put(new Token("AYE4", 0, 3), AYE);
- lotsaSuggestions.put(new Token("BEE4", 4, 7), BEE);
- lotsaSuggestions.put(new Token("CEE4", 8, 11), CEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE4", 0, 3), AYE);
+ lotsaSuggestions.put(new SpellCheckToken("BEE4", 4, 7), BEE);
+ lotsaSuggestions.put(new SpellCheckToken("CEE4", 8, 11), CEE);
PossibilityIterator iter = new PossibilityIterator(lotsaSuggestions, 1000,
10000, false);
int count = 0;
@@ -116,10 +116,10 @@ public class SpellPossibilityIteratorTest extends
SolrTestCaseJ4 {
}
assertEquals(1000, count);
- lotsaSuggestions.put(new Token("AYE_BEE1", 0, 7), AYE_BEE);
- lotsaSuggestions.put(new Token("AYE_BEE2", 0, 7), AYE_BEE);
- lotsaSuggestions.put(new Token("AYE_BEE3", 0, 7), AYE_BEE);
- lotsaSuggestions.put(new Token("AYE_BEE4", 0, 7), AYE_BEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE1", 0, 7), AYE_BEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE2", 0, 7), AYE_BEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE3", 0, 7), AYE_BEE);
+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE4", 0, 7), AYE_BEE);
iter = new PossibilityIterator(lotsaSuggestions, 1000, 10000, true);
count = 0;
while (iter.hasNext()) {
@@ -131,7 +131,7 @@ public class SpellPossibilityIteratorTest extends
SolrTestCaseJ4 {
@Test
public void testSpellPossibilityIterator() {
- Map<Token, LinkedHashMap<String, Integer>> suggestions = new
LinkedHashMap<>();
+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions = new
LinkedHashMap<>();
suggestions.put(TOKEN_AYE, AYE);
suggestions.put(TOKEN_BEE, BEE);
suggestions.put(TOKEN_CEE, CEE);
@@ -184,7 +184,8 @@ public class SpellPossibilityIteratorTest extends
SolrTestCaseJ4 {
@Test
public void testOverlappingTokens() {
- Map<Token, LinkedHashMap<String, Integer>> overlappingSuggestions = new
LinkedHashMap<>();
+ Map<SpellCheckToken, LinkedHashMap<String, Integer>>
overlappingSuggestions =
+ new LinkedHashMap<>();
overlappingSuggestions.put(TOKEN_AYE, AYE);
overlappingSuggestions.put(TOKEN_BEE, BEE);
overlappingSuggestions.put(TOKEN_AYE_BEE, AYE_BEE);
@@ -197,10 +198,10 @@ public class SpellPossibilityIteratorTest extends
SolrTestCaseJ4 {
Set<PossibilityIterator.RankedSpellPossibility> dupChecker = new
HashSet<>();
while (iter.hasNext()) {
PossibilityIterator.RankedSpellPossibility rsp = iter.next();
- Token a = null;
- Token b = null;
- Token ab = null;
- Token c = null;
+ SpellCheckToken a = null;
+ SpellCheckToken b = null;
+ SpellCheckToken ab = null;
+ SpellCheckToken c = null;
for (SpellCheckCorrection scc : rsp.corrections) {
if (scc.getOriginal().equals(TOKEN_AYE)) {
a = scc.getOriginal();
diff --git
a/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java
b/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java
index 42172c89e45..d1f16a0d38f 100644
---
a/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java
+++
b/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java
@@ -16,10 +16,15 @@
*/
package org.apache.solr.spelling;
-import java.util.ArrayList;
-import java.util.Collection;
+import java.io.IOException;
import java.util.List;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.solr.SolrTestCase;
import org.apache.solr.common.util.NamedList;
import org.junit.Test;
@@ -32,17 +37,17 @@ import org.junit.Test;
public class SpellingQueryConverterTest extends SolrTestCase {
@Test
- public void test() {
+ public void test() throws IOException {
SpellingQueryConverter converter = new SpellingQueryConverter();
converter.init(new NamedList<>());
converter.setAnalyzer(new WhitespaceAnalyzer());
- Collection<Token> tokens = converter.convert("field:foo");
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert("field:foo"));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not: " + 1, 1,
tokens.size());
}
@Test
- public void testNumeric() {
+ public void testNumeric() throws IOException {
SpellingQueryConverter converter = new SpellingQueryConverter();
converter.init(new NamedList<>());
converter.setAnalyzer(new WhitespaceAnalyzer());
@@ -58,7 +63,7 @@ public class SpellingQueryConverterTest extends SolrTestCase {
};
int[] tokensToExpect = {1, 1, 2, 2, 2, 2, 2, 2};
for (int i = 0; i < queries.length; i++) {
- Collection<Token> tokens = converter.convert(queries[i]);
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert(queries[i]));
assertEquals(
"tokens Size: " + tokens.size() + " is not: " + tokensToExpect[i],
tokens.size(),
@@ -67,24 +72,24 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
}
@Test
- public void testSpecialChars() {
+ public void testSpecialChars() throws IOException {
SpellingQueryConverter converter = new SpellingQueryConverter();
converter.init(new NamedList<>());
converter.setAnalyzer(new WhitespaceAnalyzer());
String original = "field_with_underscore:value_with_underscore";
- Collection<Token> tokens = converter.convert(original);
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert(original));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 1", 1,
tokens.size());
assertTrue("Token offsets do not match", isOffsetCorrect(original,
tokens));
original = "field_with_digits123:value_with_digits123";
- tokens = converter.convert(original);
+ tokens = SpellCheckToken.drain(converter.convert(original));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 1", 1,
tokens.size());
assertTrue("Token offsets do not match", isOffsetCorrect(original,
tokens));
original = "field-with-hyphens:value-with-hyphens";
- tokens = converter.convert(original);
+ tokens = SpellCheckToken.drain(converter.convert(original));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 1", 1,
tokens.size());
assertTrue("Token offsets do not match", isOffsetCorrect(original,
tokens));
@@ -97,7 +102,7 @@ public class SpellingQueryConverterTest extends SolrTestCase
{
// assertTrue("Token offsets do not match", isOffsetCorrect(original,
tokens));
original = "foo:bar^5.0";
- tokens = converter.convert(original);
+ tokens = SpellCheckToken.drain(converter.convert(original));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 1", 1,
tokens.size());
assertTrue("Token offsets do not match", isOffsetCorrect(original,
tokens));
@@ -105,22 +110,16 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
String firstKeyword = "value1";
String secondKeyword = "value2";
original = "field-with-parenthesis:(" + firstKeyword + " " + secondKeyword
+ ")";
- tokens = converter.convert(original);
+ tokens = SpellCheckToken.drain(converter.convert(original));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 2", 2,
tokens.size());
assertTrue("Token offsets do not match", isOffsetCorrect(original,
tokens));
- assertEquals(
- "first Token is not " + firstKeyword,
- new ArrayList<>(tokens).get(0).toString(),
- firstKeyword);
- assertEquals(
- "second Token is not " + secondKeyword,
- new ArrayList<>(tokens).get(1).toString(),
- secondKeyword);
+ assertEquals("first Token is not " + firstKeyword,
tokens.get(0).toString(), firstKeyword);
+ assertEquals("second Token is not " + secondKeyword,
tokens.get(1).toString(), secondKeyword);
}
- private boolean isOffsetCorrect(String s, Collection<Token> tokens) {
- for (Token token : tokens) {
+ private boolean isOffsetCorrect(String s, List<SpellCheckToken> tokens) {
+ for (SpellCheckToken token : tokens) {
int start = token.startOffset();
int end = token.endOffset();
if (!s.substring(start, end).equals(token.toString())) return false;
@@ -129,50 +128,52 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
}
@Test
- public void testUnicode() {
+ public void testUnicode() throws IOException {
SpellingQueryConverter converter = new SpellingQueryConverter();
converter.init(new NamedList<>());
converter.setAnalyzer(new WhitespaceAnalyzer());
// chinese text value
- Collection<Token> tokens = converter.convert("text_field:我购买了道具和服装。");
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(converter.convert("text_field:我购买了道具和服装。"));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 1", 1,
tokens.size());
- tokens = converter.convert("text_购field:我购买了道具和服装。");
+ tokens =
SpellCheckToken.drain(converter.convert("text_购field:我购买了道具和服装。"));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 1", 1,
tokens.size());
- tokens = converter.convert("text_field:我购xyz买了道具和服装。");
+ tokens =
SpellCheckToken.drain(converter.convert("text_field:我购xyz买了道具和服装。"));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 1", 1,
tokens.size());
}
@Test
- public void testMultipleClauses() {
+ public void testMultipleClauses() throws IOException {
SpellingQueryConverter converter = new SpellingQueryConverter();
converter.init(new NamedList<>());
converter.setAnalyzer(new WhitespaceAnalyzer());
// two field:value pairs should give two tokens
- Collection<Token> tokens = converter.convert("买text_field:我购买了道具和服装。
field2:bar");
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(converter.convert("买text_field:我购买了道具和服装。
field2:bar"));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 2", 2,
tokens.size());
// a field:value pair and a search term should give two tokens
- tokens = converter.convert("text_field:我购买了道具和服装。 bar");
+ tokens = SpellCheckToken.drain(converter.convert("text_field:我购买了道具和服装。
bar"));
assertNotNull("tokens is null and it shouldn't be", tokens);
assertEquals("tokens Size: " + tokens.size() + " is not 2", 2,
tokens.size());
}
@Test
- public void testRequiredOrProhibitedFlags() {
+ public void testRequiredOrProhibitedFlags() throws IOException {
SpellingQueryConverter converter = new SpellingQueryConverter();
converter.init(new NamedList<>());
converter.setAnalyzer(new WhitespaceAnalyzer());
{
- List<Token> tokens = new ArrayList<>(converter.convert("aaa bbb ccc"));
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert("aaa bbb ccc"));
assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);
assertTrue(
"token 1 should be optional",
@@ -185,7 +186,7 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
!hasRequiredFlag(tokens.get(2)) &&
!hasProhibitedFlag(tokens.get(2)));
}
{
- List<Token> tokens = new ArrayList<>(converter.convert("+aaa bbb -ccc"));
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert("+aaa bbb -ccc"));
assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);
assertTrue(
"token 1 should be required",
@@ -198,7 +199,7 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
!hasRequiredFlag(tokens.get(2)) && hasProhibitedFlag(tokens.get(2)));
}
{
- List<Token> tokens = new ArrayList<>(converter.convert("aaa AND bbb
ccc"));
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert("aaa AND bbb ccc"));
assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);
assertTrue(
"token 1 doesn't precede n.b.o.",
@@ -211,7 +212,7 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));
}
{
- List<Token> tokens = new ArrayList<>(converter.convert("aaa OR bbb OR
ccc"));
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert("aaa OR bbb OR ccc"));
assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);
assertTrue(
"token 1 doesn't precede n.b.o.",
@@ -224,7 +225,8 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));
}
{
- List<Token> tokens = new ArrayList<>(converter.convert("aaa AND bbb NOT
ccc"));
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(converter.convert("aaa AND bbb NOT ccc"));
assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);
assertTrue(
"token 1 doesn't precede n.b.o.",
@@ -236,7 +238,8 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));
}
{
- List<Token> tokens = new ArrayList<>(converter.convert("aaa NOT bbb AND
ccc"));
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(converter.convert("aaa NOT bbb AND ccc"));
assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);
assertTrue(
"token 1 precedes n.b.o.", hasNBOFlag(tokens.get(0)) &&
hasInBooleanFlag(tokens.get(0)));
@@ -247,7 +250,8 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));
}
{
- List<Token> tokens = new ArrayList<>(converter.convert("aaa AND NOT bbb
AND ccc"));
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(converter.convert("aaa AND NOT bbb AND ccc"));
assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);
assertTrue(
"token 1 precedes n.b.o.", hasNBOFlag(tokens.get(0)) &&
hasInBooleanFlag(tokens.get(0)));
@@ -259,22 +263,64 @@ public class SpellingQueryConverterTest extends
SolrTestCase {
}
}
- private boolean hasRequiredFlag(Token t) {
- return (t.getFlags() & QueryConverter.REQUIRED_TERM_FLAG) ==
QueryConverter.REQUIRED_TERM_FLAG;
+ /**
+ * A query word whose analysis throws is skipped, and the query's other
words are still converted.
+ */
+ @Test
+ public void testWordFailingAnalysisIsSkipped() throws IOException {
+ SpellingQueryConverter converter = new SpellingQueryConverter();
+ converter.init(new NamedList<>());
+ converter.setAnalyzer(
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new WhitespaceTokenizer();
+ return new TokenStreamComponents(source, new
FailOnTermFilter(source, "bbb"));
+ }
+ });
+
+ List<SpellCheckToken> tokens =
SpellCheckToken.drain(converter.convert("aaa bbb ccc"));
+
+ assertEquals(List.of("aaa", "ccc"),
tokens.stream().map(SpellCheckToken::text).toList());
+ }
+
+ /** Fails on one term, the way a filter reading an external dictionary would
on a bad read. */
+ private static class FailOnTermFilter extends TokenFilter {
+ private final CharTermAttribute termAtt =
addAttribute(CharTermAttribute.class);
+ private final String failOn;
+
+ FailOnTermFilter(TokenStream input, String failOn) {
+ super(input);
+ this.failOn = failOn;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (!input.incrementToken()) {
+ return false;
+ }
+ if (failOn.contentEquals(termAtt)) {
+ throw new IOException("analysis of '" + failOn + "' failed");
+ }
+ return true;
+ }
+ }
+
+ private boolean hasRequiredFlag(SpellCheckToken t) {
+ return (t.flags() & QueryConverter.REQUIRED_TERM_FLAG) ==
QueryConverter.REQUIRED_TERM_FLAG;
}
- private boolean hasProhibitedFlag(Token t) {
- return (t.getFlags() & QueryConverter.PROHIBITED_TERM_FLAG)
- == QueryConverter.PROHIBITED_TERM_FLAG;
+ private boolean hasProhibitedFlag(SpellCheckToken t) {
+ return (t.flags() & QueryConverter.PROHIBITED_TERM_FLAG) ==
QueryConverter.PROHIBITED_TERM_FLAG;
}
- private boolean hasNBOFlag(Token t) {
- return (t.getFlags() &
QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)
+ private boolean hasNBOFlag(SpellCheckToken t) {
+ return (t.flags() & QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)
== QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;
}
- private boolean hasInBooleanFlag(Token t) {
- return (t.getFlags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)
+ private boolean hasInBooleanFlag(SpellCheckToken t) {
+ return (t.flags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)
== QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG;
}
}
diff --git
a/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java
b/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java
index e664e408287..ad0729b123b 100644
---
a/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java
+++
b/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java
@@ -16,7 +16,7 @@
*/
package org.apache.solr.spelling;
-import java.util.Collection;
+import java.io.IOException;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.miscellaneous.TrimFilter;
import org.apache.lucene.analysis.pattern.PatternReplaceFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
@@ -66,12 +67,18 @@ public class TestSuggestSpellingConverter extends
BaseTokenStreamTestCase {
}
public void assertConvertsTo(String text, String expected[]) {
- Collection<Token> tokens = converter.convert(text);
- assertEquals(tokens.size(), expected.length);
- int i = 0;
- for (Token token : tokens) {
- assertEquals(token.toString(), expected[i]);
- i++;
+ try (TokenStream stream = converter.convert(text)) {
+ stream.reset();
+ CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
+ int i = 0;
+ while (stream.incrementToken()) {
+ assertEquals(termAtt.toString(), expected[i]);
+ i++;
+ }
+ stream.end();
+ assertEquals(i, expected.length);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
}
}
diff --git
a/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
index 87db8f6176a..4922b3c5c1e 100644
---
a/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
+++
b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
@@ -16,8 +16,8 @@
*/
package org.apache.solr.spelling;
-import java.util.Collection;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks;
@@ -73,7 +73,7 @@ public class WordBreakSolrSpellCheckerTest extends
SolrTestCaseJ4 {
{
// Prior to SOLR-8175, the required term would cause an AIOOBE.
- Collection<Token> tokens = qc.convert("+pine apple good ness");
+ List<SpellCheckToken> tokens = SpellCheckToken.drain(qc.convert("+pine
apple good ness"));
SpellingOptions spellOpts = new SpellingOptions(tokens,
searcher.get().getIndexReader(), 10);
SpellingResult result = checker.getSuggestions(spellOpts);
searcher.decref();
@@ -81,7 +81,8 @@ public class WordBreakSolrSpellCheckerTest extends
SolrTestCaseJ4 {
assertEquals(5, result.getSuggestions().size());
}
- Collection<Token> tokens = qc.convert("paintable pine apple good ness");
+ List<SpellCheckToken> tokens =
+ SpellCheckToken.drain(qc.convert("paintable pine apple good ness"));
SpellingOptions spellOpts = new SpellingOptions(tokens,
searcher.get().getIndexReader(), 10);
SpellingResult result = checker.getSuggestions(spellOpts);
searcher.decref();
@@ -89,8 +90,9 @@ public class WordBreakSolrSpellCheckerTest extends
SolrTestCaseJ4 {
assertTrue(result != null && result.getSuggestions() != null);
assertEquals(9, result.getSuggestions().size());
- for (Map.Entry<Token, LinkedHashMap<String, Integer>> s :
result.getSuggestions().entrySet()) {
- Token orig = s.getKey();
+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> s :
+ result.getSuggestions().entrySet()) {
+ SpellCheckToken orig = s.getKey();
String[] corr = s.getValue().keySet().toArray(new String[0]);
if (orig.toString().equals("paintable")) {
assertEquals(0, orig.startOffset());
diff --git
a/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java
b/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java
index 4c36106fd42..a5e1a214be3 100644
---
a/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java
+++
b/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java
@@ -24,9 +24,9 @@ import java.util.List;
import org.apache.solr.core.SolrCore;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.spelling.SolrSpellChecker;
+import org.apache.solr.spelling.SpellCheckToken;
import org.apache.solr.spelling.SpellingOptions;
import org.apache.solr.spelling.SpellingResult;
-import org.apache.solr.spelling.Token;
/** A Dummy SpellChecker for testing purposes */
public class DummyCustomParamSpellChecker extends SolrSpellChecker {
@@ -54,7 +54,7 @@ public class DummyCustomParamSpellChecker extends
SolrSpellChecker {
int i = 0;
for (String name : lst) {
String value = options.customParams.get(name);
- result.add(new Token(name, i, i + 1), List.of(value));
+ result.add(new SpellCheckToken(name, i, i + 1), List.of(value));
i += 2;
}
return result;
