airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4073753179
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -226,26 +858,381 @@ private static String resolveTokenFilterIdentity(String
filterList) {
* IMPORTANT: Order is preserved because filter order is semantically
significant.
*/
private static String resolveCharFilterIdentity(String filterList) {
+ return resolveCharFilterIdentity(filterList, null);
+ }
+
+ private static String resolveCharFilterIdentity(String filterList,
FoldContext downstreamFold) {
+ ArrayDeque<String> identities = new ArrayDeque<>();
+ walkCharFilters(filterList, downstreamFold, identities);
+ return String.join(",", identities);
+ }
+
+ /**
+ * Resolve the chain from its last filter to its first, collecting
identities, and return the
+ * case-folding context that a filter placed in front of the chain would
run in.
+ */
+ private static FoldContext walkCharFilters(
+ String filterList, FoldContext downstreamFold, Deque<String>
identities) {
+ FoldContext fold = downstreamFold;
if (Strings.isNullOrEmpty(filterList)) {
- return "";
+ return fold;
}
- StringBuilder sb = new StringBuilder();
String[] filters = filterList.split(",\\s*");
// DO NOT sort - filter order is semantically significant
- for (int i = 0; i < filters.length; i++) {
- String filter = filters[i].trim();
- if (i > 0) {
- sb.append(",");
+ for (int i = filters.length - 1; i >= 0; --i) {
+ String filterName = filters[i].trim();
+ String filter = resolveComponentIdentity(filterName,
IndexPolicyTypeEnum.CHAR_FILTER, fold);
+ if (Strings.isNullOrEmpty(filter)) {
+ continue;
}
+ identities.addFirst(filter);
+ fold = foldContextBefore(filterName, fold);
+ }
+ return fold;
+ }
- if (IndexPolicy.BUILTIN_CHAR_FILTERS.contains(filter)) {
- sb.append(filter);
- } else {
- sb.append(resolveComponentIdentity(filter,
IndexPolicyTypeEnum.CHAR_FILTER));
+ /**
+ * Context for the filter that runs before this one: a case fold starts a
fresh context, a
+ * char_replace filter adds the bytes it rewrites, and any other filter
ends the context.
+ */
+ private static FoldContext foldContextBefore(String filterName,
FoldContext fold) {
+ FoldContext caseFold = caseFoldingCharFilterContext(filterName);
+ if (caseFold != null) {
+ return caseFold;
+ }
+ if (fold == null) {
+ return null;
+ }
+ boolean[] sourceBytes = charReplaceSourceBytes(filterName);
+ if (sourceBytes == null) {
+ return null;
+ }
+ fold.block(sourceBytes);
+ return fold;
+ }
+
+ /** Bytes a named char_replace filter rewrites, or null for any other
filter. */
+ private static boolean[] charReplaceSourceBytes(String filterName) {
+ IndexPolicy policy = findPolicy(filterName,
IndexPolicyTypeEnum.CHAR_FILTER);
+ if (policy == null || policy.isInvalid() || policy.getProperties() ==
null) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ String pattern = properties.get("pattern");
+ if (!"char_replace".equals(type) || pattern == null) {
+ return null;
+ }
+ // Replacing the single replacement byte with itself leaves the stream
unchanged.
+ String replacement = properties.getOrDefault("replacement", " ");
+ int replacementByte = replacement.length() == 1 &&
replacement.charAt(0) < 128 ? replacement.charAt(0) : -1;
+ boolean[] sourceBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
+ char patternByte = pattern.charAt(i);
+ if (patternByte < sourceBytes.length && patternByte !=
replacementByte) {
+ sourceBytes[patternByte] = true;
}
}
- return sb.toString();
+ return sourceBytes;
+ }
+
+ /** The named policy when one exists with the expected type, or null. */
+ private static IndexPolicy findPolicy(String name, IndexPolicyTypeEnum
expectedType) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ return policy;
+ }
+ }
+ } catch (RuntimeException e) {
+ // Treat lookup failures as an unknown policy.
+ }
+ return null;
+ }
+
+ /** Fold context started by a named or built-in case-folding char filter,
or null for any other filter. */
+ private static FoldContext caseFoldingCharFilterContext(String name) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() ==
IndexPolicyTypeEnum.CHAR_FILTER) {
+ if (policy.isInvalid()) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ if (properties != null && !properties.isEmpty()) {
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ return "icu_normalizer".equals(type) ?
icuNormalizerFoldContext(properties) : null;
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution.
+ }
+
+ return "icu_normalizer".equals(normalizeBuiltinComponentName(name,
IndexPolicyTypeEnum.CHAR_FILTER))
+ ? FoldContext.unfiltered() : null;
+ }
+
+ /**
+ * Fold context of an icu_normalizer component: the default nfkc_cf form
folds case over every
+ * code point, or only inside a parsable non-empty unicode_set_filter.
Null for other forms.
+ */
+ private static FoldContext icuNormalizerFoldContext(Map<String, String>
properties) {
+ if (!"nfkc_cf".equals(icuNormalizerName(properties))) {
+ return null;
+ }
+ String filter = properties.get("unicode_set_filter");
+ if (filter == null || filter.isEmpty()) {
+ return FoldContext.unfiltered();
+ }
+ try {
+ UnicodeSet unicodeSet = new UnicodeSet(filter);
+ return unicodeSet.isEmpty() ? FoldContext.unfiltered() : new
FoldContext(unicodeSet.freeze());
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ /** Whether an icu_normalizer component leaves ASCII letters as they are.
*/
+ private static boolean isAsciiCaseTransparentIcuNormalizer(Map<String,
String> properties) {
+ String name = icuNormalizerName(properties);
+ return "nfc".equals(name) || "nfd".equals(name) || "nfkc".equals(name)
|| "nfkd".equals(name);
+ }
+
+ private static String icuNormalizerName(Map<String, String> properties) {
+ return properties.getOrDefault("name",
"nfkc_cf").trim().toLowerCase(Locale.ROOT);
+ }
+
+ /** The outer char filter runs before everything else, so it takes the
analyzer's fold context. */
+ private static String appendOuterCharFilterIdentity(
+ String analyzerIdentity, Map<String, String> properties,
FoldContext fold) {
+ String type =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE);
+ String pattern =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN);
+ if (!"char_replace".equals(type) || Strings.isNullOrEmpty(pattern)) {
+ return analyzerIdentity;
+ }
+ String replacement = properties.getOrDefault(
+
InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " ");
+ String canonicalPattern = canonicalizeCharReplacePattern(pattern,
replacement, fold);
+ if (canonicalPattern.isEmpty()) {
+ return analyzerIdentity;
+ }
+ return analyzerIdentity + "|outer_char_filter=char_replace:"
+ + canonicalPattern.length() + ":" + canonicalPattern + ":"
+ + replacement.length() + ":" + replacement + ";";
+ }
+
+ /**
+ * Canonicalize the ASCII pattern to the BE filter's byte set.
+ * Order, duplicate bytes, and replacements of a byte with itself do not
change the stream.
+ */
+ private static String canonicalizeCharReplacePattern(
+ String pattern, String replacement, FoldContext fold) {
+ if (replacement.length() != 1) {
+ return pattern;
+ }
+ char replacementByte = replacement.charAt(0);
Review Comment:
Fixed in b0924c2b7f4. Reproduced first: `a -> A` ahead of a lowercase token
filter, a default `icu_normalizer`, a filtered `[A]` fold and the outer char
filter all kept the mapping in the identity, while the opposite `A -> a` was
already dropped.
`canonicalizeCharReplacePattern` now also handles an upper-case replacement
byte: when the fold context proves `foldsByte(X, x)` (same blocked-byte and set
conditions as the other direction), the lower-case pattern byte `x` is removed
because `x -> X -> fold` is `x` again. With no fold, a `[B]` set, or an
intermediate filter that rewrites `A` or `a`, the mapping is kept.
`testDownstreamFoldErasesLowerToUpperReplacement` covers the four positive
contexts and the negatives; CREATE/ALTER are in the shared cases from the
previous thread.
##########
be/src/storage/index/inverted/token_filter/pinyin_filter.cpp:
##########
@@ -214,29 +245,94 @@ bool PinyinFilter::readTerm(Token* token) {
return false;
}
-bool PinyinFilter::processCurrentToken() {
- processed_candidate_ = true;
+bool PinyinFilter::prepareCurrentSource(std::vector<UChar32>&
source_codepoints) {
+ size_t source_start = 0;
+ size_t source_end = current_token_text_.size();
+ if (config_->trimWhitespace) {
+ source_start = current_token_text_.find_first_not_of(" \t\n\r");
+ if (source_start == std::string::npos) {
+ return false;
+ }
+ source_end = current_token_text_.find_last_not_of(" \t\n\r") + 1;
+ }
+ current_source_ = current_token_text_.substr(source_start, source_end -
source_start);
- if (!has_current_token_) {
+ if (current_source_.empty()) {
return false;
}
- current_source_ = current_token_text_;
+ if (config_->ignorePinyinOffset) {
+ convertToCodepoints(current_source_, source_codepoints);
+ return !source_codepoints.empty();
+ }
- // Apply trimming if configured
- if (config_->trimWhitespace) {
- current_source_ = trim(current_source_);
+ current_runes_ = convertToRunes(current_source_, source_codepoints);
+
+ std::vector<RuneInfo> original_runes;
+ if (!current_source_byte_offsets_.empty()) {
+ std::vector<UChar32> original_codepoints;
+ original_runes = convertToRunes(current_token_text_,
original_codepoints);
+ if (current_source_byte_offsets_.size() != original_runes.size() + 1) {
+ current_source_byte_offsets_.clear();
+ current_source_byte_end_offsets_.clear();
+ }
+ }
+ if (!current_source_byte_offsets_.empty()) {
+ DORIS_CHECK(current_source_byte_end_offsets_.empty() ||
+ current_source_byte_end_offsets_.size() ==
original_runes.size());
+ const auto start_rune = std::ranges::lower_bound(
+ original_runes, static_cast<int32_t>(source_start), {},
&RuneInfo::byte_start);
+ const auto end_rune = std::ranges::lower_bound(
+ original_runes, static_cast<int32_t>(source_end), {},
&RuneInfo::byte_start);
+ const auto start_index = static_cast<size_t>(start_rune -
original_runes.begin());
+ const auto end_index = static_cast<size_t>(end_rune -
original_runes.begin());
+ DORIS_CHECK_EQ(end_index - start_index, current_runes_.size());
+ const int32_t token_start_offset = current_start_offset_;
+ current_start_offset_ += current_source_byte_offsets_[start_index];
+ current_end_offset_ =
+ token_start_offset + (current_source_byte_end_offsets_.empty()
+ ?
current_source_byte_offsets_[end_index]
+ :
current_source_byte_end_offsets_[end_index - 1]);
+ for (size_t i = 0; i < current_runes_.size(); ++i) {
+ current_runes_[i].byte_start =
current_source_byte_offsets_[start_index + i] -
+
current_source_byte_offsets_[start_index];
+ current_runes_[i].byte_end =
+ (current_source_byte_end_offsets_.empty()
+ ? current_source_byte_offsets_[start_index + i +
1]
+ : current_source_byte_end_offsets_[start_index +
i]) -
+ current_source_byte_offsets_[start_index];
+ }
+ } else if (has_current_conservative_source_span_) {
+ DORIS_CHECK_GE(current_conservative_source_start_, 0);
+ DORIS_CHECK_GE(current_conservative_source_end_,
current_conservative_source_start_);
+ const int32_t token_start_offset = current_start_offset_;
+ current_start_offset_ = token_start_offset +
current_conservative_source_start_;
+ current_end_offset_ = token_start_offset +
current_conservative_source_end_;
+ const int32_t source_length =
+ current_conservative_source_end_ -
current_conservative_source_start_;
+ for (auto& rune : current_runes_) {
+ rune.byte_start = 0;
+ rune.byte_end = source_length;
+ }
+ } else {
+ current_start_offset_ += static_cast<int32_t>(source_start);
+ current_end_offset_ =
+ current_start_offset_ + static_cast<int32_t>(source_end -
source_start);
}
- if (current_source_.empty()) {
+ return !source_codepoints.empty();
+}
+
+bool PinyinFilter::processCurrentToken() {
+ processed_candidate_ = true;
+
+ if (!has_current_token_) {
return false;
}
- // Convert to Unicode codepoints for processing
+ // Convert to Unicode codepoints for processing.
std::vector<UChar32> source_codepoints;
- convertToRunes(current_source_, source_codepoints);
-
- if (source_codepoints.empty()) {
+ if (!prepareCurrentSource(source_codepoints)) {
return false;
}
Review Comment:
Fixed in 787e0ab4f3f. Reproduced first: in the default mode a 4096-byte
ASCII token still built a 4096-entry rune index vector although
`setTokenAttributes()` gives every candidate the whole token span when offsets
are ignored.
`PinyinFilter` now collects the rune indices only when
`ignore_pinyin_offset=false`; `processAsciiBuffer()` uses letter positions in
their place otherwise (those ranges are only compared for the whole-token check
and never published). `TestPinyinFilterCollectsRuneIndicesOnlyForOffsets`
covers both modes on a large token.
##########
be/src/storage/index/inverted/tokenizer/icu/icu_tokenizer.cpp:
##########
@@ -54,26 +58,78 @@ Token* ICUTokenizer::next(Token* token) {
utf8Str_.clear();
int32_t length = std::min(end - start, LUCENE_MAX_WORD_LEN);
+ if (length < end - start && length > 0 && U16_IS_LEAD(buffer_.charAt(start
+ length - 1)) &&
+ U16_IS_TRAIL(buffer_.charAt(start + length))) {
+ --length;
+ }
auto subString = buffer_.tempSubString(start, length);
+ sourceUtf8Str_.clear();
Review Comment:
Fixed in 787e0ab4f3f. Reproduced first: a plain (non-opted-in) ICU tokenizer
still filled `sourceUtf8Str_` with a second UTF-8 conversion of every token.
The source text is now transcoded only when provenance is enabled and the
tokenizer lowercases; otherwise the term itself is passed as the provenance
source, since without lowercasing they are byte-identical. In the current
pipelines `ICUTokenizer::lowercase` is never set (lowercasing is done by a
separate `LowerCaseFilter`), so the second conversion is gone from both paths.
`TestIcuTokenizerTranscodesSourceOnlyForOffsets` covers disabled mode and
enabled mode (no extra buffer, provenance still published).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
