airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4073751436
##########
be/src/storage/index/inverted/tokenizer/ngram/ngram_tokenizer.cpp:
##########
@@ -74,6 +75,10 @@ Token* NGramTokenizer::next(Token* token) {
to_chars(_buffer, _buffer_start, _gram_size);
set(token, _utf8_buffer);
+ set_source_byte_offsets(_utf8_buffer, _offset);
+ token->setStartOffset(correct_source_offset(_offset));
Review Comment:
Fixed in 787e0ab4f3f. Reproduced first: `nfkc_cf` on U+FB01 -> 1-gram ->
offset-aware Pinyin reported `i` at `[3,3)`, because the gram starts inside the
`fi` expansion and `correct_offset()` maps an interior position to the edit's
source end.
`DorisCharFilter` gains `correct_start_offset()`: a position strictly inside
a changed edit maps to that edit's source start, anything else matches
`correct_offset()`. `ICUNormalizerCharFilter` implements it from the same
`Edits` iterator (`findDestinationIndex`), nested char filters chain it, and
every tokenizer now uses it for term starts (NGram, CharGroup, Standard, Basic,
ICU, Pinyin, and IK), including the start used by the shared
`set_source_byte_offsets()` projection. The second gram now reports `[0,3)`
with its provenance. `TestNGramInsideCharFilterExpansionKeepsSourceSpan` covers
the ligature -> 1-gram chain and a reset to full-width input with exact
per-letter spans.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -226,26 +858,381 @@ private static String resolveTokenFilterIdentity(String
filterList) {
* IMPORTANT: Order is preserved because filter order is semantically
significant.
*/
private static String resolveCharFilterIdentity(String filterList) {
+ return resolveCharFilterIdentity(filterList, null);
+ }
+
+ private static String resolveCharFilterIdentity(String filterList,
FoldContext downstreamFold) {
+ ArrayDeque<String> identities = new ArrayDeque<>();
+ walkCharFilters(filterList, downstreamFold, identities);
+ return String.join(",", identities);
+ }
+
+ /**
+ * Resolve the chain from its last filter to its first, collecting
identities, and return the
+ * case-folding context that a filter placed in front of the chain would
run in.
+ */
+ private static FoldContext walkCharFilters(
+ String filterList, FoldContext downstreamFold, Deque<String>
identities) {
+ FoldContext fold = downstreamFold;
if (Strings.isNullOrEmpty(filterList)) {
- return "";
+ return fold;
}
- StringBuilder sb = new StringBuilder();
String[] filters = filterList.split(",\\s*");
// DO NOT sort - filter order is semantically significant
- for (int i = 0; i < filters.length; i++) {
- String filter = filters[i].trim();
- if (i > 0) {
- sb.append(",");
+ for (int i = filters.length - 1; i >= 0; --i) {
+ String filterName = filters[i].trim();
+ String filter = resolveComponentIdentity(filterName,
IndexPolicyTypeEnum.CHAR_FILTER, fold);
+ if (Strings.isNullOrEmpty(filter)) {
+ continue;
}
+ identities.addFirst(filter);
+ fold = foldContextBefore(filterName, fold);
+ }
+ return fold;
+ }
- if (IndexPolicy.BUILTIN_CHAR_FILTERS.contains(filter)) {
- sb.append(filter);
- } else {
- sb.append(resolveComponentIdentity(filter,
IndexPolicyTypeEnum.CHAR_FILTER));
+ /**
+ * Context for the filter that runs before this one: a case fold starts a
fresh context, a
+ * char_replace filter adds the bytes it rewrites, and any other filter
ends the context.
+ */
+ private static FoldContext foldContextBefore(String filterName,
FoldContext fold) {
+ FoldContext caseFold = caseFoldingCharFilterContext(filterName);
+ if (caseFold != null) {
+ return caseFold;
+ }
+ if (fold == null) {
+ return null;
+ }
+ boolean[] sourceBytes = charReplaceSourceBytes(filterName);
+ if (sourceBytes == null) {
+ return null;
+ }
+ fold.block(sourceBytes);
+ return fold;
+ }
+
+ /** Bytes a named char_replace filter rewrites, or null for any other
filter. */
+ private static boolean[] charReplaceSourceBytes(String filterName) {
+ IndexPolicy policy = findPolicy(filterName,
IndexPolicyTypeEnum.CHAR_FILTER);
+ if (policy == null || policy.isInvalid() || policy.getProperties() ==
null) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ String pattern = properties.get("pattern");
+ if (!"char_replace".equals(type) || pattern == null) {
+ return null;
+ }
+ // Replacing the single replacement byte with itself leaves the stream
unchanged.
+ String replacement = properties.getOrDefault("replacement", " ");
+ int replacementByte = replacement.length() == 1 &&
replacement.charAt(0) < 128 ? replacement.charAt(0) : -1;
+ boolean[] sourceBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
+ char patternByte = pattern.charAt(i);
+ if (patternByte < sourceBytes.length && patternByte !=
replacementByte) {
+ sourceBytes[patternByte] = true;
}
}
- return sb.toString();
+ return sourceBytes;
+ }
+
+ /** The named policy when one exists with the expected type, or null. */
+ private static IndexPolicy findPolicy(String name, IndexPolicyTypeEnum
expectedType) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ return policy;
+ }
+ }
+ } catch (RuntimeException e) {
+ // Treat lookup failures as an unknown policy.
+ }
+ return null;
+ }
+
+ /** Fold context started by a named or built-in case-folding char filter,
or null for any other filter. */
+ private static FoldContext caseFoldingCharFilterContext(String name) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() ==
IndexPolicyTypeEnum.CHAR_FILTER) {
+ if (policy.isInvalid()) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ if (properties != null && !properties.isEmpty()) {
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ return "icu_normalizer".equals(type) ?
icuNormalizerFoldContext(properties) : null;
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution.
+ }
+
+ return "icu_normalizer".equals(normalizeBuiltinComponentName(name,
IndexPolicyTypeEnum.CHAR_FILTER))
+ ? FoldContext.unfiltered() : null;
+ }
+
+ /**
+ * Fold context of an icu_normalizer component: the default nfkc_cf form
folds case over every
+ * code point, or only inside a parsable non-empty unicode_set_filter.
Null for other forms.
+ */
+ private static FoldContext icuNormalizerFoldContext(Map<String, String>
properties) {
+ if (!"nfkc_cf".equals(icuNormalizerName(properties))) {
+ return null;
+ }
+ String filter = properties.get("unicode_set_filter");
+ if (filter == null || filter.isEmpty()) {
+ return FoldContext.unfiltered();
+ }
+ try {
+ UnicodeSet unicodeSet = new UnicodeSet(filter);
+ return unicodeSet.isEmpty() ? FoldContext.unfiltered() : new
FoldContext(unicodeSet.freeze());
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ /** Whether an icu_normalizer component leaves ASCII letters as they are.
*/
+ private static boolean isAsciiCaseTransparentIcuNormalizer(Map<String,
String> properties) {
+ String name = icuNormalizerName(properties);
+ return "nfc".equals(name) || "nfd".equals(name) || "nfkc".equals(name)
|| "nfkd".equals(name);
+ }
+
+ private static String icuNormalizerName(Map<String, String> properties) {
+ return properties.getOrDefault("name",
"nfkc_cf").trim().toLowerCase(Locale.ROOT);
+ }
+
+ /** The outer char filter runs before everything else, so it takes the
analyzer's fold context. */
+ private static String appendOuterCharFilterIdentity(
+ String analyzerIdentity, Map<String, String> properties,
FoldContext fold) {
+ String type =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE);
+ String pattern =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN);
+ if (!"char_replace".equals(type) || Strings.isNullOrEmpty(pattern)) {
+ return analyzerIdentity;
+ }
+ String replacement = properties.getOrDefault(
+
InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " ");
+ String canonicalPattern = canonicalizeCharReplacePattern(pattern,
replacement, fold);
+ if (canonicalPattern.isEmpty()) {
+ return analyzerIdentity;
+ }
+ return analyzerIdentity + "|outer_char_filter=char_replace:"
+ + canonicalPattern.length() + ":" + canonicalPattern + ":"
+ + replacement.length() + ":" + replacement + ";";
+ }
+
+ /**
+ * Canonicalize the ASCII pattern to the BE filter's byte set.
+ * Order, duplicate bytes, and replacements of a byte with itself do not
change the stream.
+ */
+ private static String canonicalizeCharReplacePattern(
+ String pattern, String replacement, FoldContext fold) {
+ if (replacement.length() != 1) {
+ return pattern;
+ }
+ char replacementByte = replacement.charAt(0);
+ boolean[] replacedBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
+ char patternByte = pattern.charAt(i);
+ if (patternByte < replacedBytes.length && patternByte !=
replacementByte) {
+ replacedBytes[patternByte] = true;
+ }
+ }
+ if (fold != null && replacementByte >= 'a' && replacementByte <= 'z') {
+ // The downstream fold maps the upper-case byte to the replacement
anyway.
+ int upperByte = replacementByte - ('a' - 'A');
+ if (fold.foldsByte(upperByte, replacementByte)) {
+ replacedBytes[upperByte] = false;
+ }
+ }
+
+ StringBuilder canonical = new StringBuilder();
+ for (int i = 0; i < replacedBytes.length; ++i) {
+ if (replacedBytes[i]) {
+ canonical.append((char) i);
+ }
+ }
+ return canonical.toString();
+ }
+
+ private static FoldContext builtinIkFoldContext(String analyzerIdentity) {
+ return isDefaultLowercaseBuiltinIkIdentity(analyzerIdentity) ?
FoldContext.unfiltered() : null;
+ }
+
+ private static boolean isDefaultLowercaseBuiltinIkIdentity(String
analyzerIdentity) {
+ return (IndexPolicyTypeEnum.ANALYZER.name() +
":tokenizer=ik_smart;").equals(analyzerIdentity)
+ || (IndexPolicyTypeEnum.ANALYZER.name() +
":tokenizer=ik_max_word;").equals(analyzerIdentity);
+ }
+
+ /**
+ * Fold context for the outer char filter of a custom analyzer or
normalizer, which BE applies
+ * before the policy's own char filters. Unknown or unresolvable policies
get no context.
+ */
+ private static FoldContext customAnalyzerFoldContext(String analyzerName) {
+ if (IndexPolicy.BUILTIN_ANALYZERS.contains(analyzerName)) {
+ return null;
+ }
+ if (IndexPolicy.BUILTIN_NORMALIZERS.contains(analyzerName)) {
+ // The built-in normalizer lowercases keyword tokens without char
filters of its own.
+ return FoldContext.unfiltered();
+ }
+ IndexPolicy policy = findPolicy(analyzerName,
IndexPolicyTypeEnum.ANALYZER);
+ if (policy == null) {
+ policy = findPolicy(analyzerName, IndexPolicyTypeEnum.NORMALIZER);
+ }
+ if (policy == null || policy.isInvalid() || policy.getProperties() ==
null
+ || policy.getProperties().isEmpty()) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ try {
+ String tokenizerIdentity = resolveComponentIdentity(
+ properties.get(IndexPolicy.PROP_TOKENIZER),
IndexPolicyTypeEnum.TOKENIZER);
+ return
walkCharFilters(properties.get(IndexPolicy.PROP_CHAR_FILTER),
+ foldsAsciiCaseAfterCharFilters(policy.getType(),
properties, tokenizerIdentity),
+ new ArrayDeque<>());
+ } catch (RuntimeException e) {
+ return null;
+ }
+ }
+
+ /**
+ * The fold the tokenizer and token filters apply to ASCII letters, so a
char filter that only
+ * lowercases such a letter cannot change the output, or null when they
keep case.
+ */
+ private static FoldContext foldsAsciiCaseAfterCharFilters(
+ IndexPolicyTypeEnum type, Map<String, String> properties, String
tokenizerIdentity) {
+ if (type == IndexPolicyTypeEnum.NORMALIZER) {
+ // A normalizer always tokenizes with keyword, which is case
transparent.
+ return
tokenFiltersFoldAsciiCase(properties.get(IndexPolicy.PROP_TOKEN_FILTER));
+ }
+ if ("ik_smart".equals(tokenizerIdentity) ||
"ik_max_word".equals(tokenizerIdentity)) {
+ return FoldContext.unfiltered();
+ }
+ return
isCaseTransparentTokenizer(properties.get(IndexPolicy.PROP_TOKENIZER))
+ ?
tokenFiltersFoldAsciiCase(properties.get(IndexPolicy.PROP_TOKEN_FILTER)) : null;
+ }
+
+ /** Whether the tokenizer splits and emits ASCII letters the same way
regardless of their case. */
+ private static boolean isCaseTransparentTokenizer(String name) {
+ TreeMap<String, String> settings = resolveComponentSettings(name,
IndexPolicyTypeEnum.TOKENIZER);
+ if (settings == null) {
+ return false;
+ }
+ switch (settings.get(IndexPolicy.PROP_TYPE)) {
Review Comment:
Fixed in b0924c2b7f4. Reproduced first: an NGram alias with
`token_chars=letter,custom; custom_token_chars=A` and a CharGroup alias with a
redundant `[A]` literal kept the outer `A -> a` in their identities, because
the transparency check re-read the raw tokenizer properties.
`isCaseTransparentTokenizer` now runs the tokenizer settings through the
same canonicalization as the base identity before deciding, so a literal that
the identity already drops no longer makes the tokenizer look case sensitive;
`digit,custom` with `A` still does.
`testCaseTransparencyUsesCanonicalTokenizerSettings` covers both tokenizers,
and CREATE/ALTER are in
`testCreateTableRejectsRedundantTokenCharAndReverseCaseAliases` /
`testAddInvertedIndexRejectsRedundantTokenCharAndReverseCaseAliases`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
