airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4070125105
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -150,49 +221,458 @@ private static String
buildIdentityFromPolicyProperties(IndexPolicyTypeEnum type
* Resolve a component (tokenizer) to its identity.
*/
private static String resolveComponentIdentity(String name,
IndexPolicyTypeEnum expectedType) {
+ return resolveComponentIdentity(name, expectedType, null);
+ }
+
+ /**
+ * {@code foldBlockedBytes} is the case-folding context of a char filter:
null without a
+ * downstream fold, otherwise the bytes that filters between this one and
the fold rewrite.
+ */
+ private static String resolveComponentIdentity(
+ String name, IndexPolicyTypeEnum expectedType, boolean[]
foldBlockedBytes) {
if (Strings.isNullOrEmpty(name)) {
return "";
}
- // Check if it's a built-in component
- if (expectedType == IndexPolicyTypeEnum.TOKENIZER
- && IndexPolicy.BUILTIN_TOKENIZERS.contains(name)) {
- return name;
+ // Existing named policies take precedence over built-ins for upgrade
compatibility.
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ if (policy.isInvalid()) {
+ return "invalid-policy:" + policy.getId() + ":" +
policy.getName();
+ }
+ Map<String, String> props = policy.getProperties();
+ if (props != null && !props.isEmpty()) {
+ TreeMap<String, String> sortedProps = new
TreeMap<>(props);
+ String type = sortedProps.get(IndexPolicy.PROP_TYPE);
+ String normalizedType =
normalizeBuiltinComponentName(type, expectedType);
+ if (normalizedType != null) {
+ if ("empty".equals(normalizedType)) {
+ return "";
+ }
+ sortedProps.put(IndexPolicy.PROP_TYPE,
normalizedType);
+ canonicalizeEffectiveComponentProperties(
+ sortedProps, normalizedType, expectedType);
+ if (sortedProps.size() == 1) {
+ return normalizedType;
+ }
+ }
+ if (expectedType == IndexPolicyTypeEnum.TOKENIZER
+ &&
"ngram".equals(sortedProps.get(IndexPolicy.PROP_TYPE))) {
+ // This setting only limits policy creation; it
does not change emitted tokens.
+ sortedProps.remove(PROP_MAX_NGRAM_DIFF);
+ }
+ if (expectedType == IndexPolicyTypeEnum.CHAR_FILTER
+ &&
"char_replace".equals(sortedProps.get(IndexPolicy.PROP_TYPE))) {
+ String replacement =
sortedProps.getOrDefault("replacement", " ");
+ String pattern = canonicalizeCharReplacePattern(
+ sortedProps.get("pattern"), replacement,
foldBlockedBytes);
+ if (pattern.isEmpty()) {
+ return "";
+ }
+ sortedProps.put("pattern", pattern);
+ sortedProps.put("replacement", replacement);
+ }
+ if (normalizedType != null && sortedProps.size() == 1)
{
+ return normalizedType;
+ }
+ return sortedProps.toString();
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution or the original name.
+ }
+
+ String normalizedName = normalizeBuiltinComponentName(name,
expectedType);
+ return "empty".equals(normalizedName) ? "" : normalizedName == null ?
name : normalizedName;
+ }
+
+ private static void canonicalizeEffectiveComponentProperties(
+ TreeMap<String, String> properties, String type,
IndexPolicyTypeEnum expectedType) {
+ if ("pinyin".equals(type)) {
Review Comment:
Fixed in 77fa36419ba. Confirmed on BE: `PinyinTokenizer::addCandidate` trims
only the candidate term, and without `keep_original` every candidate is
dictionary pinyin, a `segmentChinese` character or ASCII alphanumerics (the
shipped pinyin dictionary contains no whitespace), so `trim_whitespace` cannot
change any term, position or offset. Reproduced first: `{type=pinyin}` and
`{type=pinyin, trim_whitespace=false}` had different tokenizer identities.
The tokenizer identity now drops `trim_whitespace` when the effective
`keep_original` is false; the token-filter form keeps it because `PinyinFilter`
also trims the incoming token.
`testPinyinTokenizerTrimWhitespaceOnlyAffectsOriginalCandidate` covers both
forms and the kept-original negative; CREATE and ALTER are in
`testCreateTableRejectsDefaultRestatingAndCoveredComponentAliases` / the shared
ALTER case.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -226,26 +704,374 @@ private static String resolveTokenFilterIdentity(String
filterList) {
* IMPORTANT: Order is preserved because filter order is semantically
significant.
*/
private static String resolveCharFilterIdentity(String filterList) {
+ return resolveCharFilterIdentity(filterList, false);
+ }
+
+ private static String resolveCharFilterIdentity(String filterList, boolean
lowercaseDownstream) {
+ ArrayDeque<String> identities = new ArrayDeque<>();
+ walkCharFilters(filterList, lowercaseDownstream, identities);
+ return String.join(",", identities);
+ }
+
+ /**
+ * Resolve the chain from its last filter to its first, collecting
identities, and return the
+ * case-folding context that a filter placed in front of the chain would
run in.
+ */
+ private static boolean[] walkCharFilters(
+ String filterList, boolean lowercaseDownstream, Deque<String>
identities) {
+ boolean[] foldBlockedBytes = lowercaseDownstream ? new boolean[256] :
null;
if (Strings.isNullOrEmpty(filterList)) {
- return "";
+ return foldBlockedBytes;
}
- StringBuilder sb = new StringBuilder();
String[] filters = filterList.split(",\\s*");
// DO NOT sort - filter order is semantically significant
- for (int i = 0; i < filters.length; i++) {
- String filter = filters[i].trim();
- if (i > 0) {
- sb.append(",");
+ for (int i = filters.length - 1; i >= 0; --i) {
+ String filterName = filters[i].trim();
+ String filter = resolveComponentIdentity(
+ filterName, IndexPolicyTypeEnum.CHAR_FILTER,
foldBlockedBytes);
+ if (Strings.isNullOrEmpty(filter)) {
+ continue;
}
+ identities.addFirst(filter);
+ foldBlockedBytes = foldBlockedBytesBefore(filterName,
foldBlockedBytes);
+ }
+ return foldBlockedBytes;
+ }
- if (IndexPolicy.BUILTIN_CHAR_FILTERS.contains(filter)) {
- sb.append(filter);
- } else {
- sb.append(resolveComponentIdentity(filter,
IndexPolicyTypeEnum.CHAR_FILTER));
+ /**
+ * Context for the filter that runs before this one: a case fold starts a
fresh context, a
+ * char_replace filter adds the bytes it rewrites, and any other filter
ends the context.
+ */
+ private static boolean[] foldBlockedBytesBefore(String filterName,
boolean[] foldBlockedBytes) {
+ if (isCaseFoldingCharFilter(filterName)) {
+ return new boolean[256];
+ }
+ if (foldBlockedBytes == null) {
+ return null;
+ }
+ boolean[] sourceBytes = charReplaceSourceBytes(filterName);
+ if (sourceBytes == null) {
+ return null;
+ }
+ for (int i = 0; i < foldBlockedBytes.length; ++i) {
+ foldBlockedBytes[i] |= sourceBytes[i];
+ }
+ return foldBlockedBytes;
+ }
+
+ /** Bytes a named char_replace filter rewrites, or null for any other
filter. */
+ private static boolean[] charReplaceSourceBytes(String filterName) {
+ IndexPolicy policy = findPolicy(filterName,
IndexPolicyTypeEnum.CHAR_FILTER);
+ if (policy == null || policy.isInvalid() || policy.getProperties() ==
null) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ String pattern = properties.get("pattern");
+ if (!"char_replace".equals(type) || pattern == null) {
+ return null;
+ }
+ boolean[] sourceBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
Review Comment:
Fixed in 77fa36419ba. Confirmed on BE: `char_replace` rewrites each pattern
byte to `replacement[0]`, so a pattern byte equal to the single replacement
byte is a no-op. Reproduced first: `lower_a,x_to_A,fold` and `x_to_A,fold`
(with `x_to_A={pattern=Ax, replacement=A}`) had different identities because
the raw `A` blocked the elimination.
`charReplaceSourceBytes` now excludes a pattern byte equal to the single
ASCII replacement byte from the blocked set.
`testCharReplaceReplacementByteDoesNotBlockCaseFold` covers the mixed no-op
pattern, keeps `{pattern=Ab, replacement=b}` blocking, and CREATE/ALTER
duplicate rejection is covered in the shared cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
