airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4003074708
##########
be/src/storage/index/inverted/token_filter/pinyin_filter.cpp:
##########
@@ -392,37 +402,58 @@ void PinyinFilter::addCandidate(const TermItem& item) {
candidate_.push_back(new_item);
}
-void PinyinFilter::processAsciiBuffer(const std::string& ascii_buffer, int
start_pos, int end_pos) {
+void PinyinFilter::processAsciiBuffer(const std::string& ascii_buffer,
+ const std::vector<int>&
source_rune_indices) {
if (ascii_buffer.empty() || !config_->keepNoneChinese) {
return;
}
+ DORIS_CHECK_EQ(ascii_buffer.size(), source_rune_indices.size());
if (config_->noneChinesePinyinTokenize) {
// Use PinyinAlphabetTokenizer to split ASCII buffer into meaningful
tokens
std::vector<std::string> tokens =
PinyinAlphabetTokenizer::walk(ascii_buffer);
- int current_offset = start_pos;
+ size_t compact_offset = 0;
+ int fixed_offset = source_rune_indices.front();
for (const auto& token : tokens) {
+ const size_t compact_end = compact_offset + token.size();
+ DORIS_CHECK_LE(compact_end, source_rune_indices.size());
position_++;
- int token_end = (config_->fixedPinyinOffset)
- ? (current_offset + 1)
- : (current_offset +
static_cast<int>(token.length()));
- addCandidate(TermItem(token, current_offset, token_end,
position_));
- current_offset = token_end;
+ if (config_->fixedPinyinOffset) {
+ addCandidate(TermItem(token, fixed_offset, fixed_offset + 1,
position_));
+ ++fixed_offset;
+ } else {
+ const int source_start = source_rune_indices[compact_offset];
+ const int source_end = source_rune_indices[compact_end - 1] +
1;
+ addCandidate(TermItem(token, source_start, source_end,
position_));
+ }
+ compact_offset = compact_end;
}
+ DORIS_CHECK_EQ(compact_offset, source_rune_indices.size());
} else {
// Treat the entire ASCII buffer as a single token
position_++;
- addCandidate(TermItem(ascii_buffer, start_pos, end_pos, position_));
+ addCandidate(TermItem(ascii_buffer, source_rune_indices.front(),
+ source_rune_indices.back() + 1, position_));
}
}
void PinyinFilter::setTokenAttributes(Token* token, const std::string& term,
int start_offset,
int end_offset, int position) {
set_text(token, term);
- token->setStartOffset(start_offset);
- token->setEndOffset(end_offset);
+ int absolute_start = current_start_offset_;
+ int absolute_end = current_end_offset_;
Review Comment:
Fixed in 0fa73bb034d. KeywordTokenizer now publishes [0, length) and
StandardTokenizer publishes absolute scanner spans across refills and reset, so
Pinyin whole-token alternatives inherit source spans rather than default or
stale values. PinyinFilterTest adds exact keyword alternatives for 刘德华 plus
multi-token standard reset/reuse assertions. The ASAN IndexPolicyMgr,
PinyinFilter, and ICU target passes 71/71 tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
