airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4003075715
##########
be/src/storage/index/inverted/char_filter/icu_normalizer_char_filter.cpp:
##########
@@ -59,33 +60,68 @@ void ICUNormalizerCharFilter::fill() {
input.resize(_reader->size());
_reader->readCopy(input.data(), 0, static_cast<int32_t>(input.size()));
normalize_text(input, _buf);
+ build_source_byte_offset_map();
_transformed_input.init(_buf.data(), static_cast<int32_t>(_buf.size()),
false);
}
void ICUNormalizerCharFilter::normalize_text(const std::string& input,
std::string& output) {
output.clear();
+ _edits.reset();
if (input.empty()) {
return;
}
UErrorCode status = U_ZERO_ERROR;
- icu::UnicodeString src16 = icu::UnicodeString::fromUTF8(input);
- UNormalizationCheckResult quick_result = _normalizer->quickCheck(src16,
status);
- if (U_SUCCESS(status) && quick_result == UNORM_YES) {
- output = input;
- return;
- }
-
- icu::UnicodeString result16;
- status = U_ZERO_ERROR;
- _normalizer->normalize(src16, result16, status);
+ icu::StringByteSink<std::string> sink(&output);
+ _normalizer->normalizeUTF8(0, icu::StringPiece(input), sink, &_edits,
status);
if (U_FAILURE(status)) {
LOG(WARNING) << "ICU normalize failed: " << u_errorName(status) << ",
using original text";
output = input;
+ _edits.reset();
+ _edits.addUnchanged(static_cast<int32_t>(input.size()));
return;
}
+}
+
+void ICUNormalizerCharFilter::build_source_byte_offset_map() {
+ _source_byte_offsets.clear();
+ _source_byte_offsets.reserve(_buf.size() + 1);
+ _source_byte_offsets.push_back(0);
+
+ UErrorCode status = U_ZERO_ERROR;
+ auto iterator = _edits.getFineIterator();
+ while (iterator.next(status)) {
+ if (U_FAILURE(status) ||
+ iterator.destinationIndex() !=
static_cast<int32_t>(_source_byte_offsets.size() - 1)) {
+ _source_byte_offsets.clear();
+ return;
+ }
+
+ const int32_t source_start = iterator.sourceIndex();
+ const int32_t source_end = source_start + iterator.oldLength();
+ if (iterator.hasChange()) {
+ // ICU maps the start of a replacement to the start of its source
span, and every
+ // later destination boundary in that replacement to the end of
the source span.
+ for (int32_t i = 0; i < iterator.newLength(); ++i) {
Review Comment:
Fixed in 0fa73bb034d. ICU correction runs map a zero-length edit at its
shared destination boundary to the deleted source span end, and continue to
delegate through nested character filters. PinyinFilterTest now covers nfkc_cf
removal of U+00AD through nested ICU -> IK -> Pinyin, including reset, with liu
[0,5) and de [5,7). The focused ASAN target passes 71/71 tests.
##########
be/src/storage/index/inverted/char_filter/icu_normalizer_char_filter.cpp:
##########
@@ -59,33 +60,68 @@ void ICUNormalizerCharFilter::fill() {
input.resize(_reader->size());
_reader->readCopy(input.data(), 0, static_cast<int32_t>(input.size()));
normalize_text(input, _buf);
+ build_source_byte_offset_map();
_transformed_input.init(_buf.data(), static_cast<int32_t>(_buf.size()),
false);
}
void ICUNormalizerCharFilter::normalize_text(const std::string& input,
std::string& output) {
output.clear();
+ _edits.reset();
if (input.empty()) {
return;
}
UErrorCode status = U_ZERO_ERROR;
- icu::UnicodeString src16 = icu::UnicodeString::fromUTF8(input);
- UNormalizationCheckResult quick_result = _normalizer->quickCheck(src16,
status);
- if (U_SUCCESS(status) && quick_result == UNORM_YES) {
- output = input;
- return;
- }
-
- icu::UnicodeString result16;
- status = U_ZERO_ERROR;
- _normalizer->normalize(src16, result16, status);
+ icu::StringByteSink<std::string> sink(&output);
+ _normalizer->normalizeUTF8(0, icu::StringPiece(input), sink, &_edits,
status);
if (U_FAILURE(status)) {
LOG(WARNING) << "ICU normalize failed: " << u_errorName(status) << ",
using original text";
output = input;
+ _edits.reset();
+ _edits.addUnchanged(static_cast<int32_t>(input.size()));
return;
}
+}
+
+void ICUNormalizerCharFilter::build_source_byte_offset_map() {
+ _source_byte_offsets.clear();
+ _source_byte_offsets.reserve(_buf.size() + 1);
Review Comment:
Fixed in 0fa73bb034d. ICU offset correction now stores compact repeated edit
runs and uses binary search, rather than retaining an int32 entry for each
normalized byte. SparseOffsetCorrectionsRemainCompact uses an 8 MiB sparse-edit
input, asserts one correction run, and verifies the changed-byte boundaries.
The focused ASAN target passes 71/71 tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
