airborne12 commented on code in PR #70262:
URL: https://github.com/apache/doris/pull/70262#discussion_r4559254565
##########
be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp:
##########
@@ -43,25 +43,110 @@
#include "storage/index/inverted/analysis/analyzer/kuromoji/KuromojiAnalyzer.h"
#include
"storage/index/inverted/analysis/analyzer/kuromoji/kuromoji_normalize.h"
#include
"storage/index/inverted/analysis/char_filter/char_replace_char_filter_factory.h"
+#include "storage/index/inverted/analysis/gram/gram_scheme.h"
+#include "storage/index/inverted/analysis/scratch.h"
#include "storage/index/inverted/analysis/token_filter/lower_case_filter.h"
+#include "storage/index/inverted/clucene/lucene_backed_text_analyzer.h"
#include "util/string_util.h"
namespace doris::segment_v2::inverted_index {
namespace {
+// The parser of a configuration whose analyzer is builtin, named by the
parser or by the analyzer.
+InvertedIndexParserType builtin_parser_type(const InvertedIndexAnalyzerConfig&
config) {
+ return config.analyzer_name.empty()
+ ? config.parser_type
+ :
get_inverted_index_parser_type_from_string(config.analyzer_name);
+}
+
+TextAnalyzerPtr builtin_text_analyzer(const InvertedIndexAnalyzerConfig&
config) {
+ return std::make_shared<LuceneBackedTextAnalyzer>(
+
InvertedIndexAnalyzer::create_builtin_analyzer(builtin_parser_type(config),
+ config.parser_mode,
config.lower_case,
+ config.stop_words));
+}
+
class BuiltinAnalyzerProvider final : public AnalyzerProvider {
public:
- explicit BuiltinAnalyzerProvider(InvertedIndexAnalyzerConfig config)
- : _analyzer(InvertedIndexAnalyzer::create_builtin_analyzer(
- config.analyzer_name.empty()
- ? config.parser_type
- :
get_inverted_index_parser_type_from_string(config.analyzer_name),
- config.parser_mode, config.lower_case,
config.stop_words)) {}
+ explicit BuiltinAnalyzerProvider(const InvertedIndexAnalyzerConfig& config)
+ : _analyzer(builtin_text_analyzer(config)) {}
- AnalyzerPtr get_analyzer() const override { return _analyzer; }
+ TextAnalyzerPtr get_analyzer() const override { return _analyzer; }
private:
- const AnalyzerPtr _analyzer;
+ const TextAnalyzerPtr _analyzer;
+};
+
+// The index's char filter in front of an analyzer. The writers and the query
side used to apply it
+// through the reader they gave the analyzer (create_reader); it now belongs
to the analyzer, so
+// every consumer reads every value through it, whatever the analyzer: a
normalizer index with
+// char_filter_pattern "-" writes "A-B" as "a b". It runs the CLucene char
filter over the whole
+// value until the char filters are native.
+class IndexCharFilterStream final : public TextTokenStream {
+public:
+ IndexCharFilterStream(const CharFilterMap* char_filters,
std::unique_ptr<TextTokenStream> in)
+ : _char_filters(char_filters), _in(std::move(in)) {}
+
+ void reset(std::string_view text) override {
+ // Assigning a result short enough for the string's inline buffer
keeps the buffer a long
+ // value grew, so release it first when it is above the high water
mark.
+ release_oversized_scratch(_filtered);
+ _filtered = InvertedIndexAnalyzer::filter_characters(*_char_filters,
text);
Review Comment:
Right: until the char filters are native, this step runs the CLucene reader
chain over every value, with the copies and allocations you list. #70263
replaces `IndexCharFilterStream` with `CharFilteredStream` over a native
`char_replace`: `CharReplaceCharFilter::apply()` returns the value itself when
no byte matches, and otherwise copies it once into the stream's scratch string,
which is reused from value to value and released above the high water mark, so
the SNII and V1/V2 writers allocate nothing per value in the steady state.
Since this PR is merged, that is where the fix lands.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
