github-actions[bot] commented on code in PR #70262:
URL: https://github.com/apache/doris/pull/70262#discussion_r4494653971
##########
be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp:
##########
@@ -43,25 +43,106 @@
#include "storage/index/inverted/analysis/analyzer/kuromoji/KuromojiAnalyzer.h"
#include
"storage/index/inverted/analysis/analyzer/kuromoji/kuromoji_normalize.h"
#include
"storage/index/inverted/analysis/char_filter/char_replace_char_filter_factory.h"
+#include "storage/index/inverted/analysis/gram/gram_scheme.h"
#include "storage/index/inverted/analysis/token_filter/lower_case_filter.h"
+#include "storage/index/inverted/clucene/lucene_backed_text_analyzer.h"
#include "util/string_util.h"
namespace doris::segment_v2::inverted_index {
namespace {
+// The parser of a configuration whose analyzer is builtin, named by the
parser or by the analyzer.
+InvertedIndexParserType builtin_parser_type(const InvertedIndexAnalyzerConfig&
config) {
+ return config.analyzer_name.empty()
+ ? config.parser_type
+ :
get_inverted_index_parser_type_from_string(config.analyzer_name);
+}
+
+TextAnalyzerPtr builtin_text_analyzer(const InvertedIndexAnalyzerConfig&
config) {
+ return std::make_shared<LuceneBackedTextAnalyzer>(
+
InvertedIndexAnalyzer::create_builtin_analyzer(builtin_parser_type(config),
+ config.parser_mode,
config.lower_case,
+ config.stop_words));
+}
+
class BuiltinAnalyzerProvider final : public AnalyzerProvider {
public:
- explicit BuiltinAnalyzerProvider(InvertedIndexAnalyzerConfig config)
- : _analyzer(InvertedIndexAnalyzer::create_builtin_analyzer(
- config.analyzer_name.empty()
- ? config.parser_type
- :
get_inverted_index_parser_type_from_string(config.analyzer_name),
- config.parser_mode, config.lower_case,
config.stop_words)) {}
+ explicit BuiltinAnalyzerProvider(const InvertedIndexAnalyzerConfig& config)
+ : _analyzer(builtin_text_analyzer(config)) {}
+
+ TextAnalyzerPtr get_analyzer() const override { return _analyzer; }
+
+private:
+ const TextAnalyzerPtr _analyzer;
+};
+
+// The index's char filter in front of an analyzer. The writers and the query
side used to apply it
+// through the reader they gave the analyzer (create_reader); it now belongs
to the analyzer, so
+// every consumer reads every value through it, whatever the analyzer: a
normalizer index with
+// char_filter_pattern "-" writes "A-B" as "a b". It runs the CLucene char
filter over the whole
+// value until the char filters are native.
+class IndexCharFilterStream final : public TextTokenStream {
+public:
+ IndexCharFilterStream(const CharFilterMap* char_filters,
std::unique_ptr<TextTokenStream> in)
+ : _char_filters(char_filters), _in(std::move(in)) {}
+
+ void reset(std::string_view text) override {
+ _filtered = InvertedIndexAnalyzer::filter_characters(*_char_filters,
text);
Review Comment:
[P2] Translate CLucene failures from the outer char filter before returning
from reset(). `filter_characters()` still invokes CLucene Reader init/readCopy,
but this new wrapper lets a CLuceneError escape the TextTokenStream contract
(which promises only doris::Exception). The callers whose catches were removed
in this PR therefore bypass their analyzer-error Status path and can leak an
exception during any failing/invalid char-filter value. Catch and wrap it as
INVERTED_INDEX_ANALYZER_ERROR, as LuceneBackedTokenStream does.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
