This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3d8440a1b7 [Feature-WIP](inverted index) support phrase for inverted
index writer (#20193)
3d8440a1b7 is described below
commit 3d8440a1b7cbcdfe740093a42f8b6d6177c091a5
Author: airborne12 <[email protected]>
AuthorDate: Tue May 30 17:07:45 2023 +0800
[Feature-WIP](inverted index) support phrase for inverted index writer
(#20193)
---
be/src/olap/inverted_index_parser.cpp | 9 +++++++++
be/src/olap/inverted_index_parser.h | 6 ++++++
be/src/olap/rowset/segment_v2/inverted_index_writer.cpp | 6 ++++++
3 files changed, 21 insertions(+)
diff --git a/be/src/olap/inverted_index_parser.cpp
b/be/src/olap/inverted_index_parser.cpp
index e920a4a930..d1e04e9df1 100644
--- a/be/src/olap/inverted_index_parser.cpp
+++ b/be/src/olap/inverted_index_parser.cpp
@@ -70,4 +70,13 @@ std::string get_parser_mode_string_from_properties(
return INVERTED_INDEX_PARSER_FINE_GRANULARITY;
}
}
+
+std::string get_parser_phrase_support_string_from_properties(
+ const std::map<std::string, std::string>& properties) {
+ if (properties.find(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY) !=
properties.end()) {
+ return properties.at(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY);
+ } else {
+ return INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
+ }
+}
} // namespace doris
diff --git a/be/src/olap/inverted_index_parser.h
b/be/src/olap/inverted_index_parser.h
index aeb3232c39..87e2ef991a 100644
--- a/be/src/olap/inverted_index_parser.h
+++ b/be/src/olap/inverted_index_parser.h
@@ -49,6 +49,10 @@ const std::string INVERTED_INDEX_PARSER_STANDARD =
"standard";
const std::string INVERTED_INDEX_PARSER_ENGLISH = "english";
const std::string INVERTED_INDEX_PARSER_CHINESE = "chinese";
+const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY = "support_phrase";
+const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES = "true";
+const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO = "false";
+
std::string inverted_index_parser_type_to_string(InvertedIndexParserType
parser_type);
InvertedIndexParserType get_inverted_index_parser_type_from_string(const
std::string& parser_str);
@@ -56,5 +60,7 @@ InvertedIndexParserType
get_inverted_index_parser_type_from_string(const std::st
std::string get_parser_string_from_properties(const std::map<std::string,
std::string>& properties);
std::string get_parser_mode_string_from_properties(
const std::map<std::string, std::string>& properties);
+std::string get_parser_phrase_support_string_from_properties(
+ const std::map<std::string, std::string>& properties);
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 253a187fa4..019499c54f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -182,6 +182,12 @@ public:
field_config |= int(lucene::document::Field::INDEX_TOKENIZED);
}
_field = new lucene::document::Field(_field_name.c_str(),
field_config);
+ if
(get_parser_phrase_support_string_from_properties(_index_meta->properties()) ==
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
+ _field->setOmitTermFreqAndPositions(false);
+ } else {
+ _field->setOmitTermFreqAndPositions(true);
+ }
_doc->add(*_field);
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
