This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2b64096faf4 [improvement](be) Move jieba into the Doris tree and cut
chinese text over a flat dictionary (#70266)
2b64096faf4 is described below
commit 2b64096faf4d796d5f836a88f6a22f2d4588c0ef
Author: Jack <[email protected]>
AuthorDate: Sat Nov 21 19:10:44 2026 +0800
[improvement](be) Move jieba into the Doris tree and cut chinese text over
a flat dictionary (#70266)
### What problem does this PR solve?
Issue Number: None
Related PR: #70265 (depends on)
Problem Summary: Move jieba into the Doris tree for the chinese parser
and replace its trie of hash maps with flat arrays: 3.2x (coarse) and
2.3x (fine) as fast, less memory, the same tokens.
---
.licenserc.yaml | 5 +
LICENSE.txt | 19 +
.../index/inverted/analysis/analyzer/analyzer.cpp | 58 +--
.../inverted/analysis/tokenizer/jieba/dict_trie.h | 200 +++++++++
.../inverted/analysis/tokenizer/jieba/flat_dict.h | 349 +++++++++++++++
.../inverted/analysis/tokenizer/jieba/hmm_model.h | 160 +++++++
.../inverted/analysis/tokenizer/jieba/jieba.h | 85 ++++
.../analysis/tokenizer/jieba/jieba_scratch.h | 61 +++
.../analysis/tokenizer/jieba/jieba_tokenizer.cpp | 112 +++++
.../analysis/tokenizer/jieba/jieba_tokenizer.h | 102 +++++
.../inverted/analysis/tokenizer/jieba/segmenter.h | 408 +++++++++++++++++
.../analysis/tokenizer/jieba/string_util.h | 70 +++
.../inverted/analysis/tokenizer/jieba/unicode.h | 36 ++
.../inverted/analysis/component_stream_test.cpp | 13 +
.../analysis/tokenizer/jieba_tokenizer_test.cpp | 496 +++++++++++++++++++++
build-support/check-header-deps.py | 4 -
dist/LICENSE-dist.txt | 2 +
dist/licenses/LICENSE-cppjieba.txt | 20 +
dist/licenses/LICENSE-limonp.txt | 20 +
19 files changed, 2167 insertions(+), 53 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 5cc8026f6e4..f8ae024a78f 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -135,6 +135,11 @@ header:
- "be/src/util/sse2neo.h"
- "be/src/util/sse2neon.h"
- "be/src/storage/index/inverted/analysis/analyzer/kuromoji/dict/darts.h"
+ - "be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+ - "be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+ - "be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h"
+ - "be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h"
+ - "be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
- "be/src/pch/*"
- "be/test/data"
- "be/test/expected_result"
diff --git a/LICENSE.txt b/LICENSE.txt
index b54c0b65503..5f50887ef91 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -378,6 +378,25 @@
be/src/storage/index/inverted/analysis/analyzer/kuromoji/dict/darts.h : 2-Clause
--------------------------------------------------------------------------------
+be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h
+be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h
+be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h
+be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h : MIT license
(cppjieba)
+
+ Copyright (c) 2013
+ Derived from https://github.com/yanyiwu/cppjieba
+ Full text: dist/licenses/LICENSE-cppjieba.txt
+
+--------------------------------------------------------------------------------
+
+be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h : MIT
license (limonp)
+
+ Copyright (c) 2013
+ Derived from https://github.com/yanyiwu/limonp
+ Full text: dist/licenses/LICENSE-limonp.txt
+
+--------------------------------------------------------------------------------
+
be/src/util/murmur_hash3.h
be/src/util/murmur_hash3.cpp
diff --git a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
index 1a837527954..ef7b1eefdc5 100644
--- a/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
+++ b/be/src/storage/index/inverted/analysis/analyzer/analyzer.cpp
@@ -19,21 +19,6 @@
#include <glog/logging.h>
-#include "CLucene.h"
-#include "CLucene/analysis/LanguageBasedAnalyzer.h"
-
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wshadow-field"
-#endif
-// clang-format off
-#include "common/compile_check_avoid_begin.h"
-#include "CLucene/analysis/standard95/StandardAnalyzer.h"
-#include "common/compile_check_avoid_end.h"
-// clang-format on
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
#include "runtime/exec_env.h"
#include "runtime/index_policy/index_policy_mgr.h"
#include "storage/index/inverted/analysis/analyzer/chain_analyzer.h"
@@ -51,9 +36,9 @@
#include
"storage/index/inverted/analysis/tokenizer/english/english_tokenizer.h"
#include
"storage/index/inverted/analysis/tokenizer/icu/icu_tokenizer_factory.h"
#include "storage/index/inverted/analysis/tokenizer/ik/ik_tokenizer_factory.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
#include
"storage/index/inverted/analysis/tokenizer/kuromoji/kuromoji_tokenizer_factory.h"
#include "storage/index/inverted/analysis/tokenizer/standard/uax29_tokenizer.h"
-#include "storage/index/inverted/clucene/lucene_backed_text_analyzer.h"
namespace doris::segment_v2::inverted_index {
namespace {
@@ -188,23 +173,6 @@ bool builtin_lowercase(const std::string& lower_case, bool
parser_default) {
return parser_default;
}
-// A builtin parser whose analyzer is still CLucene's, with lower_case and
stop_words applied.
-TextAnalyzerPtr
lucene_builtin_analyzer(std::shared_ptr<lucene::analysis::Analyzer> analyzer,
- const std::string& lower_case,
- const std::string& stop_words) {
- if (lower_case == INVERTED_INDEX_PARSER_TRUE) {
- analyzer->set_lowercase(true);
- } else if (lower_case == INVERTED_INDEX_PARSER_FALSE) {
- analyzer->set_lowercase(false);
- }
- if (stop_words == "none") {
- analyzer->set_stopwords(nullptr);
- } else {
- analyzer->set_stopwords(&lucene::analysis::standard95::stop_words);
- }
- return std::make_shared<LuceneBackedTextAnalyzer>(std::move(analyzer));
-}
-
// `tokenizer`, followed by the lowercase filter when `lowercase`.
TextAnalyzerPtr tokenizer_chain(TokenizerFactoryPtr tokenizer, bool lowercase)
{
std::vector<TokenFilterFactoryPtr> token_filters;
@@ -230,15 +198,13 @@ TextAnalyzerPtr
build_builtin_analyzer(InvertedIndexParserType parser_type,
std::vector<TokenFilterFactoryPtr> {});
}
if (parser_type == InvertedIndexParserType::PARSER_CHINESE) {
- auto chinese_analyzer =
-
std::make_shared<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
- chinese_analyzer->initDict(config::inverted_index_dict_path);
- if (parser_mode == INVERTED_INDEX_PARSER_COARSE_GRANULARITY) {
- chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::Default);
- } else {
- chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::All);
- }
- return lucene_builtin_analyzer(std::move(chinese_analyzer),
lower_case, stop_words);
+ // The fine mode cuts every dictionary word; coarse_grained mixes the
dictionary and the HMM.
+ return std::make_shared<ChainAnalyzer>(
+ std::vector<CharFilterFactoryPtr> {},
+ std::make_shared<JiebaTokenizerFactory>(
+ parser_mode !=
INVERTED_INDEX_PARSER_COARSE_GRANULARITY,
+ builtin_lowercase(lower_case, false),
config::inverted_index_dict_path),
+ std::vector<TokenFilterFactoryPtr> {});
}
if (parser_type == InvertedIndexParserType::PARSER_ICU) {
return tokenizer_chain(std::make_shared<ICUTokenizerFactory>(),
@@ -300,13 +266,7 @@ TextAnalyzerPtr
InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexPars
const
std::string& parser_mode,
const
std::string& lower_case,
const
std::string& stop_words) {
- try {
- return build_builtin_analyzer(parser_type, parser_mode, lower_case,
stop_words);
- } catch (const CLuceneError& e) {
- throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
- "building the {} analyzer failed: {}",
- inverted_index_parser_type_to_string(parser_type),
e.what());
- }
+ return build_builtin_analyzer(parser_type, parser_mode, lower_case,
stop_words);
}
TextAnalyzerPtr InvertedIndexAnalyzer::create_analyzer(const
InvertedIndexAnalyzerConfig* config) {
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h
new file mode 100644
index 00000000000..fca697ca7ab
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h
@@ -0,0 +1,200 @@
+// Derived from cppjieba's DictTrie.hpp (https://github.com/yanyiwu/cppjieba),
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-cppjieba.txt.
+
+#pragma once
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <cstdlib>
+#include <fstream>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/string_util.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+inline constexpr double MIN_DOUBLE = -3.14e+100;
+inline constexpr size_t DICT_COLUMN_NUM = 3;
+
+// A dictionary word and its weight. Upstream's part-of-speech tag is gone:
only tagging read it.
+struct DictUnit {
+ Unicode word;
+ double weight = 0.0;
+};
+
+// The words of the main dictionary and the user dictionaries, with their
weights. FlatDict builds
+// the trie the cuts walk out of them; upstream built a trie of pointers to
these units here.
+//
+// A main dictionary line is "word frequency tag", three columns split at
single spaces, with the
+// frequency read by atof; any other line fails the load. A word's weight is
log(frequency / the sum
+// of all frequencies). A user dictionary line is "word", "word tag" or "word
frequency tag"; the
+// first two take the median weight of the main dictionary (upstream's default
WordWeightMedian, the
+// only option Jieba used), the third log(frequency / the main dictionary's
sum), with the frequency
+// read by atoi. A user line with more columns adds a unit with no word, and
every user word of one
+// rune is remembered in user_dict_single_chinese_words(), whatever its script.
+class DictTrie {
+public:
+ // `user_dict_paths` lists the user dictionaries, separated by '|' or ';'.
+ DictTrie(const std::string& dict_path, const std::string& user_dict_paths)
{
+ init(dict_path, user_dict_paths);
+ }
+
+ // One unit for every line of the main dictionary and then of the user
dictionaries, in the
+ // order of the files and their lines. A unit whose word did not decode
has an empty word and
+ // is no word of the trie; of several units of the same word, the trie
takes the last, which
+ // lets a user word replace the main dictionary's unit of that word, as
upstream's trie did
+ // when it inserted the units in this order.
+ const std::vector<DictUnit>& units() const { return _static_node_infos; }
+
+ const std::unordered_set<Rune>& user_dict_single_chinese_words() const {
+ return _user_dict_single_chinese_word;
+ }
+
+ double get_min_weight() const { return _min_weight; }
+
+private:
+ // Upstream also shrank the units to fit before it pointed its trie into
them; nothing keeps
+ // them now.
+ void init(const std::string& dict_path, const std::string&
user_dict_paths) {
+ load_dict(dict_path);
+ _freq_sum = calc_freq_sum(_static_node_infos);
+ calculate_weight(_static_node_infos, _freq_sum);
+ set_static_word_weights(dict_path);
+
+ // Upstream skipped this for an empty list, which splits into no file
anyway.
+ load_user_dict(user_dict_paths);
+ }
+
+ // Decodes `word` into the unit and sets its weight. A word that does not
decode leaves the
+ // unit with an empty word and its weight as it was, and returns false; no
caller looks at the
+ // result, as upstream's did not.
+ static bool make_node_info(DictUnit& node_info, const std::string& word,
double weight) {
+ if (!decode_unicode(word, node_info.word)) {
+ return false;
+ }
+ node_info.weight = weight;
+ return true;
+ }
+
+ void load_dict(const std::string& file_path) {
+ std::ifstream ifs(file_path);
+ if (!ifs.is_open()) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "jieba main dictionary file not found: {}",
file_path);
+ }
+ std::string line;
+ std::vector<std::string> buf;
+
+ // One unit for every line, as upstream: a line whose word does not
decode adds a unit with
+ // no word and the frequency of the line before it, which then counts
twice in the sum of
+ // the frequencies. (A first line that does not decode added an
indeterminate frequency
+ // upstream, and adds 0 here.)
+ DictUnit node_info;
+ while (std::getline(ifs, line)) {
+ split(line, buf, " ");
+ if (buf.size() != DICT_COLUMN_NUM) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "jieba main dictionary file {} has a line that
is not {} columns "
+ "split at spaces: {}",
+ file_path, DICT_COLUMN_NUM, line);
+ }
+ make_node_info(node_info, buf[0], std::atof(buf[1].c_str()));
+ _static_node_infos.push_back(node_info);
+ }
+ }
+
+ // A user dictionary file that cannot be opened fails the load; an empty
line is skipped.
+ void load_user_dict(const std::string& file_paths) {
+ const std::vector<std::string> files = split(file_paths, "|;");
+ for (const std::string& file : files) {
+ std::ifstream ifs(file);
+ if (!ifs.is_open()) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "jieba user dictionary file not found: {}",
file);
+ }
+ std::string line;
+ while (std::getline(ifs, line)) {
+ if (line.empty()) {
+ continue;
+ }
+ insert_user_dict_node(line);
+ }
+ }
+ }
+
+ // Upstream's InserUserDictNode.
+ void insert_user_dict_node(const std::string& line) {
+ std::vector<std::string> buf;
+ DictUnit node_info;
+ split(line, buf, " ");
+ if (buf.size() == 1 || buf.size() == 2) {
+ // The second column is the tag.
+ make_node_info(node_info, buf[0], _user_word_default_weight);
+ } else if (buf.size() == 3) {
+ int freq = std::atoi(buf[1].c_str());
+ DCHECK(_freq_sum > 0.0);
+ double weight = std::log(1.0 * freq / _freq_sum);
+ make_node_info(node_info, buf[0], weight);
+ }
+ _static_node_infos.push_back(node_info);
+ if (node_info.word.size() == 1) {
+ _user_dict_single_chinese_word.insert(node_info.word[0]);
+ }
+ }
+
+ static bool weight_compare(const DictUnit& lhs, const DictUnit& rhs) {
+ return lhs.weight < rhs.weight;
+ }
+
+ // Sorts a copy of the main dictionary's units by weight for the smallest
and the median, as
+ // upstream; the median is the default weight of a user word. Only these
two order statistics
+ // are read, which any sort gives alike; std::ranges::sort is libstdc++'s
std::sort anyway.
+ void set_static_word_weights(const std::string& dict_path) {
+ if (_static_node_infos.empty()) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "jieba main dictionary file {} is empty",
dict_path);
+ }
+ std::vector<DictUnit> x = _static_node_infos;
+ std::ranges::sort(x, weight_compare);
+ _min_weight = x[0].weight;
+ _median_weight = x[x.size() / 2].weight;
+ _user_word_default_weight = _median_weight;
+ }
+
+ // In the order of the file.
+ static double calc_freq_sum(const std::vector<DictUnit>& node_infos) {
+ double sum = 0.0;
+ for (const DictUnit& node_info : node_infos) {
+ sum += node_info.weight;
+ }
+ return sum;
+ }
+
+ // The DCHECKs are upstream's asserts. A debug build stops at a frequency,
or a sum of them, of
+ // 0 or less, as upstream's did, and so at an empty dictionary before
set_static_word_weights()
+ // can throw; a release build loads such a dictionary with weights of -inf
or NaN, as upstream's
+ // did, and throws for an empty one.
+ static void calculate_weight(std::vector<DictUnit>& node_infos, double
sum) {
+ DCHECK(sum > 0.0);
+ for (DictUnit& node_info : node_infos) {
+ DCHECK(node_info.weight > 0.0);
+ node_info.weight = std::log(node_info.weight / sum);
+ }
+ }
+
+ std::vector<DictUnit> _static_node_infos;
+
+ double _freq_sum = 0.0;
+ double _min_weight = 0.0;
+ double _median_weight = 0.0;
+ double _user_word_default_weight = 0.0;
+ std::unordered_set<Rune> _user_dict_single_chinese_word;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h
new file mode 100644
index 00000000000..772ce375265
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h
@@ -0,0 +1,349 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// What the two cuts read, built once from the loaded dictionaries, model and
stop words, which
+// can be dropped then, and laid out so that every lookup of a cut is an array
probe or a short
+// scan instead of a hash lookup behind a pointer.
+//
+// The dictionary is a trie of flat arrays. Node 0 is the root and the other
nodes are numbered
+// breadth first, so the children of node n are the consecutive nodes
child_begin[n] to
+// child_begin[n + 1] - 1, in increasing rune, and key[c] is the rune that
leads to child c. A
+// node at which a word ends has has_value set and the word's weight. Say the
words are A, AB, ABC
+// and C, each letter a rune and A < C: the root's children are 1 (A) and 2
(C), node 1's child is
+// 3 (B) and node 3's is 4 (C); child_begin is {1, 3, 4, 4, 5, 5}, so node 2
has 4 - 4 = no child,
+// and has_value is set for nodes 1 to 4.
+//
+// The trie keeps the words of the units with a word, the last unit of a word
where there are
+// several, which is the trie of pointers upstream built by inserting the
units in order. Nothing
+// is cached per value: the cuts keep what a value needs in a JiebaScratch.
+class FlatDict {
+public:
+ // The root, which is nobody's child, also stands for no node.
+ static constexpr uint32_t kNoNode = 0;
+
+ FlatDict(const DictTrie& dict_trie, const HMMModel& model, const
std::string& stop_word_path)
+ : _min_weight(dict_trie.get_min_weight()),
+ _start_prob(model.start_prob),
+ _trans_prob(model.trans_prob) {
+ build_trie(dict_trie.units());
+ build_user_single_words(dict_trie.user_dict_single_chinese_words());
+ build_emit_prob(model);
+ load_stop_words(stop_word_path);
+ }
+ FlatDict(const FlatDict&) = delete;
+ FlatDict& operator=(const FlatDict&) = delete;
+
+ // The child of the root for `rune`, or kNoNode: a table for the first
plane, a search among
+ // the root's children above it.
+ uint32_t root_child(Rune rune) const {
+ return rune < kDirectRunes ? _root_children[rune] : child(kNoNode,
rune);
+ }
+
+ // The child of `node` for `rune`, or kNoNode. A node of up to
kLinearChildren children is
+ // scanned, one of more is searched by bisection.
+ uint32_t child(uint32_t node, Rune rune) const {
+ const uint32_t begin = _child_begin[node];
+ const uint32_t end = _child_begin[node + 1];
+ if (end - begin <= kLinearChildren) {
+ for (uint32_t c = begin; c < end; ++c) {
+ if (_key[c] == rune) {
+ return c;
+ }
+ }
+ return kNoNode;
+ }
+ const Rune* const first = _key.data() + begin;
+ const Rune* const last = _key.data() + end;
+ const Rune* const it = std::lower_bound(first, last, rune);
+ return it != last && *it == rune ? static_cast<uint32_t>(it -
_key.data()) : kNoNode;
+ }
+
+ // Whether a word ends at `node`, and its weight.
+ bool has_value(uint32_t node) const { return _has_value[node] != 0; }
+ double weight(uint32_t node) const { return _weight[node]; }
+
+ // The weight of a rune that is no word: the smallest weight of the main
dictionary.
+ double min_weight() const { return _min_weight; }
+
+ // Whether a user dictionary has `rune` alone as a word, upstream's
+ // IsUserDictSingleChineseWord.
+ bool is_user_single_word(Rune rune) const {
+ if (rune < kDirectRunes) {
+ return ((_user_single_words[rune / 64] >> (rune % 64)) & 1U) != 0;
+ }
+ return _user_single_supplementary.contains(rune);
+ }
+
+ const std::array<double, HMMModel::STATUS_SUM>& start_prob() const {
return _start_prob; }
+ const std::array<std::array<double, HMMModel::STATUS_SUM>,
HMMModel::STATUS_SUM>& trans_prob()
+ const {
+ return _trans_prob;
+ }
+
+ // The emission probabilities of `rune` for B, E, M and S, in this order,
with MIN_DOUBLE for a
+ // status the model gives no probability for `rune`, as upstream's
GetEmitProb.
+ const double* emit_prob(Rune rune) const {
+ if (rune < kDirectRunes) {
+ return _emit_prob.data() + size_t {rune} * HMMModel::STATUS_SUM;
+ }
+ const auto it = _supplementary_emit_prob.find(rune);
+ return it == _supplementary_emit_prob.end() ? kNoEmitProb.data() :
it->second.data();
+ }
+
+ // Whether `word` is a line of the stop word file. A word of up to
kPackedStopWordBytes bytes
+ // is probed for in an open-addressing table keyed by its bytes, a longer
one searched for among
+ // the longer stop words.
+ bool is_stop_word(std::string_view word) const {
+ if (word.size() > _longest_stop_word) {
+ return false;
+ }
+ if (word.size() > kPackedStopWordBytes) {
+ return std::ranges::binary_search(_long_stop_words, word);
+ }
+ const uint64_t bytes = pack_bytes(word);
+ const size_t mask = _stop_word_slots.size() - 1;
+ for (size_t slot = stop_word_slot(bytes, word.size());; slot = (slot +
1) & mask) {
+ const StopWordSlot& entry = _stop_word_slots[slot];
+ if (entry.size == 0) {
+ return false;
+ }
+ if (entry.bytes == bytes && entry.size == word.size()) {
+ return true;
+ }
+ }
+ }
+
+private:
+ // The runes below this have a table of their own: the first rune of a
word, whether a user
+ // word is that rune alone, and the emission probabilities.
+ static constexpr Rune kDirectRunes = 0x10000;
+ static constexpr uint32_t kLinearChildren = 8;
+ static constexpr size_t kPackedStopWordBytes = 8;
+ // 30 times the 542 distinct stop words of up to 8 bytes in the shipped
file, so that a probe
+ // nearly always ends at its first slot; a file with more of them gets a
table of at least
+ // twice as many slots as words, so that a probe always meets an empty
slot.
+ static constexpr size_t kMinStopWordSlots = size_t {1} << 14;
+ static constexpr std::array<double, HMMModel::STATUS_SUM> kNoEmitProb = {
+ MIN_DOUBLE, MIN_DOUBLE, MIN_DOUBLE, MIN_DOUBLE};
+
+ // A stop word of up to 8 bytes; an empty slot has size 0, which no stop
word in the table has.
+ struct StopWordSlot {
+ uint64_t bytes = 0;
+ uint64_t size = 0;
+ };
+
+ // The units the trie keeps, by word, breadth first. For each node the
words below it are a
+ // run of the sorted words, and those below a child are the run that has
the child's rune
+ // after the node's runes; a word that ends at the node sorts first in its
run.
+ void build_trie(const std::vector<DictUnit>& units) {
+ std::vector<uint32_t> words;
+ words.reserve(units.size());
+ for (uint32_t i = 0; i < units.size(); ++i) {
+ if (!units[i].word.empty()) {
+ words.push_back(i);
+ }
+ }
+ // Stable, so that the units of one word stay in the order of the
files; the last of them
+ // is the one upstream's trie kept.
+ std::ranges::stable_sort(words, [&units](uint32_t lhs, uint32_t rhs) {
+ return units[lhs].word < units[rhs].word;
+ });
+ size_t kept = 0;
+ for (size_t i = 0; i < words.size(); ++i) {
+ if (i + 1 == words.size() || units[words[i]].word != units[words[i
+ 1]].word) {
+ words[kept++] = words[i];
+ }
+ }
+ words.resize(kept);
+
+ // The words below each node: words[begin, end), which all start with
the node's `depth`
+ // runes. Nodes are appended in the order they are numbered, so that a
node is handled
+ // after every node of a smaller number has appended its children.
+ struct Below {
+ uint32_t begin;
+ uint32_t end;
+ uint32_t depth;
+ };
+ std::vector<Below> below;
+ below.push_back({.begin = 0, .end =
static_cast<uint32_t>(words.size()), .depth = 0});
+ _key.push_back(0);
+ _has_value.push_back(0);
+ _weight.push_back(0.0);
+ for (size_t node = 0; node < below.size(); ++node) {
+ auto [begin, end, depth] = below[node];
+ _child_begin.push_back(static_cast<uint32_t>(below.size()));
+ if (begin < end && units[words[begin]].word.size() == depth) {
+ ++begin;
+ }
+ while (begin < end) {
+ const DictUnit& first = units[words[begin]];
+ const Rune rune = first.word[depth];
+ uint32_t run_end = begin + 1;
+ while (run_end < end && units[words[run_end]].word[depth] ==
rune) {
+ ++run_end;
+ }
+ const bool ends_here = first.word.size() == depth + 1;
+ below.push_back({.begin = begin, .end = run_end, .depth =
depth + 1});
+ _key.push_back(rune);
+ _has_value.push_back(ends_here ? 1 : 0);
+ _weight.push_back(ends_here ? first.weight : 0.0);
+ begin = run_end;
+ }
+ }
+ _child_begin.push_back(static_cast<uint32_t>(below.size()));
+ _child_begin.shrink_to_fit();
+ _key.shrink_to_fit();
+ _has_value.shrink_to_fit();
+ _weight.shrink_to_fit();
+
+ _root_children.assign(kDirectRunes, kNoNode);
+ for (uint32_t c = _child_begin[kNoNode]; c < _child_begin[kNoNode +
1]; ++c) {
+ if (_key[c] < kDirectRunes) {
+ _root_children[_key[c]] = c;
+ }
+ }
+ }
+
+ void build_user_single_words(const std::unordered_set<Rune>& runes) {
+ _user_single_words.assign(kDirectRunes / 64, 0);
+ for (const Rune rune : runes) {
+ if (rune < kDirectRunes) {
+ _user_single_words[rune / 64] |= uint64_t {1} << (rune % 64);
+ } else {
+ _user_single_supplementary.insert(rune);
+ }
+ }
+ }
+
+ // Rune by rune instead of status by status, so that the four
probabilities Viterbi reads for
+ // a rune are next to each other.
+ void build_emit_prob(const HMMModel& model) {
+ _emit_prob.assign(size_t {kDirectRunes} * HMMModel::STATUS_SUM,
MIN_DOUBLE);
+ for (size_t status = 0; status < HMMModel::STATUS_SUM; ++status) {
+ for (const auto& [rune, prob] : *model.emit_prob_vec[status]) {
+ if (rune < kDirectRunes) {
+ _emit_prob[size_t {rune} * HMMModel::STATUS_SUM + status]
= prob;
+ } else {
+ // A status the model has no pair of the rune for keeps
MIN_DOUBLE.
+ auto& probs =
+ _supplementary_emit_prob.try_emplace(rune,
kNoEmitProb).first->second;
+ probs[status] = prob;
+ }
+ }
+ }
+ }
+
+ // A stop word file that cannot be opened leaves no stop word, as
upstream. Every line is a
+ // stop word as it is, untrimmed; an empty line is one too, but no word is
empty, so it is not
+ // kept.
+ void load_stop_words(const std::string& file_path) {
+ std::vector<std::string> short_words;
+ std::ifstream ifs(file_path);
+ if (ifs.is_open()) {
+ std::string line;
+ while (std::getline(ifs, line)) {
+ _longest_stop_word = std::max(_longest_stop_word, line.size());
+ if (line.size() > kPackedStopWordBytes) {
+ _long_stop_words.push_back(line);
+ } else if (!line.empty()) {
+ short_words.push_back(line);
+ }
+ }
+ }
+ std::ranges::sort(_long_stop_words);
+ const auto [last, end] = std::ranges::unique(_long_stop_words);
+ _long_stop_words.erase(last, end);
+
+ size_t slots = kMinStopWordSlots;
+ while (slots < 2 * short_words.size()) {
+ slots *= 2;
+ }
+ _stop_word_slots.assign(slots, StopWordSlot {});
+ _stop_word_shift = 64 - std::countr_zero(slots);
+ for (const std::string& word : short_words) {
+ const uint64_t bytes = pack_bytes(word);
+ size_t slot = stop_word_slot(bytes, word.size());
+ while (_stop_word_slots[slot].size != 0 &&
+ (_stop_word_slots[slot].bytes != bytes ||
+ _stop_word_slots[slot].size != word.size())) {
+ slot = (slot + 1) & (slots - 1);
+ }
+ _stop_word_slots[slot].bytes = bytes;
+ _stop_word_slots[slot].size = word.size();
+ }
+ }
+
+ // The bytes of a word of at most 8 bytes as one integer, the first byte
lowest.
+ static uint64_t pack_bytes(std::string_view word) {
+ uint64_t bytes = 0;
+ for (size_t i = 0; i < word.size(); ++i) {
+ bytes |= uint64_t {static_cast<uint8_t>(word[i])} << (8 * i);
+ }
+ return bytes;
+ }
+
+ // Fibonacci hashing: the top bits of the product, as many as the table
has slots.
+ size_t stop_word_slot(uint64_t bytes, size_t size) const {
+ return static_cast<size_t>(((bytes ^ size) * 0x9E3779B97F4A7C15ULL) >>
_stop_word_shift);
+ }
+
+ // The trie.
+ std::vector<uint32_t> _child_begin;
+ std::vector<Rune> _key;
+ std::vector<uint8_t> _has_value;
+ std::vector<double> _weight;
+ std::vector<uint32_t> _root_children;
+ double _min_weight = 0.0;
+
+ // The user words of one rune: a bit for each rune of the first plane, a
set above it.
+ std::vector<uint64_t> _user_single_words;
+ std::unordered_set<Rune> _user_single_supplementary;
+
+ // The HMM.
+ std::array<double, HMMModel::STATUS_SUM> _start_prob;
+ std::array<std::array<double, HMMModel::STATUS_SUM>, HMMModel::STATUS_SUM>
_trans_prob;
+ std::vector<double> _emit_prob;
+ std::unordered_map<Rune, std::array<double, HMMModel::STATUS_SUM>>
_supplementary_emit_prob;
+
+ // The stop words.
+ std::vector<StopWordSlot> _stop_word_slots;
+ int _stop_word_shift = 0;
+ std::vector<std::string> _long_stop_words;
+ size_t _longest_stop_word = 0;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h
new file mode 100644
index 00000000000..9322dfec441
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h
@@ -0,0 +1,160 @@
+// Derived from cppjieba's HMMModel.hpp (https://github.com/yanyiwu/cppjieba),
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-cppjieba.txt.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdlib>
+#include <fstream>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+#include "common/exception.h"
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/string_util.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/unicode.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+using EmitProbMap = std::unordered_map<Rune, double>;
+
+// The HMM of jieba's unknown-word segmentation. Every line of the model file
is trimmed of white
+// space, and blank lines and lines that start with '#' are skipped. The rest
are one line of the
+// 4 start probabilities, 4 lines of 4 transition probabilities, all split at
single spaces and
+// read by atof, and one line of emission probabilities for each of B, E, M
and S:
+// "rune:probability" pairs split at ','. A missing or illegal line fails the
load; a later pair of
+// the same rune replaces the earlier one, and what follows the S line is not
read. Upstream's
+// statMap, which only printing read, is gone, and so is GetEmitProb: the cuts
read the model as
+// FlatDict lays it out.
+struct HMMModel {
+ /*
+ * STATUS:
+ * 0: HMMModel::B, 1: HMMModel::E, 2: HMMModel::M, 3:HMMModel::S
+ * */
+ enum { B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4 };
+
+ explicit HMMModel(const std::string& model_path) {
+ emit_prob_vec.push_back(&emit_prob_b);
+ emit_prob_vec.push_back(&emit_prob_e);
+ emit_prob_vec.push_back(&emit_prob_m);
+ emit_prob_vec.push_back(&emit_prob_s);
+ load_model(model_path);
+ }
+ // emit_prob_vec points at the maps of this model.
+ HMMModel(const HMMModel&) = delete;
+ HMMModel& operator=(const HMMModel&) = delete;
+
+ void load_model(const std::string& file_path) {
+ std::ifstream ifile(file_path);
+ if (!ifile.is_open()) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "jieba hmm model file not found: {}", file_path);
+ }
+ std::string line;
+ std::vector<std::string> tmp;
+ //Load start_prob
+ if (!get_line(ifile, line)) {
+ throw_illegal(file_path, "start probability");
+ }
+ split(line, tmp, " ");
+ if (tmp.size() != STATUS_SUM) {
+ throw_illegal(file_path, "start probability");
+ }
+ for (size_t j = 0; j < tmp.size(); j++) {
+ start_prob[j] = std::atof(tmp[j].c_str());
+ }
+
+ //Load trans_prob
+ for (size_t i = 0; i < STATUS_SUM; i++) {
+ if (!get_line(ifile, line)) {
+ throw_illegal(file_path, "transition probability");
+ }
+ split(line, tmp, " ");
+ if (tmp.size() != STATUS_SUM) {
+ throw_illegal(file_path, "transition probability");
+ }
+ for (size_t j = 0; j < STATUS_SUM; j++) {
+ trans_prob[i][j] = std::atof(tmp[j].c_str());
+ }
+ }
+
+ //Load emit_prob_b
+ if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_b)) {
+ throw_illegal(file_path, "B emission probability");
+ }
+
+ //Load emit_prob_e
+ if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_e)) {
+ throw_illegal(file_path, "E emission probability");
+ }
+
+ //Load emit_prob_m
+ if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_m)) {
+ throw_illegal(file_path, "M emission probability");
+ }
+
+ //Load emit_prob_s
+ if (!get_line(ifile, line) || !load_emit_prob(line, emit_prob_s)) {
+ throw_illegal(file_path, "S emission probability");
+ }
+ }
+
+ // The next line that is neither blank nor a comment, trimmed.
+ static bool get_line(std::ifstream& ifile, std::string& line) {
+ while (std::getline(ifile, line)) {
+ trim(line);
+ if (line.empty()) {
+ continue;
+ }
+ if (line.starts_with("#")) {
+ continue;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ // Fails on a pair that is not two pieces split at ':' and on a rune that
does not decode to
+ // exactly one code point. The pairs before it are in `mp` by then, which
does not matter: the
+ // load fails.
+ static bool load_emit_prob(const std::string& line, EmitProbMap& mp) {
+ // Upstream returned false for an empty line, which get_line() never
returns.
+ DCHECK(!line.empty());
+ std::vector<std::string> tmp;
+ std::vector<std::string> tmp2;
+ Unicode unicode;
+ split(line, tmp, ",");
+ for (const std::string& item : tmp) {
+ split(item, tmp2, ":");
+ if (2 != tmp2.size()) {
+ return false;
+ }
+ if (!decode_unicode(tmp2[0], unicode) || unicode.size() != 1) {
+ return false;
+ }
+ mp[unicode[0]] = std::atof(tmp2[1].c_str());
+ }
+ return true;
+ }
+
+ std::array<double, STATUS_SUM> start_prob {};
+ std::array<std::array<double, STATUS_SUM>, STATUS_SUM> trans_prob {};
+ EmitProbMap emit_prob_b;
+ EmitProbMap emit_prob_e;
+ EmitProbMap emit_prob_m;
+ EmitProbMap emit_prob_s;
+ // Indexed by status.
+ std::vector<EmitProbMap*> emit_prob_vec;
+
+private:
+ // A line of the model that upstream's checks failed the load on.
+ [[noreturn]] static void throw_illegal(const std::string& file_path,
std::string_view part) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "jieba hmm model file {} has no legal {} line",
file_path, part);
+ }
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba.h
new file mode 100644
index 00000000000..f08638672dd
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba.h
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/segmenter.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// The dictionaries of the chinese parser, loaded once, and its two cuts.
Loading reads the main
+// dictionary and the user dictionaries, then the model, then the stop words,
and throws
+// INVERTED_INDEX_ANALYZER_ERROR when the dictionary, a user dictionary or the
model cannot be
+// opened or has an illegal line; a stop word file that cannot be opened
leaves no stop word. The
+// loaders' results are laid out in a FlatDict and dropped once it is built.
Unlike CLucene's
+// Jieba it reads no idf file and has no keyword extractor, whose loading also
failed on a stop
+// word file that could not be opened.
+//
+// Cutting is const and keeps what it needs in the caller's scratch, so one
Jieba serves every
+// thread.
+class Jieba {
+public:
+ Jieba(const std::string& dict_path, const std::string& model_path,
+ const std::string& user_dict_path, const std::string& stop_word_path)
+ : _dict(load(dict_path, model_path, user_dict_path,
stop_word_path)) {}
+ Jieba(const Jieba&) = delete;
+ Jieba& operator=(const Jieba&) = delete;
+
+ // The coarse mode, CLucene's Cut with the HMM. The words are views of
`sentence`.
+ void cut(std::string_view sentence, JiebaScratch& scratch,
+ std::vector<std::string_view>& words) const {
+ Segmenter(_dict, sentence, scratch, words).mix_cut();
+ }
+
+ // The fine mode, CLucene's CutAll. The words are views of `sentence`.
+ void cut_all(std::string_view sentence, JiebaScratch& scratch,
+ std::vector<std::string_view>& words) const {
+ Segmenter(_dict, sentence, scratch, words).full_cut();
+ }
+
+ // The same with a scratch of their own, for a caller that cuts once.
+ void cut(std::string_view sentence, std::vector<std::string_view>& words)
const {
+ JiebaScratch scratch;
+ cut(sentence, scratch, words);
+ }
+ void cut_all(std::string_view sentence, std::vector<std::string_view>&
words) const {
+ JiebaScratch scratch;
+ cut_all(sentence, scratch, words);
+ }
+
+private:
+ // In this order, as CLucene's Jieba built its members, so that a bad
dictionary fails the
+ // load before a bad model does.
+ static FlatDict load(const std::string& dict_path, const std::string&
model_path,
+ const std::string& user_dict_path, const std::string&
stop_word_path) {
+ const DictTrie dict_trie(dict_path, user_dict_path);
+ const HMMModel model(model_path);
+ return {dict_trie, model, stop_word_path};
+ }
+
+ const FlatDict _dict;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h
new file mode 100644
index 00000000000..012d7faba84
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+#include "storage/index/inverted/analysis/scratch.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// What a cut works on besides the dictionary, all of it in proportion to the
value: the caller
+// keeps one from value to value, so that a steady stream of values allocates
nothing, and calls
+// release_oversized() between values. The dictionary is shared and read only;
this is not.
+struct JiebaScratch {
+ // The code point of every rune of the value and its byte offset, and
after the last rune the
+ // size of the value: the runes [l, r] are the bytes [offsets[l],
offsets[r + 1]).
+ std::vector<uint32_t> runes;
+ std::vector<uint32_t> offsets;
+ // The maximum-probability cut of a range: from each rune, the best weight
of the rest of the
+ // range and the runes of the word that starts it.
+ std::vector<double> dp;
+ std::vector<uint32_t> best_len;
+ // Viterbi over a run of runes: for each rune and status, the status of
the rune before; then
+ // the status of each rune.
+ std::vector<int8_t> path;
+ std::vector<uint8_t> status;
+
+ void release_oversized() {
+ release_oversized_scratch(runes);
+ release_oversized_scratch(offsets);
+ release_oversized_scratch(dp);
+ release_oversized_scratch(best_len);
+ release_oversized_scratch(path);
+ release_oversized_scratch(status);
+ }
+
+ size_t reserved_bytes() const {
+ return runes.capacity() * sizeof(uint32_t) + offsets.capacity() *
sizeof(uint32_t) +
+ dp.capacity() * sizeof(double) + best_len.capacity() *
sizeof(uint32_t) +
+ path.capacity() * sizeof(int8_t) + status.capacity() *
sizeof(uint8_t);
+ }
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.cpp
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.cpp
new file mode 100644
index 00000000000..fa5ef0206a7
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.cpp
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <fstream>
+#include <mutex>
+#include <unordered_map>
+#include <utility>
+
+#include "common/exception.h"
+#include "storage/index/inverted/analysis/common/rune.h"
+#include "storage/index/inverted/analysis/scratch.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/byte_classifier.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba.h"
+
+namespace doris::segment_v2::inverted_index {
+
+namespace {
+
+bool has_ascii_upper(std::string_view term) {
+ return std::ranges::any_of(term,
+ [](char c) { return
is_ascii_upper(static_cast<uint8_t>(c)); });
+}
+
+} // namespace
+
+std::shared_ptr<const jieba::Jieba> load_jieba(const std::string& dict_dir) {
+ static std::mutex mutex;
+ static std::unordered_map<std::string, std::shared_ptr<const
jieba::Jieba>> loaded;
+ std::lock_guard lock(mutex);
+ if (const auto it = loaded.find(dict_dir); it != loaded.end()) {
+ return it->second;
+ }
+ // idf.utf8, which CLucene's keyword extractor read, is no longer needed.
The segmenters skip a
+ // stop word file they cannot open, which the extractor refused, so an
unreadable file is
+ // refused here.
+ for (const char* name :
+ {"jieba.dict.utf8", "hmm_model.utf8", "user.dict.utf8",
"stop_words.utf8"}) {
+ const std::string path = dict_dir + "/" + name;
+ if (!std::ifstream(path).is_open()) {
+ throw Exception(ErrorCode::INVERTED_INDEX_ANALYZER_ERROR,
+ "chinese tokenizer dict file not found: " + path);
+ }
+ }
+ auto jieba = std::make_shared<const jieba::Jieba>(
+ dict_dir + "/jieba.dict.utf8", dict_dir + "/hmm_model.utf8",
+ dict_dir + "/user.dict.utf8", dict_dir + "/stop_words.utf8");
+ loaded.emplace(dict_dir, jieba);
+ return jieba;
+}
+
+JiebaTokenizer::JiebaTokenizer(std::shared_ptr<const jieba::Jieba> jieba, bool
fine, bool lowercase)
+ : _jieba(std::move(jieba)), _fine(fine), _lowercase(lowercase) {}
+
+JiebaTokenizer::~JiebaTokenizer() = default;
+
+void JiebaTokenizer::reset(std::string_view text) {
+ _next = 0;
+ release_oversized_scratch(_value);
+ release_oversized_scratch(_words);
+ _scratch.release_oversized();
+ _value.assign(text);
+ if (_fine) {
+ _jieba->cut_all(_value, _scratch, _words);
+ } else {
+ _jieba->cut(_value, _scratch, _words);
+ }
+}
+
+bool JiebaTokenizer::next_batch(TokenBatch& batch) {
+ batch.clear();
+ while (_next < _words.size() && batch.size() < kBatchTokens) {
+ const std::string_view word = _words[_next++];
+ if (_lowercase && !word.empty() && static_cast<uint8_t>(word[0]) <
0x80) {
+ // The whole word, even past what the term keeps: an overlapping
word reads those bytes.
+ char* const begin = _value.data() + (word.data() - _value.data());
+ for (size_t i = 0; i < word.size(); ++i) {
+ begin[i] = ascii_lower(begin[i]);
+ }
+ }
+ const std::string_view term = word.substr(0, MAX_WORD_LEN);
+ if (_lowercase && has_ascii_upper(term)) {
+ batch.push_copy(term, 1);
+ } else {
+ batch.push(term, 1);
+ }
+ }
+ return batch.size() != 0;
+}
+
+JiebaTokenizerFactory::JiebaTokenizerFactory(bool fine, bool lowercase, const
std::string& dict_dir)
+ : _fine(fine), _lowercase(lowercase), _jieba(load_jieba(dict_dir)) {}
+
+} // namespace doris::segment_v2::inverted_index
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h
new file mode 100644
index 00000000000..29a01a83c44
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "storage/index/inverted/analysis/text_token_stream.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h"
+#include "storage/index/inverted/analysis/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+namespace jieba {
+class Jieba;
+} // namespace jieba
+
+// The dictionaries of the chinese parser in `dict_dir`: jieba.dict.utf8,
hmm_model.utf8,
+// user.dict.utf8 and stop_words.utf8. They are loaded at the first call for a
directory and shared
+// by every stream after it; a load that fails throws, and the next call tries
again.
+std::shared_ptr<const jieba::Jieba> load_jieba(const std::string& dict_dir);
+
+// The tokenizer of the builtin chinese parser, which makes the tokens
CLucene's ChineseTokenizer
+// made: jieba's words of the value, cut by MixSegment with the HMM in the
coarse mode and by
+// FullSegment, which hands out every dictionary word it finds, in the fine
mode, with jieba's stop
+// words dropped before anything else happens to them. A value jieba cannot
decode has no token. A
+// word is cut to MAX_WORD_LEN bytes, which may split a character, and every
word is one position
+// after the last.
+//
+// With `lowercase`, a word whose first byte is ASCII is lowercased whole, and
a word that starts with
+// another byte is left as it is, capitals included: with "HHOK" in the user
dictionary, H standing
+// for a Han character, the coarse mode makes "HHOK", and "QHH" makes "qHH".
The words are
+// lowercased where they lie, in the copy
+// of the value, when they are handed out, as the old tokenizer did to its
buffer, so a word sees the
+// lowercasing of the words handed out before it. That matters where the fine
mode's words overlap:
+// it hands out a run of ASCII letters and digits whole and then the runs of
letters and of digits
+// in it, and jieba reads a stray continuation byte and the byte after it as
one character, so
+// "" "7abcB" is the run "7abcB", which starts with a byte that is not ASCII
and keeps its
+// capital, followed by "abcB", which lowers it. A batch holds many words at
once, so a word that
+// still has a capital after its turn is copied into the batch, where a later
word cannot lower it.
+class JiebaTokenizer final : public TextTokenizer {
+public:
+ JiebaTokenizer(std::shared_ptr<const jieba::Jieba> jieba, bool fine, bool
lowercase);
+ ~JiebaTokenizer() override;
+
+ void reset(std::string_view text) override;
+ bool next_batch(TokenBatch& batch) override;
+ // The copy of the value, the list of its words and what jieba cut them
with.
+ size_t reserved_bytes() const override {
+ return _value.capacity() + _words.capacity() *
sizeof(std::string_view) +
+ _scratch.reserved_bytes();
+ }
+
+private:
+ const std::shared_ptr<const jieba::Jieba> _jieba;
+ const bool _fine;
+ const bool _lowercase;
+ // The value, which jieba cuts and lowercasing rewrites; the words are
views of it.
+ std::string _value;
+ std::vector<std::string_view> _words;
+ // The runes of the value and the arrays of the cut, kept for the next
value.
+ jieba::JiebaScratch _scratch;
+ size_t _next = 0;
+};
+
+// The tokenizer of the builtin chinese parser, which is no custom tokenizer.
It loads the
+// dictionaries when it is made, so that a missing file fails the analyzer,
not its first stream.
+class JiebaTokenizerFactory final : public TokenizerFactory {
+public:
+ JiebaTokenizerFactory(bool fine, bool lowercase, const std::string&
dict_dir);
+
+ void initialize(const Settings& settings) override {}
+
+ std::unique_ptr<TextTokenizer> create() const override {
+ return std::make_unique<JiebaTokenizer>(_jieba, _fine, _lowercase);
+ }
+
+private:
+ const bool _fine;
+ const bool _lowercase;
+ const std::shared_ptr<const jieba::Jieba> _jieba;
+};
+
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h
new file mode 100644
index 00000000000..ae38e618437
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/segmenter.h
@@ -0,0 +1,408 @@
+// Derived from cppjieba's MPSegment.hpp, HMMSegment.hpp, MixSegment.hpp,
FullSegment.hpp and
+// PreFilter.hpp (https://github.com/yanyiwu/cppjieba), MIT License, Copyright
(c) 2013. See
+// dist/licenses/LICENSE-cppjieba.txt.
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <string_view>
+#include <vector>
+
+#include "common/logging.h"
+#include "storage/index/inverted/analysis/common/rune.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/dict_trie.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/flat_dict.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/hmm_model.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_scratch.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// The longest dictionary word a cut looks for, in runes.
+inline constexpr uint32_t MAX_WORD_LENGTH = 512;
+
+// One cut of a sentence into words, which are views of it, with the stop
words dropped as they
+// are, before anything lowercases them. A sentence that does not decode by
decode_rune() has no
+// word at all.
+//
+// Both cuts split the sentence into ranges first, as upstream's PreFilter:
every separator (space,
+// tab, newline, the fullwidth comma U+FF0C and the ideographic full stop
U+3002, upstream's
+// SPECIAL_SEPARATORS) is a range of its own, and the runes between two
separators are one range;
+// the full cut also makes each run of ASCII letters and digits a range of its
own. "ab,c d" has
+// the ranges "ab" "," "c" " " "d" in both cuts; "Hab", H being a Han
character, is one range in
+// the mix cut and "H" "ab" in the full cut. A separator is always one word,
so it is handed out
+// without a cut: the maximum-probability cut of one rune is that rune, which
the HMM keeps whole
+// too, and the full cut hands out the only rune of a range, since no word
starts there but the
+// rune alone and none before it covers it.
+//
+// Upstream decoded into runes of 20 bytes, built a Dag of a few hundred bytes
for every rune, and
+// allocated each step's vectors anew. Here the runes and the steps' arrays
are in the scratch, and
+// the dictionary walk from a rune feeds the step that reads it directly, in
the order the Dag held
+// the words: the rune alone, a word or not, then the longer words in
increasing end.
+class Segmenter {
+public:
+ Segmenter(const FlatDict& dict, std::string_view sentence, JiebaScratch&
scratch,
+ std::vector<std::string_view>& words)
+ : _dict(dict), _sentence(sentence), _scratch(scratch),
_words(words) {}
+
+ // Jieba's default cut, upstream's MixSegment::Cut with the HMM: the
maximum-probability cut of
+ // each range, with every run of its single runes that are no user word of
one rune cut again
+ // by the HMM.
+ void mix_cut() {
+ _words.clear();
+ if (!decode()) {
+ return;
+ }
+ const uint32_t* const runes = _scratch.runes.data();
+ const auto count = static_cast<uint32_t>(_scratch.runes.size());
+ uint32_t begin = 0;
+ for (uint32_t cursor = 0; cursor < count; ++cursor) {
+ if (is_separator(runes[cursor])) {
+ if (begin < cursor) {
+ mix_range(begin, cursor);
+ }
+ emit(cursor, cursor);
+ begin = cursor + 1;
+ }
+ }
+ if (begin < count) {
+ mix_range(begin, count);
+ }
+ }
+
+ // Jieba's full cut, upstream's FullSegment::Cut: every dictionary word of
two runes or more in
+ // a range, overlapping, and the runes no word covers; a run of ASCII
letters and digits is a
+ // word, and so is each run of two or more letters or of two or more
digits inside it.
+ void full_cut() {
+ _words.clear();
+ if (!decode()) {
+ return;
+ }
+ const uint32_t* const runes = _scratch.runes.data();
+ const auto count = static_cast<uint32_t>(_scratch.runes.size());
+ uint32_t cursor = 0;
+ while (cursor < count) {
+ const uint32_t begin = cursor;
+ if (is_separator(runes[cursor])) {
+ emit(cursor, cursor);
+ ++cursor;
+ } else if (is_alnum(runes[cursor])) {
+ do {
+ ++cursor;
+ } while (cursor < count && is_alnum(runes[cursor]));
+ alnum_range(begin, cursor);
+ } else {
+ do {
+ ++cursor;
+ } while (cursor < count && !is_separator(runes[cursor]) &&
+ !is_alnum(runes[cursor]));
+ full_range(begin, cursor);
+ }
+ }
+ }
+
+private:
+ // Decodes the sentence by decode_rune(), the rules of decode_runes(): the
lead byte alone
+ // sets a rune's length, so a stray continuation byte reads as the lead of
a two-byte rune, and
+ // "\x80" "7" is U+0037. False when a rune does not decode.
+ bool decode() {
+ DCHECK(_sentence.size() < std::numeric_limits<uint32_t>::max());
+ _scratch.runes.resize(_sentence.size());
+ _scratch.offsets.resize(_sentence.size() + 1);
+ uint32_t* const runes = _scratch.runes.data();
+ uint32_t* const offsets = _scratch.offsets.data();
+ uint32_t count = 0;
+ for (size_t offset = 0; offset < _sentence.size();) {
+ const RuneStrLite decoded =
+ decode_rune(_sentence.data() + offset, _sentence.size() -
offset);
+ if (decoded.len == 0) {
+ return false;
+ }
+ runes[count] = decoded.rune;
+ offsets[count] = static_cast<uint32_t>(offset);
+ ++count;
+ offset += decoded.len;
+ }
+ offsets[count] = static_cast<uint32_t>(_sentence.size());
+ _scratch.runes.resize(count);
+ _scratch.offsets.resize(count + 1);
+ return true;
+ }
+
+ // MixSegment::Cut over the runes [lo, hi): the maximum-probability cut,
whose words of two
+ // runes or more, and single runes that are user words of one rune, are
words, and whose other
+ // single runes are cut again by the HMM, run by run.
+ void mix_range(uint32_t lo, uint32_t hi) {
+ const uint32_t size = hi - lo;
+ max_probability_cut(lo, size);
+ const uint32_t* const runes = _scratch.runes.data() + lo;
+ const uint32_t* const best_len = _scratch.best_len.data();
+ uint32_t i = 0;
+ while (i < size) {
+ const uint32_t len = best_len[i];
+ if (len > 1 || _dict.is_user_single_word(runes[i])) {
+ emit(lo + i, lo + i + len - 1);
+ i += len;
+ continue;
+ }
+ uint32_t j = i + 1;
+ while (j < size && best_len[j] == 1 &&
!_dict.is_user_single_word(runes[j])) {
+ ++j;
+ }
+ hmm_range(lo + i, lo + j);
+ i = j;
+ }
+ }
+
+ // MPSegment's CalcDP over the `size` runes from `lo`, from the last rune
to the first: the best
+ // weight of the rest of the range from each rune, and the runes of the
word that starts it. A
+ // rune that is no word weighs the smallest weight of the dictionary. The
weight of a word is
+ // added to the best weight after it, in that order, and the strict >
keeps the first of equal
+ // sums: a tie goes to the shorter word. dp[size] is 0.0, where upstream
added nothing: 0.0 +
+ // 0.0 + w is 0.0 + w bit for bit, whatever w is.
+ void max_probability_cut(uint32_t lo, uint32_t size) {
+ _scratch.dp.resize(size + 1);
+ _scratch.best_len.resize(size);
+ double* const dp = _scratch.dp.data();
+ uint32_t* const best_len = _scratch.best_len.data();
+ const uint32_t* const runes = _scratch.runes.data() + lo;
+ dp[size] = 0.0;
+ for (uint32_t i = size; i-- > 0;) {
+ uint32_t node = _dict.root_child(runes[i]);
+ double val = 0.0;
+ val += dp[i + 1];
+ val += node != FlatDict::kNoNode && _dict.has_value(node) ?
_dict.weight(node)
+ :
_dict.min_weight();
+ // Upstream started at MIN_DOUBLE and took the rune alone when it
was above: a NaN, or
+ // a sum of MIN_DOUBLE or less, leaves MIN_DOUBLE and still a cut
of the rune alone.
+ double best = std::max(MIN_DOUBLE, val);
+ uint32_t len = 1;
+ const uint32_t end = walk_end(i, size);
+ for (uint32_t k = i + 1; node != FlatDict::kNoNode && k < end;
++k) {
+ node = _dict.child(node, runes[k]);
+ if (node != FlatDict::kNoNode && _dict.has_value(node)) {
+ val = 0.0;
+ val += dp[k + 1];
+ val += _dict.weight(node);
+ if (val > best) {
+ best = val;
+ len = k - i + 1;
+ }
+ }
+ }
+ dp[i] = best;
+ best_len[i] = len;
+ }
+ }
+
+ // HMMSegment::Cut over the runes [lo, hi): an ASCII rune ends the run of
other runes before
+ // it, which Viterbi cuts, and starts a word of its own: a letter followed
by letters and
+ // digits, a digit followed by digits and '.', or else the rune alone,
none reaching past hi.
+ void hmm_range(uint32_t lo, uint32_t hi) {
+ const uint32_t* const runes = _scratch.runes.data();
+ uint32_t left = lo;
+ uint32_t right = lo;
+ while (right != hi) {
+ if (runes[right] >= 0x80) {
+ ++right;
+ continue;
+ }
+ if (left != right) {
+ viterbi(left, right);
+ }
+ left = right;
+ right = ascii_word_end(runes, left, hi);
+ emit(left, right - 1);
+ left = right;
+ }
+ if (left != right) {
+ viterbi(left, right);
+ }
+ }
+
+ // The end of the word that starts at the ASCII rune `begin`: upstream's
sequential letter
+ // rule, its numbers rule, or the rune alone.
+ static uint32_t ascii_word_end(const uint32_t* runes, uint32_t begin,
uint32_t end) {
+ const uint32_t first = runes[begin];
+ uint32_t cursor = begin + 1;
+ if (is_letter(first)) {
+ while (cursor != end && (is_letter(runes[cursor]) ||
is_digit(runes[cursor]))) {
+ ++cursor;
+ }
+ } else if (is_digit(first)) {
+ while (cursor != end && (is_digit(runes[cursor]) || runes[cursor]
== '.')) {
+ ++cursor;
+ }
+ }
+ return cursor;
+ }
+
+ // HMMSegment's Viterbi over the runes [lo, hi), which is not empty, and
the words it makes: a
+ // word ends at every rune tagged E or S, and the last rune is always one
of them. The sums run
+ // weight + transition + emission in that order; the strict > keeps the
first of equal previous
+ // statuses, and E where none beats MIN_DOUBLE; the last rune is E when E
and S are equal. Only
+ // the path is kept for every rune: the next rune needs the weights of
this one alone.
+ void viterbi(uint32_t lo, uint32_t hi) {
+ constexpr size_t kStatuses = HMMModel::STATUS_SUM;
+ const uint32_t size = hi - lo;
+ _scratch.path.resize(size_t {size} * kStatuses);
+ _scratch.status.resize(size);
+ int8_t* const path = _scratch.path.data();
+ uint8_t* const status = _scratch.status.data();
+ const uint32_t* const runes = _scratch.runes.data() + lo;
+ const auto& start_prob = _dict.start_prob();
+ const auto& trans_prob = _dict.trans_prob();
+
+ std::array<double, kStatuses> weight;
+ const double* emit_prob = _dict.emit_prob(runes[0]);
+ for (size_t y = 0; y < kStatuses; ++y) {
+ weight[y] = start_prob[y] + emit_prob[y];
+ path[y] = -1;
+ }
+ for (uint32_t x = 1; x < size; ++x) {
+ emit_prob = _dict.emit_prob(runes[x]);
+ std::array<double, kStatuses> next;
+ for (size_t y = 0; y < kStatuses; ++y) {
+ double best = MIN_DOUBLE;
+ auto from = static_cast<int8_t>(HMMModel::E);
+ for (size_t pre_y = 0; pre_y < kStatuses; ++pre_y) {
+ const double tmp = weight[pre_y] + trans_prob[pre_y][y] +
emit_prob[y];
+ if (tmp > best) {
+ best = tmp;
+ from = static_cast<int8_t>(pre_y);
+ }
+ }
+ next[y] = best;
+ path[size_t {x} * kStatuses + y] = from;
+ }
+ weight = next;
+ }
+
+ int stat = weight[HMMModel::E] >= weight[HMMModel::S] ? HMMModel::E :
HMMModel::S;
+ // The path of the first rune is -1, which `stat` takes after its last
use.
+ for (uint32_t x = size; x-- > 0;) {
+ status[x] = static_cast<uint8_t>(stat);
+ stat = path[size_t {x} * kStatuses + static_cast<size_t>(stat)];
+ }
+ uint32_t left = lo;
+ for (uint32_t x = 0; x < size; ++x) {
+ // E and S are odd.
+ if (status[x] % 2 != 0) {
+ emit(left, lo + x);
+ left = lo + x + 1;
+ }
+ }
+ }
+
+ // FullSegment::Cut over the runes [lo, hi). Every word of two runes or
more is handed out. A
+ // rune that starts no longer word is handed out, a word or not, when
max_idx, the end of the
+ // words before it as far as word_len tells, does not reach past it.
+ //
+ // word_len keeps its value from one word to the next and from one rune to
the next: a rune
+ // that is no word does not set it, yet still raises max_idx to u_idx +
the length of the last
+ // word seen. This looks like a bug, but it decides which single runes are
handed out, so it is
+ // kept as upstream has it. Say A, B, C and D are CJK runes, and AB is the
only word of the
+ // dictionary that starts with any of them. The runes ABCD give only AB: B
is skipped because AB
+ // covers it (max_idx 2 > u_idx 1), and the stale 2 then raises max_idx to
3; C meets 3 > 2, is
+ // skipped and raises it to 4, and D meets 4 > 3. With a word_len of 1 for
a rune that is no
+ // word, max_idx would stay 2 after B, and C and D would both be handed
out.
+ //
+ // Upstream had the words of a rune in its Dag before it looked at the
rune alone, and handed
+ // the rune out only when no longer word started there. The walk here
takes the rune alone at
+ // the first longer word, where it hands nothing out, or after the walk
when there is none;
+ // word_len and max_idx change in upstream's order either way.
+ void full_range(uint32_t lo, uint32_t hi) {
+ const uint32_t size = hi - lo;
+ const uint32_t* const runes = _scratch.runes.data() + lo;
+ size_t max_idx = 0;
+ size_t word_len = 0;
+ for (uint32_t u_idx = 0; u_idx < size; ++u_idx) {
+ uint32_t node = _dict.root_child(runes[u_idx]);
+ const bool alone_is_word = node != FlatDict::kNoNode &&
_dict.has_value(node);
+ bool has_longer = false;
+ const uint32_t end = walk_end(u_idx, size);
+ for (uint32_t k = u_idx + 1; node != FlatDict::kNoNode && k < end;
++k) {
+ node = _dict.child(node, runes[k]);
+ if (node == FlatDict::kNoNode || !_dict.has_value(node)) {
+ continue;
+ }
+ if (!has_longer) {
+ has_longer = true;
+ if (alone_is_word) {
+ word_len = 1;
+ }
+ max_idx = std::max(max_idx, u_idx + word_len);
+ }
+ word_len = size_t {k} - u_idx + 1;
+ emit(lo + u_idx, lo + k);
+ max_idx = std::max(max_idx, u_idx + word_len);
+ }
+ if (!has_longer) {
+ if (max_idx <= u_idx) {
+ emit(lo + u_idx, lo + u_idx);
+ }
+ if (alone_is_word) {
+ word_len = 1;
+ }
+ max_idx = std::max(max_idx, u_idx + word_len);
+ }
+ }
+ }
+
+ // FullSegment::CutAlnum over a run of ASCII letters and digits [lo, hi):
the whole run, then
+ // the runs of two or more digits or of two or more letters in it, unless
one is the whole run:
+ // "abc123" gives "abc123", "abc" and "123", and "abc" only "abc".
+ void alnum_range(uint32_t lo, uint32_t hi) {
+ emit(lo, hi - 1);
+ const uint32_t* const runes = _scratch.runes.data();
+ uint32_t cursor = lo;
+ while (cursor != hi) {
+ const bool digits = is_digit(runes[cursor]);
+ DCHECK(digits || is_letter(runes[cursor]));
+ const uint32_t start = cursor;
+ do {
+ ++cursor;
+ } while (cursor != hi && (digits ? is_digit(runes[cursor]) :
is_letter(runes[cursor])));
+ if (cursor - start > 1 && (start != lo || cursor != hi)) {
+ emit(start, cursor - 1);
+ }
+ }
+ }
+
+ // Where the dictionary walk from the rune `begin` of a range of `size`
runes stops: a word of
+ // more than MAX_WORD_LENGTH runes is never found, as upstream.
+ static uint32_t walk_end(uint32_t begin, uint32_t size) {
+ return static_cast<uint32_t>(std::min<size_t>(size, size_t {begin} +
MAX_WORD_LENGTH));
+ }
+
+ // Hands out the runes [left, right] as a word, unless it is a stop word.
+ void emit(uint32_t left, uint32_t right) {
+ const uint32_t begin = _scratch.offsets[left];
+ const std::string_view word(_sentence.data() + begin,
_scratch.offsets[right + 1] - begin);
+ if (!_dict.is_stop_word(word)) {
+ _words.push_back(word);
+ }
+ }
+
+ static bool is_separator(uint32_t rune) {
+ return rune == ' ' || rune == '\t' || rune == '\n' || rune == 0xFF0C
|| rune == 0x3002;
+ }
+
+ static bool is_digit(uint32_t rune) { return rune >= '0' && rune <= '9'; }
+
+ static bool is_letter(uint32_t rune) {
+ return (rune >= 'A' && rune <= 'Z') || (rune >= 'a' && rune <= 'z');
+ }
+
+ static bool is_alnum(uint32_t rune) { return is_digit(rune) ||
is_letter(rune); }
+
+ const FlatDict& _dict;
+ const std::string_view _sentence;
+ JiebaScratch& _scratch;
+ std::vector<std::string_view>& _words;
+};
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git
a/be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h
new file mode 100644
index 00000000000..9c8029dd714
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/string_util.h
@@ -0,0 +1,70 @@
+// Derived from limonp's StringUtil.hpp (https://github.com/yanyiwu/limonp),
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-limonp.txt.
+//
+// Only Split and Trim, which jieba's dictionary and model loading use.
+
+#pragma once
+
+#include <algorithm>
+#include <cctype>
+#include <cstddef>
+#include <string>
+#include <vector>
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+// Splits `src` at every byte that is one of `pattern`. Two separators in a
row give the empty piece
+// between them and a separator at the start an empty first piece, but a
separator at the end gives
+// no piece after it: "a b " split at " " is {"a", "", "b"}, and "" is no
piece at all. Upstream's
+// maxsplit parameter is gone: no caller passed one, and with its default of
npos it never stopped
+// a split.
+inline void split(const std::string& src, std::vector<std::string>& res,
+ const std::string& pattern) {
+ res.clear();
+ size_t start = 0;
+ size_t end = 0;
+ while (start < src.size()) {
+ end = src.find_first_of(pattern, start);
+ if (std::string::npos == end) {
+ res.push_back(src.substr(start));
+ return;
+ }
+ res.push_back(src.substr(start, end - start));
+ start = end + 1;
+ }
+}
+
+inline std::vector<std::string> split(const std::string& src, const
std::string& pattern) {
+ std::vector<std::string> res;
+ split(src, res, pattern);
+ return res;
+}
+
+// Whether `c` is a space by std::isspace, so by the C locale, which the BE
never changes. A char
+// is sign-extended on the way in, so where char is signed a byte of 0x80 and
above is above 0xFF
+// here, and no space.
+inline bool is_space(unsigned c) {
+ // when passing large int as the argument of isspace, it core dump, so
here need a type cast.
+ return c > 0xff ? false : std::isspace(static_cast<int>(c & 0xff)) != 0;
+}
+
+inline std::string& ltrim(std::string& s) {
+ s.erase(s.begin(), std::ranges::find_if(s, [](const char& c) {
+ return !is_space(static_cast<unsigned>(c));
+ }));
+ return s;
+}
+
+inline std::string& rtrim(std::string& s) {
+ s.erase(std::ranges::find_if(s.rbegin(), s.rend(),
+ [](const char& c) { return
!is_space(static_cast<unsigned>(c)); })
+ .base(),
+ s.end());
+ return s;
+}
+
+inline std::string& trim(std::string& s) {
+ return ltrim(rtrim(s));
+}
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h
b/be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h
new file mode 100644
index 00000000000..59ffa55a69c
--- /dev/null
+++ b/be/src/storage/index/inverted/analysis/tokenizer/jieba/unicode.h
@@ -0,0 +1,36 @@
+// Derived from cppjieba's Unicode.hpp (https://github.com/yanyiwu/cppjieba),
MIT License,
+// Copyright (c) 2013. See dist/licenses/LICENSE-cppjieba.txt.
+//
+// Only what the dictionary and model loading use. The decoder comes from
analysis/common/rune.h,
+// which carries jieba's decoding rules; the cuts decode with it too
(segmenter.h).
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "storage/index/inverted/analysis/common/rune.h"
+
+namespace doris::segment_v2::inverted_index::jieba {
+
+using Rune = uint32_t;
+// The code points of a dictionary word: the key of the trie.
+using Unicode = std::vector<Rune>;
+
+// The code points of `s`, decoded with decode_runes(). Returns false with no
code point when one of
+// them does not decode. Upstream's DecodeRunesInString(const string&,
Unicode&).
+inline bool decode_unicode(const std::string& s, Unicode& unicode) {
+ unicode.clear();
+ RuneStrArray runes;
+ if (!decode_runes(s.data(), s.size(), runes)) {
+ return false;
+ }
+ unicode.reserve(runes.size());
+ for (const RuneStr& rune : runes) {
+ unicode.push_back(rune.rune);
+ }
+ return true;
+}
+
+} // namespace doris::segment_v2::inverted_index::jieba
diff --git a/be/test/storage/index/inverted/analysis/component_stream_test.cpp
b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
index d2f1fc1fff3..da8984885c5 100644
--- a/be/test/storage/index/inverted/analysis/component_stream_test.cpp
+++ b/be/test/storage/index/inverted/analysis/component_stream_test.cpp
@@ -50,6 +50,7 @@
#include "storage/index/inverted/analysis/tokenizer/basic/basic_tokenizer.h"
#include
"storage/index/inverted/analysis/tokenizer/english/english_tokenizer.h"
#include "storage/index/inverted/analysis/tokenizer/icu/icu_tokenizer.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
#include "storage/index/inverted/analysis/tokenizer/ngram/gram_tokenizer.h"
#include "storage/index/inverted/analysis/tokenizer/ngram/ngram_tokenizer.h"
#include "storage/index/inverted/analysis/tokenizer/pinyin/pinyin_tokenizer.h"
@@ -284,6 +285,18 @@ TEST(ComponentStreamTest,
EnglishTokenizerKeepsNoRowSizedScratch) {
}
}
+TEST(ComponentStreamTest, JiebaTokenizerReleasesItsCopyAndWordList) {
+ const JiebaTokenizerFactory factory(
+ false, true,
"./contrib/clucene/src/contribs-lib/CLucene/analysis/jieba/dict");
+ expect_text_stream_contract([&] { return factory.create(); },
contract_texts());
+ auto tokenizer = factory.create();
+ auto& jieba = as<JiebaTokenizer>(*tokenizer);
+ expect_released_at_next_row(*tokenizer, repeat_to("我们 ", kLongRowBytes),
"short", true, [&] {
+ return jieba._value.capacity() + jieba._words.capacity() *
sizeof(std::string_view) +
+ jieba._scratch.reserved_bytes();
+ });
+}
+
TEST(ComponentStreamTest, Uax29TokenizerReleasesItsLoweredCopy) {
for (bool lowercase : {true, false}) {
SCOPED_TRACE(lowercase);
diff --git
a/be/test/storage/index/inverted/analysis/tokenizer/jieba_tokenizer_test.cpp
b/be/test/storage/index/inverted/analysis/tokenizer/jieba_tokenizer_test.cpp
new file mode 100644
index 00000000000..80ad6e7636f
--- /dev/null
+++ b/be/test/storage/index/inverted/analysis/tokenizer/jieba_tokenizer_test.cpp
@@ -0,0 +1,496 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba_tokenizer.h"
+
+#include <CLucene.h> // IWYU pragma: keep
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <random>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <vector>
+
+#include "CLucene/analysis/LanguageBasedAnalyzer.h"
+#include "common/config.h"
+#include "common/exception.h"
+#include "storage/index/inverted/analysis/analyzer/analyzer.h"
+#include "storage/index/inverted/analysis/analyzer/chain_analyzer.h"
+#include "storage/index/inverted/analysis/common/rune.h"
+#include "storage/index/inverted/analysis/golden/golden_driver.h"
+#include "storage/index/inverted/analysis/text_stream_contract.h"
+#include "storage/index/inverted/analysis/token_batch.h"
+#include "storage/index/inverted/analysis/tokenizer/byte_classifier.h"
+#include "storage/index/inverted/analysis/tokenizer/jieba/jieba.h"
+
+namespace doris::segment_v2::inverted_index {
+namespace {
+
+const std::filesystem::path kShippedDict =
+ "./contrib/clucene/src/contribs-lib/CLucene/analysis/jieba/dict";
+
+// Points the dictionary path at jieba's shipped dictionaries while a test
runs.
+class JiebaDictScope {
+public:
+ JiebaDictScope() : _saved(config::inverted_index_dict_path) {
+ config::inverted_index_dict_path = kShippedDict.string();
+ }
+ ~JiebaDictScope() { config::inverted_index_dict_path = _saved; }
+
+private:
+ const std::string _saved;
+};
+
+// A dictionary directory of a test's own: the shipped files, with
user.dict.utf8 replaced by
+// `user_words` and the files in `missing` left out.
+class OwnDict {
+public:
+ explicit OwnDict(const std::vector<std::string>& user_words,
+ const std::vector<std::string>& missing = {}) {
+ static int counter = 0;
+ _dir = std::filesystem::temp_directory_path() /
+ ("jieba_own_dict_" + std::to_string(::getpid()) + "_" +
std::to_string(counter++));
+ std::filesystem::create_directories(_dir);
+ for (const char* name :
+ {"jieba.dict.utf8", "hmm_model.utf8", "stop_words.utf8",
"idf.utf8"}) {
+ if (std::ranges::find(missing, name) == missing.end()) {
+ std::filesystem::copy_file(kShippedDict / name, _dir / name);
+ }
+ }
+ if (std::ranges::find(missing, "user.dict.utf8") == missing.end()) {
+ std::ofstream user(_dir / "user.dict.utf8");
+ for (const std::string& word : user_words) {
+ user << word << "\n";
+ }
+ }
+ }
+ ~OwnDict() { std::filesystem::remove_all(_dir); }
+
+ std::string dir() const { return _dir.string(); }
+ void add(const char* name) const {
+ std::filesystem::copy_file(kShippedDict / name, _dir / name);
+ }
+
+private:
+ std::filesystem::path _dir;
+};
+
+// The chinese parser as production builds it.
+TextAnalyzerPtr chinese_analyzer(bool fine, bool lowercase) {
+ return
InvertedIndexAnalyzer::create_builtin_analyzer(InvertedIndexParserType::PARSER_CHINESE,
+ fine ?
"fine_grained" : "coarse_grained",
+ lowercase ? "true" :
"false", "");
+}
+
+// CLucene's LanguageBasedAnalyzer, which made the chinese tokens before.
+std::unique_ptr<golden::GoldenDriver> clucene_chinese(bool fine, bool
lowercase) {
+ return golden::make_lucene_driver([fine, lowercase] {
+ auto analyzer =
+
std::make_shared<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
+ analyzer->initDict(kShippedDict.string());
+ analyzer->setMode(fine ? lucene::analysis::AnalyzerMode::All
+ : lucene::analysis::AnalyzerMode::Default);
+ analyzer->set_lowercase(lowercase);
+ return analyzer;
+ });
+}
+
+std::string bytes(std::initializer_list<int> values) {
+ std::string text;
+ for (int value : values) {
+ text.push_back(static_cast<char>(value));
+ }
+ return text;
+}
+
+// Values that stress the segmenters: Chinese text, dictionary words, ASCII in
both cases, rare
+// and supplementary-plane characters jieba has no word for, separators and
invalid bytes.
+std::string random_value(std::mt19937_64& random) {
+ static const std::vector<std::string> kPieces = {"我们",
+ "中华人民共和国",
+ "云计算",
+ "区块链",
+ "研究生命起源",
+ "T恤",
+ "B超",
+ "AT&T",
+ "C++",
+ "Hello",
+ "WORLD",
+ "abc",
+ "123",
+ "3.14",
+ " ",
+ ",",
+ "。",
+ "\t",
+ "\n",
+ "、",
+ "!",
+ "的",
+ "了",
+ "𠮷",
+ "𪚥",
+ "ㄅ",
+ "ひらがな",
+ "カタカナ",
+ "한국어",
+ "é",
+ "😀",
+ "-",
+ "_"};
+ std::string value;
+ const size_t pieces = random() % 40;
+ for (size_t i = 0; i < pieces; ++i) {
+ if (random() % 25 == 0) {
+ value.push_back(static_cast<char>(random()));
+ } else {
+ value += kPieces[random() % kPieces.size()];
+ }
+ }
+ return value;
+}
+
+void expect_same_as_clucene(const std::string& value, golden::GoldenDriver&
candidate,
+ golden::GoldenDriver& oracle) {
+ ASSERT_EQ(candidate.fresh(value), oracle.fresh(value))
+ << golden::format_line(oracle.fresh(value));
+ ASSERT_EQ(candidate.reused(value), oracle.reused(value));
+}
+
+struct Mode {
+ bool fine;
+ bool lowercase;
+};
+constexpr Mode kModes[] = {{.fine = false, .lowercase = false},
+ {.fine = false, .lowercase = true},
+ {.fine = true, .lowercase = false},
+ {.fine = true, .lowercase = true}};
+
+TEST(JiebaTokenizerTest, MatchesCLuceneOnRandomValues) {
+ JiebaDictScope dict;
+ for (const Mode mode : kModes) {
+ SCOPED_TRACE(testing::Message() << "fine " << mode.fine << " lowercase
" << mode.lowercase);
+ auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+ auto candidate = golden::make_text_driver(chinese_analyzer(mode.fine,
mode.lowercase));
+ std::mt19937_64 random(20261104 + mode.fine * 2 + mode.lowercase);
+ for (int round = 0; round < 3000; ++round) {
+ expect_same_as_clucene(random_value(random), *candidate, *oracle);
+ }
+ }
+}
+
+// Every word of the dictionaries, 64 to a value with a random separator or
none between them, in
+// its own case and with its ASCII letters raised.
+TEST(JiebaTokenizerTest, MatchesCLuceneOnEveryDictionaryWord) {
+ JiebaDictScope dict;
+ std::vector<std::string> words;
+ for (const char* name : {"jieba.dict.utf8", "user.dict.utf8"}) {
+ std::ifstream file(kShippedDict / name);
+ std::string line;
+ while (std::getline(file, line)) {
+ words.push_back(line.substr(0, line.find(' ')));
+ }
+ }
+ ASSERT_GT(words.size(), 340000);
+ static const std::vector<std::string> kSeparators = {"", "", " ", ",",
"a", "Z9"};
+ std::mt19937_64 random(20261106);
+ std::vector<std::string> values;
+ for (size_t i = 0; i < words.size(); i += 64) {
+ std::string value;
+ for (size_t j = i; j < std::min(words.size(), i + 64); ++j) {
+ std::string word = words[j];
+ if (random() % 2 == 0) {
+ for (char& c : word) {
+ c = static_cast<char>(c >= 'a' && c <= 'z' ? c - 32 : c);
+ }
+ }
+ value += word + kSeparators[random() % kSeparators.size()];
+ }
+ values.push_back(std::move(value));
+ }
+ for (const Mode mode : kModes) {
+ SCOPED_TRACE(testing::Message() << "fine " << mode.fine << " lowercase
" << mode.lowercase);
+ auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+ auto candidate = golden::make_text_driver(chinese_analyzer(mode.fine,
mode.lowercase));
+ for (const std::string& value : values) {
+ ASSERT_EQ(candidate->fresh(value), oracle->fresh(value))
+ << golden::format_line(oracle->fresh(value));
+ }
+ }
+}
+
+TEST(JiebaTokenizerTest, MatchesCLuceneOnMixedCaseUnknownWordsAndInvalidBytes)
{
+ JiebaDictScope dict;
+ const std::vector<std::string> values = {"Hello世界HELLO",
+ "我买了一件T恤和B超",
+ "AT&T和C++以及c#",
+ "MiXeD大小写MiXeD",
+ "𠮷野家的𪚥𪚥",
+ "ㄅㄆㄇ未登录词ㄈ",
+ "你好" + bytes({0xE4, 0xB8}),
+ "abc" + bytes({0xFF}) + "中文",
+ bytes({0x80}) + "中文",
+ "中文" + bytes({0xC3}),
+ std::string(600, 'A'),
+ "中" + std::string(300, 'b') + "文",
+ ""};
+ for (const Mode mode : kModes) {
+ SCOPED_TRACE(testing::Message() << "fine " << mode.fine << " lowercase
" << mode.lowercase);
+ auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+ auto candidate = golden::make_text_driver(chinese_analyzer(mode.fine,
mode.lowercase));
+ for (const std::string& value : values) {
+ expect_same_as_clucene(value, *candidate, *oracle);
+ }
+ }
+}
+
+// A value that does not decode has no token at all, the decodable head
included.
+TEST(JiebaTokenizerTest, AValueThatDoesNotDecodeHasNoToken) {
+ JiebaDictScope dict;
+ for (const Mode mode : kModes) {
+ EXPECT_TRUE(collect_terms(*chinese_analyzer(mode.fine, mode.lowercase),
+ "你好" + bytes({0xE4, 0xB8}))
+ .empty());
+ }
+}
+
+// The terms the old tokenizer handed out, one at a time: each word lowercased
where it lies when
+// its first byte is ASCII, and read right then, so that a later word's
lowercasing cannot reach it.
+std::vector<std::string> handed_out_one_at_a_time(const jieba::Jieba& jieba,
std::string value,
+ bool fine) {
+ std::vector<std::string_view> words;
+ if (fine) {
+ jieba.cut_all(value, words);
+ } else {
+ jieba.cut(value, words);
+ }
+ std::vector<std::string> terms;
+ for (const std::string_view word : words) {
+ if (!word.empty() && static_cast<uint8_t>(word[0]) < 0x80) {
+ char* begin = value.data() + (word.data() - value.data());
+ for (size_t i = 0; i < word.size(); ++i) {
+ begin[i] = ascii_lower(begin[i]);
+ }
+ }
+ terms.emplace_back(word.substr(0, MAX_WORD_LEN));
+ }
+ return terms;
+}
+
+// Random values with the words of kMixedWords in them, and runs of ASCII
letters and digits that
+// start with a stray continuation byte: after a separator or a Chinese
character, "\x80" "7abcB"
+// is one run whose first byte is not ASCII.
+std::vector<std::string> mixed_values() {
+ std::mt19937_64 random(20261108);
+ std::vector<std::string> values;
+ for (int i = 0; i < 2000; ++i) {
+ values.push_back(random_value(random) + "卡拉OK拉OK,\x80" + "7abcB" +
+ random_value(random) + "Q版本 \x80" + "9XyZ中\x80" +
"Ab1");
+ }
+ return values;
+}
+
+const std::vector<std::string> kMixedWords = {"卡拉OK", "OK", "拉O", "Q版本",
+ "AT&T", "C++", "中AB"};
+
+// The fine mode hands out a run of ASCII letters and digits whole and then
the runs of letters and
+// of digits in it, and a stray continuation byte reads as one character with
the byte after it:
+// "\x80" "7" is U+0037. So "\x807abcB" comes out keeping its capital, the
first byte not being
+// ASCII, and "abcB" lowers that capital after it; the first word must keep it
in the batch.
+TEST(JiebaTokenizerTest, BatchesHandOutWhatOneWordAtATimeWould) {
+ const OwnDict own(kMixedWords);
+ const std::shared_ptr<const jieba::Jieba> jieba = load_jieba(own.dir());
+ for (bool fine : {false, true}) {
+ JiebaTokenizerFactory factory(fine, true, own.dir());
+ auto tokenizer = factory.create();
+ for (const std::string& value : mixed_values()) {
+ ASSERT_EQ(collect_terms(*tokenizer, value),
+ handed_out_one_at_a_time(*jieba, value, fine))
+ << "fine " << fine << ": " << value;
+ }
+ }
+}
+
+// With lowercase on, a word is lowercased whole when its first byte is ASCII
and left as it is
+// otherwise, capitals and all.
+TEST(JiebaTokenizerTest, TheFirstByteDecidesWhetherAWordIsLowercased) {
+ const OwnDict own({"卡拉OK", "Q版本"});
+ JiebaTokenizerFactory coarse(false, true, own.dir());
+ EXPECT_EQ(collect_terms(*coarse.create(), "卡拉OK"),
(std::vector<std::string> {"卡拉OK"}));
+ EXPECT_EQ(collect_terms(*coarse.create(), "Q版本"),
(std::vector<std::string> {"q版本"}));
+ EXPECT_EQ(collect_terms(*coarse.create(), "T恤"), (std::vector<std::string>
{"t恤"}));
+ JiebaTokenizerFactory keep_case(false, false, own.dir());
+ EXPECT_EQ(collect_terms(*keep_case.create(), "Q版本"),
(std::vector<std::string> {"Q版本"}));
+}
+
+TEST(JiebaTokenizerTest, KeepsSupplementaryPlaneCharactersAndUserWords) {
+ const OwnDict own({"𠮷野家", "𪚥𪚥"});
+ for (bool fine : {false, true}) {
+ JiebaTokenizerFactory factory(fine, true, own.dir());
+ const std::vector<std::string> terms =
collect_terms(*factory.create(), "去𠮷野家吃𪚥𪚥饭");
+ EXPECT_NE(std::ranges::find(terms, "𠮷野家"), terms.end()) << fine;
+ EXPECT_NE(std::ranges::find(terms, "𪚥𪚥"), terms.end()) << fine;
+ const std::shared_ptr<const jieba::Jieba> jieba =
load_jieba(own.dir());
+ EXPECT_EQ(terms, handed_out_one_at_a_time(*jieba, "去𠮷野家吃𪚥𪚥饭", fine));
+ }
+}
+
+// How many of the values below the two sides cut differently, each printed.
+int mismatches_on_a_dictionary_of_its_own() {
+ const OwnDict own({"卡拉OK", "OK", "拉O", "Q版本", "𠮷野家", "𪚥𪚥", "云计算"});
+ const std::vector<std::string> values = {"卡拉OK",
+ "我们唱卡拉OK吧,OK",
+ "拉OK卡拉OK",
+ "Q版本和q版本",
+ "去𠮷野家吃𪚥𪚥饭",
+ "云计算KTV卡拉OK",
+ "OK卡拉OKOK",
+ "卡拉ok",
+ "T恤B超"};
+ int mismatches = 0;
+ for (const Mode mode : kModes) {
+ auto oracle = golden::make_lucene_driver([&own, mode] {
+ auto analyzer =
+
std::make_shared<lucene::analysis::LanguageBasedAnalyzer>(L"chinese", false);
+ analyzer->initDict(own.dir());
+ analyzer->setMode(mode.fine ? lucene::analysis::AnalyzerMode::All
+ :
lucene::analysis::AnalyzerMode::Default);
+ analyzer->set_lowercase(mode.lowercase);
+ return analyzer;
+ });
+ auto candidate =
golden::make_text_driver(std::make_shared<ChainAnalyzer>(
+ std::vector<CharFilterFactoryPtr> {},
+ std::make_shared<JiebaTokenizerFactory>(mode.fine,
mode.lowercase, own.dir()),
+ std::vector<TokenFilterFactoryPtr> {}));
+ for (const std::string& value : values) {
+ if (candidate->fresh(value) != oracle->fresh(value) ||
+ candidate->reused(value) != oracle->reused(value)) {
+ std::cerr << "fine " << mode.fine << " lowercase " <<
mode.lowercase << " " << value
+ << ": " << golden::format_line(oracle->fresh(value))
<< " vs "
+ << golden::format_line(candidate->fresh(value)) <<
"\n";
+ ++mismatches;
+ }
+ }
+ }
+ return mismatches;
+}
+
+// The live differential on a dictionary of the test's own, with words that
start with a Chinese
+// character and hold capitals, overlap ASCII words, or lie in a supplementary
plane: the shipped
+// dictionaries have none. CLucene's jieba keeps the first dictionary it loads
for the rest of the
+// process, so this runs in a process of its own, where it loads this one.
+TEST(JiebaTokenizerDeathTest, MatchesCLuceneOnADictionaryOfItsOwn) {
+ GTEST_FLAG_SET(death_test_style, "threadsafe");
+ // _Exit: static destructors of the test binary are not safe to run in the
child.
+ EXPECT_EXIT(std::_Exit(mismatches_on_a_dictionary_of_its_own() == 0 ? 0 :
1),
+ testing::ExitedWithCode(0), "");
+}
+
+TEST(JiebaTokenizerTest, LoadsWithoutIdf) {
+ const OwnDict own({"云计算"}, {"idf.utf8"});
+ JiebaTokenizerFactory factory(false, false, own.dir());
+ EXPECT_EQ(collect_terms(*factory.create(), "云计算"),
(std::vector<std::string> {"云计算"}));
+}
+
+// A missing file fails the factory, and a later load of the same directory
tries again.
+TEST(JiebaTokenizerTest, AFailedLoadIsTriedAgain) {
+ const OwnDict own({}, {"hmm_model.utf8"});
+ try {
+ JiebaTokenizerFactory factory(false, false, own.dir());
+ FAIL() << "a missing hmm_model.utf8 must fail";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::INVERTED_INDEX_ANALYZER_ERROR);
+ EXPECT_NE(std::string(e.what()).find("hmm_model.utf8"),
std::string::npos);
+ }
+ own.add("hmm_model.utf8");
+ JiebaTokenizerFactory factory(false, false, own.dir());
+ EXPECT_EQ(collect_terms(*factory.create(), "中华人民共和国"),
+ (std::vector<std::string> {"中华人民共和国"}));
+}
+
+TEST(JiebaTokenizerTest, EightThreadsShareTheDictionaries) {
+ JiebaDictScope dict;
+ std::mt19937_64 random(20261107);
+ std::vector<std::string> values;
+ for (int i = 0; i < 400; ++i) {
+ values.push_back(random_value(random));
+ }
+ for (const Mode mode : kModes) {
+ const TextAnalyzerPtr analyzer = chinese_analyzer(mode.fine,
mode.lowercase);
+ std::vector<std::vector<std::string>> expected;
+ for (const std::string& value : values) {
+ expected.push_back(collect_terms(*analyzer, value));
+ }
+ std::vector<int> mismatches(8, 0);
+ std::vector<std::thread> threads;
+ for (int t = 0; t < 8; ++t) {
+ threads.emplace_back([&, t] {
+ auto stream = analyzer->create_stream();
+ for (int round = 0; round < 3; ++round) {
+ for (size_t i = 0; i < values.size(); ++i) {
+ mismatches[t] += collect_terms(*stream, values[i]) !=
expected[i];
+ }
+ }
+ });
+ }
+ for (std::thread& thread : threads) {
+ thread.join();
+ }
+ EXPECT_EQ(mismatches, std::vector<int>(8, 0));
+ }
+}
+
+TEST(JiebaTokenizerTest, BatchesLoseNoTokenAroundTheBatchSize) {
+ JiebaDictScope dict;
+ for (size_t count : {size_t {255}, size_t {256}, size_t {257}, size_t
{512}}) {
+ std::string value;
+ for (size_t i = 0; i < count; ++i) {
+ value += "w" + std::to_string(i) + " ";
+ }
+ for (const Mode mode : kModes) {
+ auto oracle = clucene_chinese(mode.fine, mode.lowercase);
+ auto candidate =
golden::make_text_driver(chinese_analyzer(mode.fine, mode.lowercase));
+ expect_same_as_clucene(value, *candidate, *oracle);
+ }
+ }
+}
+
+TEST(JiebaTokenizerTest, KeepsTheStreamContract) {
+ JiebaDictScope dict;
+ std::string many_batches;
+ for (int i = 0; i < 1500; ++i) {
+ many_batches += "我们研究" + std::to_string(i % 97) + "号AB ";
+ }
+ for (const Mode mode : kModes) {
+ const TextAnalyzerPtr analyzer = chinese_analyzer(mode.fine,
mode.lowercase);
+ expect_text_stream_contract([&] { return analyzer->create_stream(); },
+ {"", "我们", "中华人民共和国", "Hello世界", "a",
+ "你好" + bytes({0xE4, 0xB8}),
many_batches});
+ }
+}
+
+} // namespace
+} // namespace doris::segment_v2::inverted_index
diff --git a/build-support/check-header-deps.py
b/build-support/check-header-deps.py
index 2a646d8fc18..d63a83f469f 100755
--- a/build-support/check-header-deps.py
+++ b/build-support/check-header-deps.py
@@ -655,8 +655,6 @@ CLUCENE_INCLUDING_FILES = {
"storage/index/index_file_writer.h",
"storage/index/index_fs_directory.h",
"storage/index/inverted/clucene/inverted_index_searcher.h",
- # The analysis layer (T3, T4, T8, T9).
- "storage/index/inverted/analysis/analyzer/analyzer.cpp",
# SNII (T3, T12, T13).
"storage/index/inverted/snii/snii_blob_directory.h",
"storage/index/inverted/snii/snii_blob_staging_directory.h",
@@ -671,8 +669,6 @@ CLUCENE_SYMBOL_FILES = {
"storage/index/ann/ann_index_writer.h",
"storage/index/ann/faiss_ann_index.cpp",
"storage/index/ann/faiss_ann_index.h",
- # The analysis layer (T3, T4, T8, T9).
- "storage/index/inverted/analysis/analyzer/analyzer.cpp",
# SNII (T3, T12, T13).
"storage/index/inverted/snii/snii_blob_directory.cpp",
"storage/index/inverted/snii/snii_blob_directory.h",
diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt
index 153c799845d..0955e9ac55f 100644
--- a/dist/LICENSE-dist.txt
+++ b/dist/LICENSE-dist.txt
@@ -1515,6 +1515,8 @@ Other dependencies:
* pugixml 1.15 -- licenses/LICENSE-pugixml.txt
* darts-clone (double-array trie, compiled into the kuromoji dictionary)
-- licenses/LICENSE-darts-clone.txt
* mecab-ipadic (IPADIC) 2.7.0-20250920 (Japanese dictionary data shipped
under be/dict/kuromoji) -- licenses/LICENSE-ipadic.txt
+ * cppjieba (Chinese word segmentation, ported into the chinese parser) --
licenses/LICENSE-cppjieba.txt
+ * limonp (helpers of cppjieba, ported with it) --
licenses/LICENSE-limonp.txt
* MongoDB C Driver (libmongoc/libbson): 2.5.8 -- Apache License 2.0;
bundled notices in licenses/LICENSE-mongo-c-driver.txt
* MongoDB Java Sync Driver (sync/core/bson/bson-record-codec): 5.14.0 --
Apache License 2.0
diff --git a/dist/licenses/LICENSE-cppjieba.txt
b/dist/licenses/LICENSE-cppjieba.txt
new file mode 100644
index 00000000000..6308e939b73
--- /dev/null
+++ b/dist/licenses/LICENSE-cppjieba.txt
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/dist/licenses/LICENSE-limonp.txt b/dist/licenses/LICENSE-limonp.txt
new file mode 100644
index 00000000000..6308e939b73
--- /dev/null
+++ b/dist/licenses/LICENSE-limonp.txt
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]