This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7a9d745388b [fix](inverted index) Align phrase position defaults with
analysis (#69199)
7a9d745388b is described below
commit 7a9d745388bd8b71cedcbc91c95747cd4758ab27
Author: Jack <[email protected]>
AuthorDate: Sun Oct 18 20:33:12 2026 +0800
[fix](inverted index) Align phrase position defaults with analysis (#69199)
### What problem does this PR solve?
Issue Number: CIR-22263
Related PR: #69192
Problem Summary: FE persists `support_phrase=true` for new tokenized
indexes and leaves exact indexes docs-only. BE must treat an omitted
property as false because older persisted analyzer indexes can omit it
despite having docs-only postings; this also preserves `TOKENIZE`
output. Built-in `lowercase` normalization emits one term, so phrase
predicates can use docs-only postings and BM25 can use unit frequency
and document length. Custom normalizers can emit multiple terms, so new
indexes retain positions by default, including when a legacy custom
policy shadows the built-in name.
This PR changes phrase-position defaults and normalizer compatibility.
PR #69192 adds direct SNII compaction of compatible docs-only postings.
Neither change addresses the tokenized T2 merge hotspot identified in
CIR-22263.
---
be/src/storage/index/index_reader_helper.h | 25 +++++++--
.../inverted/clucene/spi/clucene_index_source.cpp | 1 +
.../inverted/clucene/spi/clucene_postings_cursor.h | 9 ++-
.../inverted/reader/collection_statistics.cpp | 30 +++++++---
.../index/inverted/reader/collection_statistics.h | 10 ++--
.../analysis/inverted_index_parser_test.cpp | 23 ++++++++
.../query/engine/boolean_postings_bench_test.cpp | 4 +-
.../query/engine/multi_segment_collector_test.cpp | 5 +-
.../inverted/query/exec/block_doc_set_test.cpp | 16 ++++++
.../inverted/reader/collection_statistics_test.cpp | 65 ++++++++++++++++++----
.../index/inverted/snii/snii_writer_test.cpp | 51 +++++++++++++++++
.../storage/segment/index_reader_helper_test.cpp | 43 ++++++++++++++
.../main/java/org/apache/doris/catalog/Index.java | 11 ++--
.../apache/doris/analysis/InvertedIndexUtil.java | 13 ++++-
.../analysis/InvertedIndexPropertiesTest.java | 5 +-
.../java/org/apache/doris/catalog/IndexTest.java | 35 ++++++++++++
.../suites/index_p0/test_index_meta.groovy | 4 +-
17 files changed, 307 insertions(+), 43 deletions(-)
diff --git a/be/src/storage/index/index_reader_helper.h
b/be/src/storage/index/index_reader_helper.h
index bd2aacdabb1..c1b0b65008d 100644
--- a/be/src/storage/index/index_reader_helper.h
+++ b/be/src/storage/index/index_reader_helper.h
@@ -17,6 +17,10 @@
#pragma once
+#include <boost/algorithm/string/predicate.hpp>
+#include <map>
+#include <string>
+
#include "storage/index/index_iterator.h"
#include "storage/index/inverted/inverted_index_reader.h"
@@ -24,6 +28,16 @@ namespace doris::segment_v2 {
class IndexReaderHelper {
public:
+ static bool is_normalizer_only(const std::map<std::string, std::string>&
properties) {
+ const auto normalizer =
properties.find(INVERTED_INDEX_NORMALIZER_NAME_KEY);
+ const auto analyzer =
properties.find(INVERTED_INDEX_ANALYZER_NAME_KEY);
+ // A custom normalizer can split a value into multiple terms that need
positions.
+ return !properties.contains(INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY)
&&
+ normalizer != properties.end() &&
boost::iequals(normalizer->second, "lowercase") &&
+ (analyzer == properties.end() || analyzer->second.empty()) &&
+ get_parser_string_from_properties(properties) ==
INVERTED_INDEX_PARSER_NONE;
+ }
+
static bool is_fulltext_index(const IndexReaderPtr& reader) {
if (reader == nullptr || reader->index_type() != IndexType::INVERTED) {
return false;
@@ -59,7 +73,8 @@ public:
auto inverted_index_reader =
std::static_pointer_cast<InvertedIndexReader>(reader);
const auto& properties = inverted_index_reader->get_index_properties();
return get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
+ is_normalizer_only(properties);
}
// only string type or bkd index reader can be used for equal
@@ -96,7 +111,8 @@ public:
query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) {
const auto& properties = index_meta->properties();
if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
+ is_normalizer_only(properties)) {
return true;
}
}
@@ -110,7 +126,8 @@ public:
query_type == TExprOpcode::MATCH_PHRASE_PREFIX) {
const auto& properties = index_meta->properties();
if (get_parser_phrase_support_string_from_properties(properties) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES ||
+ is_normalizer_only(properties)) {
return true;
}
}
@@ -118,4 +135,4 @@ public:
}
};
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/be/src/storage/index/inverted/clucene/spi/clucene_index_source.cpp
b/be/src/storage/index/inverted/clucene/spi/clucene_index_source.cpp
index b7be72f4770..c509c07a2a6 100644
--- a/be/src/storage/index/inverted/clucene/spi/clucene_index_source.cpp
+++ b/be/src/storage/index/inverted/clucene/spi/clucene_index_source.cpp
@@ -68,6 +68,7 @@ ClucenePostingsCursor::Partition
open_postings(lucene::index::IndexReader* reade
lucene::index::Term* term, bool
positions,
bool scoring, const
io::IOContext* io_ctx) {
ClucenePostingsCursor::Partition result;
+ result.has_norms = reader->hasNorms(term->field());
if (positions) {
auto iter = make_term_positions_ptr(reader, scoring, io_ctx);
result.positions = iter.get();
diff --git
a/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
b/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
index 2786eb7a811..2bb3406eaad 100644
--- a/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
+++ b/be/src/storage/index/inverted/clucene/spi/clucene_postings_cursor.h
@@ -39,12 +39,14 @@ public:
lucene::index::TermPositions* positions = nullptr;
uint32_t begin = 0;
uint32_t end = 0;
+ bool has_norms = true;
};
explicit ClucenePostingsCursor(Partition part)
: _single_iter(std::move(part.iter)),
_iter(_single_iter.get()),
- _raw_positions(part.positions) {
+ _raw_positions(part.positions),
+ _has_norms(part.has_norms) {
_check_iterator();
_doc_freq = _iter->docFreq();
}
@@ -87,12 +89,13 @@ public:
DORIS_CHECK(_block.freq_many != nullptr);
DORIS_CHECK_GE(_block.freq_many_size_,
_block.doc_many_size_);
}
+ // CLucene fills norm_many with zeroes when the field omits
norms.
*out = {.docs = {_block.doc_many->data(),
_block.doc_many_size_},
.freqs = _block.freq_many
? std::span<const
uint32_t>(_block.freq_many->data(),
_block.freq_many_size_)
: std::span<const uint32_t>(),
- .norms = _block.norm_many
+ .norms = _has_norms && _block.norm_many
? std::span<const
uint32_t>(_block.norm_many->data(),
_block.norm_many_size_)
: std::span<const uint32_t>()};
@@ -312,6 +315,7 @@ private:
_iter = part.iter.get();
_raw_positions = part.positions;
_doc_base = part.begin;
+ _has_norms = part.has_norms;
}
}
@@ -376,6 +380,7 @@ private:
const uint32_t* _freqs = nullptr;
uint32_t _prox_cursor = 0;
lucene::index::TermPositions* _raw_positions = nullptr;
+ bool _has_norms = true;
uint32_t _position_remaining = 0;
uint32_t _position = 0;
bool _position_open = false;
diff --git a/be/src/storage/index/inverted/reader/collection_statistics.cpp
b/be/src/storage/index/inverted/reader/collection_statistics.cpp
index 67c4ac348a5..276b1d2c2dd 100644
--- a/be/src/storage/index/inverted/reader/collection_statistics.cpp
+++ b/be/src/storage/index/inverted/reader/collection_statistics.cpp
@@ -43,8 +43,10 @@ namespace collection_statistics_detail {
Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t
index_doc_count,
uint64_t
sum_total_term_freq,
- bool
has_positions, bool has_norms) {
- if (!has_positions || !has_norms) {
+ bool
has_positions, bool has_norms,
+ bool
single_term_normalizer) {
+ // Lowercase-normalized docs-only postings use unit frequency and length
for scoring.
+ if ((!has_positions || !has_norms) && !single_term_normalizer) {
return
ResultError(Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, false>(
"SNII scoring requires positions and norms; this segment was
written without "
"positions or without norms. Norms are left out when the index
sets \"norms\" = "
@@ -262,7 +264,10 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
const uint64_t segment_doc_count = scoring_stats.doc_count;
RETURN_IF_ERROR(admit_snii_scoring_segment(
ws_field_name, segment_doc_count,
scoring_stats.token_count,
- scoring_stats.has_positions, scoring_stats.has_norms,
&segment_accumulator));
+ scoring_stats.has_positions, scoring_stats.has_norms,
+ segment_v2::IndexReaderHelper::is_normalizer_only(
+ collect_info.index_meta->properties()),
+ &segment_accumulator));
for (const auto& logical_term_bytes : collect_info.unique_terms) {
const auto logical_term =
@@ -332,9 +337,12 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
// BM25 on an analyzed index needs the record length of every row, and
CLucene keeps
// them, together with the field's token count, in the norms. A
segment written without
// norms would feed a zero avgdl, or rank its rows as zero-length
documents next to the
- // segments that have norms, so refuse to score the collection, as
SNII does. An index
- // that is not analyzed never writes norms and is left as it is.
- if (!scoring_stats.has_norms &&
+ // segments that have norms, so refuse to score the collection, as
SNII does. A
+ // single-term normalizer can instead use unit frequency and length.
+ const bool single_term_normalizer =
segment_v2::IndexReaderHelper::is_normalizer_only(
+ collect_info.index_meta->properties());
+ // Docs-only lowercase indexes have no norms, but each matching row
has one term.
+ if (!scoring_stats.has_norms && !single_term_normalizer &&
segment_v2::inverted_index::InvertedIndexAnalyzer::should_analyzer(
collect_info.index_meta->properties())) {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
@@ -343,7 +351,10 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
"for a variant path, when
inverted_index_skip_norms_for_variant is on",
seg_path,
segment_v2::inverted_index::StringHelper::to_string(ws_field_name));
}
- _total_num_tokens[ws_field_name] += scoring_stats.token_count;
+ // CLucene omits term totals without norms; one token per row gives a
unit-length estimate.
+ _total_num_tokens[ws_field_name] += !scoring_stats.has_norms &&
single_term_normalizer
+ ? scoring_stats.doc_count
+ :
scoring_stats.token_count;
for (const auto& logical_term_bytes : collect_info.unique_terms) {
const auto logical_term =
@@ -364,10 +375,11 @@ Status CollectionStatistics::process_segment(const
RowsetSharedPtr& rowset,
Status CollectionStatistics::admit_snii_scoring_segment(
const std::wstring& field_name, uint64_t index_doc_count, uint64_t
sum_total_term_freq,
- bool has_positions, bool has_norms, SniiScoringSegmentAccumulator*
segment_accumulator) {
+ bool has_positions, bool has_norms, bool single_term_normalizer,
+ SniiScoringSegmentAccumulator* segment_accumulator) {
DORIS_CHECK(segment_accumulator != nullptr);
auto segment_stats =
collection_statistics_detail::resolve_snii_scoring_segment(
- index_doc_count, sum_total_term_freq, has_positions, has_norms);
+ index_doc_count, sum_total_term_freq, has_positions, has_norms,
single_term_normalizer);
if (!segment_stats.has_value()) {
clear();
return segment_stats.error();
diff --git a/be/src/storage/index/inverted/reader/collection_statistics.h
b/be/src/storage/index/inverted/reader/collection_statistics.h
index 62e6e73ad26..e62258c2b81 100644
--- a/be/src/storage/index/inverted/reader/collection_statistics.h
+++ b/be/src/storage/index/inverted/reader/collection_statistics.h
@@ -86,7 +86,7 @@ private:
io::IOContext* io_ctx);
Status admit_snii_scoring_segment(const std::wstring& field_name, uint64_t
index_doc_count,
uint64_t sum_total_term_freq, bool
has_positions,
- bool has_norms,
+ bool has_norms, bool
single_term_normalizer,
SniiScoringSegmentAccumulator*
segment_accumulator);
void commit_snii_scoring_segment(SniiScoringSegmentAccumulator&&
segment_accumulator);
void clear();
@@ -121,12 +121,12 @@ struct SniiScoringSegmentStats {
uint64_t token_count = 0;
};
-// SNII scoring requires positions (which provide term frequencies) and norms.
The current writer
-// emits norms for every analyzed index with positions. Older segments without
norms return
-// NOT_SUPPORTED until an index rebuild or compaction supplies them.
+// Single-term normalizers can score docs-only postings with frequency and
length one.
+// Other analyzed indexes need positions and norms for accurate BM25 scores.
Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t
index_doc_count,
uint64_t
sum_total_term_freq,
- bool
has_positions, bool has_norms);
+ bool
has_positions, bool has_norms,
+ bool
single_term_normalizer = false);
void add_term_doc_frequency(
std::unordered_map<std::wstring, std::unordered_map<std::wstring,
uint64_t>>*
diff --git
a/be/test/storage/index/inverted/analysis/inverted_index_parser_test.cpp
b/be/test/storage/index/inverted/analysis/inverted_index_parser_test.cpp
index 30b065b24d5..526f61461d9 100644
--- a/be/test/storage/index/inverted/analysis/inverted_index_parser_test.cpp
+++ b/be/test/storage/index/inverted/analysis/inverted_index_parser_test.cpp
@@ -130,6 +130,29 @@ TEST_F(InvertedIndexParserTest,
TestGetParserPhraseSupportStringFromProperties)
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+ properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_NONE;
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH;
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties.clear();
+ properties[INVERTED_INDEX_ANALYZER_NAME_KEY] = "english";
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties.clear();
+ properties[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "lowercase";
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
+ properties.clear();
+ properties[INVERTED_INDEX_PARSER_KEY_ALIAS] = INVERTED_INDEX_PARSER_NONE;
+ EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+
// Test with phrase support key present
properties[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] =
INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
EXPECT_EQ(get_parser_phrase_support_string_from_properties(properties),
diff --git
a/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
b/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
index c19e4f9f4ab..6e1e2dfc717 100644
---
a/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
+++
b/be/test/storage/index/inverted/query/engine/boolean_postings_bench_test.cpp
@@ -202,7 +202,7 @@ uint64_t posting_row_checksum(uint32_t doc, float score,
bool scoring) {
uint64_t expected_posting_checksum(const PostingSpec& spec, uint32_t profile,
bool sparse,
bool scoring) {
BM25Similarity similarity(2.0F, 8.0F);
- const float term_score = similarity.score(1.0F, 0);
+ const float term_score = similarity.score(1.0F, 1);
uint64_t result = 0;
for (uint32_t doc = 0; doc < kPostingRows; ++doc) {
bool required = true;
@@ -255,7 +255,7 @@ PhysicalTruthRows expected_physical_truth(const
PostingSpec& spec, uint32_t prof
PhysicalTruthRows result;
result.scores.resize(kPostingRows);
BM25Similarity similarity(2.0F, 8.0F);
- const float term_score = similarity.score(1.0F, 0);
+ const float term_score = similarity.score(1.0F, 1);
for (uint32_t doc = 0; doc < kPostingRows; ++doc) {
bool required_true = true;
bool required_possible = true;
diff --git
a/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
b/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
index fcf80f0c197..d2e096fffc0 100644
---
a/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
+++
b/be/test/storage/index/inverted/query/engine/multi_segment_collector_test.cpp
@@ -249,7 +249,8 @@ static void check_native_posting_block(const
index_query::PostingsBlock& block,
EXPECT_EQ(block.doc_at(0), doc);
EXPECT_EQ(block.docs.front(), doc);
EXPECT_EQ(block.freq_at(0), 1);
- EXPECT_EQ(block.norm_at(0), 0);
+ EXPECT_TRUE(block.norms.empty());
+ EXPECT_EQ(block.norm_at(0), 1);
}
static void check_partition_position(index_query::PostingsCursor& cursor,
uint32_t position) {
@@ -407,7 +408,7 @@ TEST_F(MultiSegmentCollectorTest,
PhraseCandidatesRemainValidAfterWeightDestruct
auto scorer = weight->scorer(segment, "");
weight.reset();
ASSERT_EQ(scorer->doc(), 3);
- const float expected_score = scoring ? similarity->score(1.0F, 0)
: 1.0F;
+ const float expected_score = scoring ? similarity->score(1.0F, 1)
: 1.0F;
EXPECT_FLOAT_EQ(scorer->score(), expected_score);
EXPECT_EQ(scorer->seek(3), 3);
EXPECT_EQ(scorer->advance(), TERMINATED);
diff --git a/be/test/storage/index/inverted/query/exec/block_doc_set_test.cpp
b/be/test/storage/index/inverted/query/exec/block_doc_set_test.cpp
index d5e631f149b..a28514a757a 100644
--- a/be/test/storage/index/inverted/query/exec/block_doc_set_test.cpp
+++ b/be/test/storage/index/inverted/query/exec/block_doc_set_test.cpp
@@ -67,6 +67,22 @@ private:
size_t _next = 0;
};
+TEST(BlockDocSetTest, MissingNormsUseUnitDocumentLength) {
+ const std::array<uint32_t, 1> docs {5};
+ PostingsBlock block {.docs = docs,
+ .freqs = {},
+ .norms = {},
+ .range_begin = 0,
+ .range_end = 0,
+ .dense = false};
+ EXPECT_EQ(block.freq_at(0), 1U);
+ EXPECT_EQ(block.norm_at(0), 1U);
+
+ const std::array<uint32_t, 1> stored_norm {0};
+ block.norms = stored_norm;
+ EXPECT_EQ(block.norm_at(0), 0U);
+}
+
TEST(BlockDocSetTest, ReadFailureDoesNotBecomeEndOfPostings) {
TestBlockCursor source({{.docs = {},
.freqs = {},
diff --git
a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
index 6d99585c55b..6d00d6da478 100644
--- a/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
+++ b/be/test/storage/index/inverted/reader/collection_statistics_test.cpp
@@ -414,7 +414,7 @@ protected:
return tablet_schema;
}
- TabletSchemaSPtr create_snii_schema(int64_t index_id = 1) {
+ TabletSchemaSPtr create_snii_schema(int64_t index_id = 1, bool
single_term_normalizer = false) {
TabletSchemaPB schema_pb;
schema_pb.set_keys_type(DUP_KEYS);
schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII);
@@ -431,8 +431,12 @@ protected:
index._index_id = index_id;
index._index_type = IndexType::INVERTED;
index._col_unique_ids.push_back(1);
- index._properties["parser"] = "standard";
- index._properties["support_phrase"] = "true";
+ if (single_term_normalizer) {
+ index._properties["normalizer"] = "lowercase";
+ } else {
+ index._properties["parser"] = "standard";
+ index._properties["support_phrase"] = "true";
+ }
tablet_schema->append_index(std::move(index));
return tablet_schema;
}
@@ -470,7 +474,7 @@ protected:
}
// A normal analyzed SNII segment with positions and norms, as emitted for
scoring indexes.
- Status write_snii_scoring_segment(const std::string& segment_path) {
+ Status write_snii_scoring_segment(const std::string& segment_path, bool
docs_only = false) {
const std::string index_path_prefix {
segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)};
io::FileWriterPtr file_writer;
@@ -486,8 +490,11 @@ protected:
snii::writer::TermPostings alpha;
alpha.term = "alpha";
alpha.docids = {0, 1};
- alpha.freqs = {1, 1};
- alpha.positions_flat = {0, 0};
+ alpha.retain_positions = !docs_only;
+ if (!docs_only) {
+ alpha.freqs = {1, 1};
+ alpha.positions_flat = {0, 0};
+ }
snii::writer::TermPostings beta;
beta.term = "beta";
@@ -497,10 +504,15 @@ protected:
snii::writer::SniiIndexInput input;
input.index_id = 1;
- input.config = snii::format::IndexConfig::kDocsPositions;
+ input.config = docs_only ? snii::format::IndexConfig::kDocsOnly
+ : snii::format::IndexConfig::kDocsPositions;
input.doc_count = 2;
- input.encoded_norms = {snii::query::encode_norm(2),
snii::query::encode_norm(1)};
- input.terms = {std::move(alpha), std::move(beta)};
+ if (!docs_only) {
+ input.encoded_norms = {snii::query::encode_norm(2),
snii::query::encode_norm(1)};
+ input.terms = {std::move(alpha), std::move(beta)};
+ } else {
+ input.terms = {std::move(alpha)};
+ }
RETURN_IF_ERROR(writer.add_logical_index(input));
RETURN_IF_ERROR(writer.finish());
@@ -655,6 +667,7 @@ protected:
uint64_t sum_total_term_freq = 0;
bool has_positions = true;
bool has_norms = true;
+ bool single_term_normalizer = false;
};
Status stage_snii_fields_for_test(
@@ -663,7 +676,8 @@ protected:
for (const auto& field : fields) {
RETURN_IF_ERROR(statistics->admit_snii_scoring_segment(
field.field_name, field.index_doc_count,
field.sum_total_term_freq,
- field.has_positions, field.has_norms,
segment_accumulator));
+ field.has_positions, field.has_norms,
field.single_term_normalizer,
+ segment_accumulator));
}
return Status::OK();
}
@@ -1006,6 +1020,27 @@ TEST_F(CollectionStatisticsTest,
SniiScoringUsesPhysicalStatistics) {
expect_collected_term(L"1", L"alpha", 2);
}
+TEST_F(CollectionStatisticsTest,
SniiDocsOnlyNormalizerStillCollectsScoringStatistics) {
+ auto tablet_schema = create_snii_schema(1,
/*single_term_normalizer=*/true);
+ const std::string segment_path = test_dir_ + "/snii_normalizer_0.dat";
+ auto write_status = write_snii_scoring_segment(segment_path,
/*docs_only=*/true);
+ ASSERT_TRUE(write_status.ok()) << write_status;
+
+ auto rowset_meta =
std::make_shared<collection_statistics::MockRowsetMeta>();
+ auto rowset =
std::make_shared<collection_statistics::MockRowset>(tablet_schema, rowset_meta);
+ rowset->set_num_segments(1);
+ rowset->set_segment_path(0, segment_path);
+ auto reader =
std::make_shared<collection_statistics::MockRowsetReader>(rowset);
+ std::vector<RowSetSplits> splits {RowSetSplits(reader)};
+
+ auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema,
+ create_match_expr_contexts("alpha"),
nullptr);
+
+ ASSERT_TRUE(status.ok()) << status;
+ expect_collected_stats(L"1", 2, 2);
+ expect_collected_term(L"1", L"alpha", 2);
+}
+
TEST_F(CollectionStatisticsTest, SniiScoringLookupUsesCallerIoContext) {
snii::snii_test::ScopedEnv
force_nonresident_dict("SNII_DICT_RESIDENT_MAX", "0");
auto tablet_schema = create_snii_schema();
@@ -1194,6 +1229,16 @@ TEST(CollectionStatisticsSniiScoringTest,
ResolveRejectsSegmentWithoutPositions)
EXPECT_EQ(result.error().code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
}
+TEST(CollectionStatisticsSniiScoringTest,
ResolveAcceptsSingleTermNormalizerWithoutPositions) {
+ auto result = resolve_snii_scoring_segment(3, 2, /*has_positions=*/false,
+ /*has_norms=*/false,
+
/*single_term_normalizer=*/true);
+
+ ASSERT_TRUE(result.has_value()) << result.error();
+ EXPECT_EQ(result->doc_count, 3U);
+ EXPECT_EQ(result->token_count, 2U);
+}
+
TEST_F(CollectionStatisticsTest,
CollectionStatisticsInstancesKeepAdmissionStateIsolated) {
CollectionStatistics first;
CollectionStatistics second;
diff --git a/be/test/storage/index/inverted/snii/snii_writer_test.cpp
b/be/test/storage/index/inverted/snii/snii_writer_test.cpp
index 2383dfd6e4c..8be7f7bba98 100644
--- a/be/test/storage/index/inverted/snii/snii_writer_test.cpp
+++ b/be/test/storage/index/inverted/snii/snii_writer_test.cpp
@@ -137,6 +137,28 @@ private:
doris::IndexPolicyMgr* previous_ = nullptr;
};
+class ScopedSplitNormalizerPolicy {
+public:
+ ScopedSplitNormalizerPolicy() {
+ auto* exec_env = doris::ExecEnv::GetInstance();
+ previous_ = exec_env->index_policy_mgr();
+ exec_env->_index_policy_mgr = &manager_;
+
+ doris::TIndexPolicy normalizer;
+ normalizer.id = 9020;
+ normalizer.name = "split_norm";
+ normalizer.type = doris::TIndexPolicyType::NORMALIZER;
+ normalizer.properties["token_filter"] = "word_delimiter";
+ manager_.apply_policy_changes({normalizer}, {});
+ }
+
+ ~ScopedSplitNormalizerPolicy() {
doris::ExecEnv::GetInstance()->_index_policy_mgr = previous_; }
+
+private:
+ doris::IndexPolicyMgr manager_;
+ doris::IndexPolicyMgr* previous_ = nullptr;
+};
+
// A fatal assertion inside a helper FUNCTION only aborts the helper; the
calling
// test keeps running and may dereference state that failed to initialize (this
// bit us as a null-analyzer SEGV). A macro expands in the test body, so the
@@ -551,6 +573,35 @@ TEST(SniiWriterNorms, WritesNormsFollowSharedNormsPolicy) {
doris::config::inverted_index_skip_norms_for_variant =
original_skip_norms_for_variant;
}
+TEST(SniiWriterTest, SplitNormalizerRetainsPhrasePositions) {
+ ScopedSplitNormalizerPolicy policies;
+ doris::TabletIndex index_meta;
+ init_gram_index_meta(&index_meta, 9021,
+ {{"normalizer", "split_norm"}, {"support_phrase",
"true"}});
+
+ doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta,
+
doris::FieldType::OLAP_FIELD_TYPE_VARCHAR);
+ ASSERT_OK(writer.init());
+ ASSERT_EQ(writer.config_for_test(), IndexConfig::kDocsPositions);
+
+ const doris::Slice value("foo-bar");
+ ASSERT_OK(writer.add_values("content", &value, 1));
+ auto postings = writer.term_buffer_for_test()->finalize_sorted();
+ bool saw_foo = false;
+ bool saw_bar = false;
+ for (const auto& posting : postings) {
+ if (posting.term == "foo") {
+ saw_foo = true;
+ EXPECT_EQ(posting.positions_flat, std::vector<uint32_t>({1}));
+ } else if (posting.term == "bar") {
+ saw_bar = true;
+ EXPECT_EQ(posting.positions_flat, std::vector<uint32_t>({2}));
+ }
+ }
+ EXPECT_TRUE(saw_foo);
+ EXPECT_TRUE(saw_bar);
+}
+
TEST(SniiWriterTest, GramTokenizerForcesDocsOnlyAndIsRecognised) {
ScopedConfiguredDensity fixed_density;
ScopedGramPolicies policies;
diff --git a/be/test/storage/segment/index_reader_helper_test.cpp
b/be/test/storage/segment/index_reader_helper_test.cpp
index d2038e8bd8f..b7a6da09eda 100644
--- a/be/test/storage/segment/index_reader_helper_test.cpp
+++ b/be/test/storage/segment/index_reader_helper_test.cpp
@@ -396,6 +396,49 @@ TEST_F(IndexReaderHelperTest,
IsNeedSimilarityScoreWithInvertedIndexQueryTypeTes
InvertedIndexQueryType::LESS_THAN_QUERY, &index_meta));
}
+TEST_F(IndexReaderHelperTest, NormalizerOnlyPhraseAndScoringNeedNoPositions) {
+ TabletIndexPB index_pb;
+ index_pb.set_index_type(IndexType::INVERTED);
+ index_pb.set_index_id(2);
+ index_pb.set_index_name("normalized");
+ index_pb.add_col_unique_id(1);
+ (*index_pb.mutable_properties())[INVERTED_INDEX_NORMALIZER_NAME_KEY] =
"lowercase";
+ TabletIndex index_meta;
+ index_meta.init_from_pb(index_pb);
+
+ auto reader =
std::make_shared<MockInvertedIndexReader>(InvertedIndexReaderType::FULLTEXT,
+
index_meta.properties());
+
EXPECT_EQ(get_parser_phrase_support_string_from_properties(index_meta.properties()),
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO);
+ EXPECT_TRUE(IndexReaderHelper::is_support_phrase(reader));
+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(InvertedIndexQueryType::MATCH_ANY_QUERY,
+ &index_meta));
+
EXPECT_TRUE(IndexReaderHelper::is_need_similarity_score(TExprOpcode::MATCH_ANY,
&index_meta));
+
+ auto explicit_disabled = index_meta.properties();
+ explicit_disabled[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] =
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO;
+ auto disabled_reader = std::make_shared<MockInvertedIndexReader>(
+ InvertedIndexReaderType::FULLTEXT, explicit_disabled);
+ EXPECT_FALSE(IndexReaderHelper::is_support_phrase(disabled_reader));
+
+ auto builtin_alias = index_meta.properties();
+ builtin_alias[INVERTED_INDEX_NORMALIZER_NAME_KEY] = "LowerCase";
+ auto alias_reader =
std::make_shared<MockInvertedIndexReader>(InvertedIndexReaderType::FULLTEXT,
+
builtin_alias);
+ EXPECT_TRUE(IndexReaderHelper::is_support_phrase(alias_reader));
+
+ TabletIndexPB custom_pb = index_pb;
+ (*custom_pb.mutable_properties())[INVERTED_INDEX_NORMALIZER_NAME_KEY] =
"split_norm";
+ TabletIndex custom_meta;
+ custom_meta.init_from_pb(custom_pb);
+ auto custom_reader = std::make_shared<MockInvertedIndexReader>(
+ InvertedIndexReaderType::FULLTEXT, custom_meta.properties());
+ EXPECT_FALSE(IndexReaderHelper::is_support_phrase(custom_reader));
+ EXPECT_FALSE(IndexReaderHelper::is_need_similarity_score(
+ InvertedIndexQueryType::MATCH_ANY_QUERY, &custom_meta));
+}
+
TEST_F(IndexReaderHelperTest, IsNeedSimilarityScoreWithTExprOpcodeTest) {
TabletIndex index_meta;
auto index_meta_pb = std::make_unique<TabletIndexPB>();
diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java
b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java
index ee85e1daf93..4045e07f7f7 100644
--- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java
+++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Index.java
@@ -75,10 +75,13 @@ public class Index implements Writable {
this.comment = comment;
if (indexType == IndexType.INVERTED) {
if (this.properties != null && !this.properties.isEmpty()) {
- if
(this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_PARSER_KEY)
- ||
this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_PARSER_KEY_ALIAS)
- ||
this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_ANALYZER_NAME_KEY)
- ||
this.properties.containsKey(InvertedIndexProperties.INVERTED_INDEX_NORMALIZER_NAME_KEY))
{
+ String normalizerName = this.properties.get(
+
InvertedIndexProperties.INVERTED_INDEX_NORMALIZER_NAME_KEY);
+ if
(!InvertedIndexProperties.INVERTED_INDEX_PARSER_NONE.equalsIgnoreCase(getInvertedIndexParser())
+ || StringUtils.isNotEmpty(this.properties.get(
+
InvertedIndexProperties.INVERTED_INDEX_ANALYZER_NAME_KEY))
+ || (StringUtils.isNotEmpty(normalizerName)
+ &&
!"lowercase".equalsIgnoreCase(normalizerName))) {
String supportPhraseKey = InvertedIndexProperties
.INVERTED_INDEX_SUPPORT_PHRASE_KEY;
if (!this.properties.containsKey(supportPhraseKey)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
index dd545e45020..0778f68e418 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
@@ -354,11 +354,20 @@ public class InvertedIndexUtil {
}
/**
- * Canonicalize analyzer and normalizer names in index properties. Legacy
metadata may contain
- * case-distinct policy names, so a resolved custom policy must keep its
exact stored name.
+ * Resolve policy names and retain positions for custom normalizers that
may split terms.
+ * Preserve exact custom policy spellings when legacy names differ only by
case.
*/
private static void normalizeInvertedIndexProperties(Map<String, String>
properties) {
resolvePolicyNames(properties);
+ String normalizer = properties.get(INVERTED_INDEX_NORMALIZER_NAME_KEY);
+ if (normalizer != null && !normalizer.isEmpty()
+ && !properties.containsKey(INVERTED_INDEX_SUPPORT_PHRASE_KEY))
{
+ boolean builtin = Env.getCurrentEnv().getIndexPolicyMgr()
+ .getTopLevelBuiltin(normalizer,
IndexPolicy.BUILTIN_NORMALIZERS) != null;
+ if (!builtin) {
+ properties.put(INVERTED_INDEX_SUPPORT_PHRASE_KEY, "true");
+ }
+ }
AnalyzerKeyNormalizer.normalizeInvertedIndexProperties(
properties,
INVERTED_INDEX_PARSER_KEY,
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
index 3cf1d5d9c57..54481181aa8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
@@ -849,10 +849,11 @@ public class InvertedIndexPropertiesTest {
policyMgr.replayCreateIndexPolicy(new IndexPolicy(
92, "norm_lower", IndexPolicyTypeEnum.NORMALIZER,
Map.of("token_filter", "lowercase")));
Map<String, String> mixedNormalizer = new
HashMap<>(Map.of("normalizer", "LowerCase"));
+ Map<String, String> customNormalizer = new
HashMap<>(Map.of("normalizer", "lowercase"));
Map<String, String> mixedAnalyzer = new HashMap<>(Map.of("analyzer",
"Standard"));
withIndexPolicyManager(policyMgr, () -> {
- for (Map<String, String> properties : List.of(mixedNormalizer,
mixedAnalyzer)) {
+ for (Map<String, String> properties : List.of(mixedNormalizer,
customNormalizer, mixedAnalyzer)) {
Assertions.assertDoesNotThrow(() ->
InvertedIndexUtil.checkInvertedIndexParser("c",
PrimitiveType.VARCHAR, properties,
TInvertedIndexFileStorageFormat.V3));
}
@@ -860,6 +861,8 @@ public class InvertedIndexPropertiesTest {
// The exact policy shadows the canonical name, so only
the mixed spelling still
// reaches the built-in normalizer on BE.
() -> Assertions.assertEquals("LowerCase",
mixedNormalizer.get("normalizer")),
+ () ->
Assertions.assertFalse(mixedNormalizer.containsKey("support_phrase")),
+ () -> Assertions.assertEquals("true",
customNormalizer.get("support_phrase")),
() -> Assertions.assertEquals("LowerCase",
InvertedIndexUtil.resolveAnalyzerName("LowerCase")),
() -> Assertions.assertEquals("lowercase",
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
index 814a00f66e3..d796ec619b8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/IndexTest.java
@@ -24,9 +24,44 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
public class IndexTest {
+ @Test
+ public void testInvertedIndexPhraseDefaultsFollowTokenization() {
+ Index exact = new Index(1, "exact", List.of("col"),
IndexType.INVERTED, Map.of(), "");
+
Assertions.assertFalse(exact.getProperties().containsKey("support_phrase"));
+
+ Index keyword = new Index(2, "keyword", List.of("col"),
IndexType.INVERTED,
+ Map.of("parser", "none"), "");
+
Assertions.assertFalse(keyword.getProperties().containsKey("support_phrase"));
+
+ Index normalized = new Index(3, "normalized", List.of("col"),
IndexType.INVERTED,
+ Map.of("normalizer", "lowercase"), "");
+
Assertions.assertFalse(normalized.getProperties().containsKey("support_phrase"));
+
+ Index customNormalizer = new Index(7, "custom_normalizer",
List.of("col"), IndexType.INVERTED,
+ Map.of("normalizer", "split_norm"), "");
+ Assertions.assertEquals("true",
customNormalizer.getProperties().get("support_phrase"));
+
+ Index disabledCustomNormalizer = new Index(8,
"disabled_custom_normalizer", List.of("col"),
+ IndexType.INVERTED, Map.of("normalizer", "split_norm",
"support_phrase", "false"), "");
+ Assertions.assertEquals("false",
disabledCustomNormalizer.getProperties().get("support_phrase"));
+
+ Index parsed = new Index(4, "parsed", List.of("col"),
IndexType.INVERTED,
+ Map.of("parser", "english"), "");
+ Assertions.assertEquals("true",
parsed.getProperties().get("support_phrase"));
+
+ Index analyzed = new Index(5, "analyzed", List.of("col"),
IndexType.INVERTED,
+ Map.of("analyzer", "english"), "");
+ Assertions.assertEquals("true",
analyzed.getProperties().get("support_phrase"));
+
+ Index disabled = new Index(6, "disabled", List.of("col"),
IndexType.INVERTED,
+ Map.of("parser", "english", "support_phrase", "false"), "");
+ Assertions.assertEquals("false",
disabled.getProperties().get("support_phrase"));
+ }
+
@Test
public void testGetColumnUniqueIds() {
// Create test columns with unique IDs
diff --git a/regression-test/suites/index_p0/test_index_meta.groovy
b/regression-test/suites/index_p0/test_index_meta.groovy
index d7647f2a9a3..bcaaf2922b1 100644
--- a/regression-test/suites/index_p0/test_index_meta.groovy
+++ b/regression-test/suites/index_p0/test_index_meta.groovy
@@ -71,7 +71,7 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "name")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for name")
- assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\", \"support_phrase\" = \"true\")")
+ assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\")")
// add index on column description
sql "create index idx_desc on ${tableName}(description) USING INVERTED
PROPERTIES(\"parser\"=\"standard\") COMMENT 'index for description';"
@@ -90,7 +90,7 @@ suite("index_meta", "p0") {
assertEquals(show_result[1][4], "name")
assertEquals(show_result[1][10], "INVERTED")
assertEquals(show_result[1][11], "index for name")
- assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\", \"support_phrase\" = \"true\")")
+ assertEquals(show_result[1][12], "(\"lower_case\" = \"true\", \"parser\" =
\"none\")")
assertEquals(show_result[2][2], "idx_desc")
assertEquals(show_result[2][4], "description")
assertEquals(show_result[2][10], "INVERTED")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
