This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 031bdd07052 [fix](storage) Backport ANN predicate pruning and index
build cleanup to 4.2 (#70792)
031bdd07052 is described below
commit 031bdd0705250e9d73708991e7e66feee197ed67
Author: Mryange <[email protected]>
AuthorDate: Sat Nov 21 12:17:38 2026 +0800
[fix](storage) Backport ANN predicate pruning and index build cleanup to
4.2 (#70792)
### What problem does this PR solve?
Issue Number: None
Related PR: #70631, #70591
Problem Summary:
Backport two ANN-related storage fixes to branch-4.2.
- Prune a non-negated equality predicate when both segment zone-map
bounds equal its value and the segment has no NULLs. A constant
delete-sign filter can otherwise force MOW ANN TopN queries to fall back
to brute force. Carry the predicate unit tests. The snapshot wrapper and
table-stream regression from #70631 are absent from branch-4.2.
- Release ANN build resources and seal rebuilt index files after each
segment in regular and SNII historical index builds. This limits
accumulation across segments, including those that only carry inherited
indexes. The backport preserves the 4.2 reader interfaces, close-error
propagation, and the rowset publication boundary.
### Release note
Avoid unnecessary brute-force fallback for MOW ANN queries with
segment-wide constant equality predicates and reduce peak memory during
historical ANN index builds.
### Testing
- ASAN BE clean build and packaging passed (`sh build.sh --be --clean
-j48`).
- clang-format 16 and `git diff --check` passed.
- Focused `BlockColumnPredicateTest.*` is running. Cluster regression
tests were not run.
### Check List (For Author)
- Behavior changed: Yes. See Problem Summary.
- Documentation needed: No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
### AI Disclosure
- Tool: Codex
- Generated: Branch-specific conflict resolution and PR description.
- Human review: Pending.
---
be/src/storage/index/ann/ann_index_writer.cpp | 20 ++-
be/src/storage/index/ann/ann_index_writer.h | 5 +-
be/src/storage/predicate/comparison_predicate.h | 6 +-
be/src/storage/task/index_builder.cpp | 195 +++++++++++----------
.../predicate/block_column_predicate_test.cpp | 66 +++++++
5 files changed, 189 insertions(+), 103 deletions(-)
diff --git a/be/src/storage/index/ann/ann_index_writer.cpp
b/be/src/storage/index/ann/ann_index_writer.cpp
index ba93f6a733e..a5ba5d8bf82 100644
--- a/be/src/storage/index/ann/ann_index_writer.cpp
+++ b/be/src/storage/index/ann/ann_index_writer.cpp
@@ -26,6 +26,7 @@
#include "common/config.h"
#include "storage/index/ann/faiss_ann_index.h"
#include "storage/index/index_fs_directory.h"
+#include "util/defer_op.h"
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
@@ -92,6 +93,11 @@ Status AnnIndexColumnWriter::add_values(const std::string
fn, const void* values
}
void AnnIndexColumnWriter::close_on_error() {
+ _release_build_resources();
+}
+
+void AnnIndexColumnWriter::_release_build_resources() {
+ _vector_index.reset();
PODArray<float> empty_buffered_vectors;
_buffered_vectors.swap(empty_buffered_vectors);
}
@@ -139,6 +145,7 @@ int64_t AnnIndexColumnWriter::size() const {
Status AnnIndexColumnWriter::finish() {
if (_total_rows == 0) {
LOG_INFO("No data to train/add for ANN index. Skipping index
building.");
+ _release_build_resources();
return _index_file_writer->delete_index(_index_meta);
}
@@ -150,8 +157,7 @@ Status AnnIndexColumnWriter::finish() {
"Total data size {} is less than minimum {} rows required for
ANN index build. "
"Skipping index building for this segment.",
_total_rows, effective_min_rows);
- PODArray<float> empty_buffered_vectors;
- _buffered_vectors.swap(empty_buffered_vectors);
+ _release_build_resources();
return _index_file_writer->delete_index(_index_meta);
}
@@ -159,6 +165,10 @@ Status AnnIndexColumnWriter::finish() {
}
Status AnnIndexColumnWriter::_build_and_save(Int64 min_train_rows, Int64
effective_min_rows) {
+ // Release the Faiss index and any remaining input buffer after saving,
including
+ // on failure, rather than retaining them until this writer is destroyed.
+ Defer release_resources([this] { _release_build_resources(); });
+
const size_t dim = _vector_index->get_dimension();
DCHECK(_buffered_vectors.size() % dim == 0);
const Int64 train_rows = cast_set<Int64>(_buffered_vectors.size() / dim);
@@ -170,8 +180,10 @@ Status AnnIndexColumnWriter::_build_and_save(Int64
min_train_rows, Int64 effecti
RETURN_IF_ERROR(_vector_index->add(train_rows, _buffered_vectors.data()));
// PODArray::clear() keeps the allocated capacity. Swap with an empty
array so the
// full-segment build buffer is released before saving the index.
- PODArray<float> empty_buffered_vectors;
- _buffered_vectors.swap(empty_buffered_vectors);
+ {
+ PODArray<float> empty_buffered_vectors;
+ _buffered_vectors.swap(empty_buffered_vectors);
+ }
Status status = _vector_index->save(_dir.get());
if (!status.ok()) {
// A failed save keeps whatever it managed to write, and under SNII
that is
diff --git a/be/src/storage/index/ann/ann_index_writer.h
b/be/src/storage/index/ann/ann_index_writer.h
index f95aba7d73b..eb9dbc053a3 100644
--- a/be/src/storage/index/ann/ann_index_writer.h
+++ b/be/src/storage/index/ann/ann_index_writer.h
@@ -73,14 +73,13 @@ public:
private:
Status _build_and_save(Int64 min_train_rows, Int64 effective_min_rows);
+ void _release_build_resources();
#ifdef BE_TEST
friend class TestAnnIndexColumnWriter;
#endif
- // VectorIndex shoule be managed by some cache.
- // VectorIndex should be weak shared by AnnIndexWriter and
VectorIndexReader
- // This should be a weak_ptr
+ // The build index is owned by this writer and released when the segment
is finished.
std::shared_ptr<VectorIndex> _vector_index;
PODArray<float> _buffered_vectors;
int64_t _total_rows = 0;
diff --git a/be/src/storage/predicate/comparison_predicate.h
b/be/src/storage/predicate/comparison_predicate.h
index 5e06953a536..0f8da5feb0a 100644
--- a/be/src/storage/predicate/comparison_predicate.h
+++ b/be/src/storage/predicate/comparison_predicate.h
@@ -246,7 +246,11 @@ public:
return false;
}
- if constexpr (PT == PredicateType::LT) {
+ if constexpr (PT == PredicateType::EQ) {
+ // Equal bounds prove the equality is always true, unless the
predicate is negated.
+ return !_opposite && Compare::equal(zone_map.min_value.template
get<Type>(), _value) &&
+ Compare::equal(zone_map.max_value.template get<Type>(),
_value);
+ } else if constexpr (PT == PredicateType::LT) {
return _value > zone_map.max_value.template get<Type>();
} else if constexpr (PT == PredicateType::LE) {
return _value >= zone_map.max_value.template get<Type>();
diff --git a/be/src/storage/task/index_builder.cpp
b/be/src/storage/task/index_builder.cpp
index 22f933b623c..b1ecc6b040d 100644
--- a/be/src/storage/task/index_builder.cpp
+++ b/be/src/storage/task/index_builder.cpp
@@ -700,101 +700,105 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
_index_file_writers.emplace(seg_ptr->id(),
std::move(index_file_writer));
DORIS_CHECK(inserted);
DORIS_CHECK(index_file_writer_it->second != nullptr);
- if (return_columns.empty()) {
- // no columns to read
- continue;
- }
- // create iterator for each segment
- StorageReadOptions read_options;
- OlapReaderStatistics stats;
- read_options.stats = &stats;
- read_options.tablet_schema = output_rowset_schema;
- std::shared_ptr<Schema> schema =
- std::make_shared<Schema>(output_rowset_schema->columns(),
return_columns);
- std::unique_ptr<RowwiseIterator> iter;
- auto res = seg_ptr->new_iterator(schema, read_options, &iter);
-
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_create_iterator_error", {
- res = Status::Error<ErrorCode::INTERNAL_ERROR>(
- "debug point:
handle_single_rowset_create_iterator_error");
- })
- if (!res.ok()) {
- LOG(WARNING) << "failed to create iterator[" << seg_ptr->id()
- << "]: " << res.to_string();
- return
Status::Error<ErrorCode::ROWSET_READER_INIT>(res.to_string());
- }
+ // Even without columns to rebuild, the inherited indexes must be
sealed below.
+ if (!return_columns.empty()) {
+ // create iterator for each segment
+ StorageReadOptions read_options;
+ OlapReaderStatistics stats;
+ read_options.stats = &stats;
+ read_options.tablet_schema = output_rowset_schema;
+ auto schema =
+
std::make_shared<Schema>(output_rowset_schema->columns(), return_columns);
+ std::unique_ptr<RowwiseIterator> iter;
+ auto res = seg_ptr->new_iterator(schema, read_options, &iter);
+
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_create_iterator_error", {
+ res = Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "debug point:
handle_single_rowset_create_iterator_error");
+ })
+ if (!res.ok()) {
+ LOG(WARNING) << "failed to create iterator[" <<
seg_ptr->id()
+ << "]: " << res.to_string();
+ return
Status::Error<ErrorCode::ROWSET_READER_INIT>(res.to_string());
+ }
- auto block =
Block::create_unique(output_rowset_schema->create_block(return_columns));
- while (true) {
- auto status = iter->next_batch(block.get());
-
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_iterator_next_batch_error",
{
- status =
Status::Error<ErrorCode::SCHEMA_CHANGE_INFO_INVALID>(
- "next_batch fault injection");
- });
- if (!status.ok()) {
- if (status.is<ErrorCode::END_OF_FILE>()) {
- break;
+ auto block =
+
Block::create_unique(output_rowset_schema->create_block(return_columns));
+ while (true) {
+ auto status = iter->next_batch(block.get());
+ DBUG_EXECUTE_IF(
+
"IndexBuilder::handle_single_rowset_iterator_next_batch_error", {
+ status =
Status::Error<ErrorCode::SCHEMA_CHANGE_INFO_INVALID>(
+ "next_batch fault injection");
+ });
+ if (!status.ok()) {
+ if (status.is<ErrorCode::END_OF_FILE>()) {
+ break;
+ }
+ LOG(WARNING) << "failed to read next block when schema
change for inverted "
+ "index."
+ << ", err=" << status.to_string();
+ return status;
}
- LOG(WARNING)
- << "failed to read next block when schema change
for inverted index."
- << ", err=" << status.to_string();
- return status;
- }
- // write inverted index data, or ann index data
- status = _write_inverted_index_data(output_rowset_schema,
iter->data_id(),
- block.get());
- DBUG_EXECUTE_IF(
-
"IndexBuilder::handle_single_rowset_write_inverted_index_data_error", {
- status = Status::Error<ErrorCode::INTERNAL_ERROR>(
- "debug point: "
-
"handle_single_rowset_write_inverted_index_data_error");
- })
- if (!status.ok()) {
- return status;
+ // write inverted index data, or ann index data
+ status = _write_inverted_index_data(output_rowset_schema,
iter->data_id(),
+ block.get());
+ DBUG_EXECUTE_IF(
+
"IndexBuilder::handle_single_rowset_write_inverted_index_data_error", {
+ status =
Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "debug point: "
+
"handle_single_rowset_write_inverted_index_data_error");
+ })
+ if (!status.ok()) {
+ return status;
+ }
+ block->clear_column_data();
}
- block->clear_column_data();
- }
- // finish write inverted index, flush data to compound file
- for (auto& writer_sign : inverted_index_writer_signs) {
- try {
- auto index_column_writer_it =
_index_column_writers.find(writer_sign);
- DORIS_CHECK(index_column_writer_it !=
_index_column_writers.end());
- DORIS_CHECK(index_column_writer_it->second != nullptr);
- RETURN_IF_ERROR(index_column_writer_it->second->finish());
-
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_index_build_finish_error", {
- _CLTHROWA(CL_ERR_IO,
- "debug point:
handle_single_rowset_index_build_finish_error");
- })
- } catch (const std::exception& e) {
- return
Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "CLuceneError occured: {}", e.what());
+ // finish write inverted index, flush data to compound file
+ for (auto& writer_sign : inverted_index_writer_signs) {
+ try {
+ auto index_column_writer_it =
_index_column_writers.find(writer_sign);
+ DORIS_CHECK(index_column_writer_it !=
_index_column_writers.end());
+ DORIS_CHECK(index_column_writer_it->second != nullptr);
+
RETURN_IF_ERROR(index_column_writer_it->second->finish());
+ DBUG_EXECUTE_IF(
+
"IndexBuilder::handle_single_rowset_index_build_finish_error", {
+ _CLTHROWA(CL_ERR_IO,
+ "debug point: "
+
"handle_single_rowset_index_build_finish_error");
+ })
+ } catch (const std::exception& e) {
+ return
Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "CLuceneError occurred: {}", e.what());
+ }
}
+ _olap_data_convertor->reset();
}
-
- _olap_data_convertor->reset();
- }
- for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
- auto st = index_file_writer->begin_close();
-
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_file_writer_close_error", {
- st = Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "debug point:
handle_single_rowset_file_writer_close_error");
- })
- if (!st.ok()) {
- LOG(ERROR) << "close index_file_writer error:" << st;
- return st;
+ // Each segment owns an independent index file. Seal it before
building the
+ // next segment: otherwise every completed ANN file remains in its
RAM directory.
+ for (auto&& [seg_id, completed_file_writer] : _index_file_writers)
{
+ auto st = completed_file_writer->begin_close();
+
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_file_writer_close_error", {
+ st =
Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "debug point:
handle_single_rowset_file_writer_close_error");
+ })
+ if (!st.ok()) {
+ LOG(ERROR) << "close completed_file_writer error:" << st;
+ return st;
+ }
+ inverted_index_size +=
completed_file_writer->get_index_file_total_size();
}
- inverted_index_size +=
index_file_writer->get_index_file_total_size();
- }
- for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
- auto st = index_file_writer->finish_close();
- if (!st.ok()) {
- LOG(ERROR) << "wait close index_file_writer error:" << st;
- return st;
+ for (auto&& [seg_id, completed_file_writer] : _index_file_writers)
{
+ auto st = completed_file_writer->finish_close();
+ if (!st.ok()) {
+ LOG(ERROR) << "wait close completed_file_writer error:" <<
st;
+ return st;
+ }
}
+ _index_column_writers.clear();
+ _index_file_writers.clear();
}
- _index_column_writers.clear();
- _index_file_writers.clear();
output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size());
output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() +
inverted_index_size);
@@ -813,21 +817,22 @@ Status IndexBuilder::_handle_single_rowset_snii(
auto input_schema_it = _input_rowset_schemas.find(rowset_id);
DORIS_CHECK(input_schema_it != _input_rowset_schemas.end());
+ size_t inverted_index_size = 0;
for (auto& seg_ptr : segments) {
RETURN_IF_ERROR(_rewrite_single_segment_snii(output_rowset_meta->fs(),
output_rowset_meta->tablet_schema(),
*input_schema_it->second,
rowset_id, seg_ptr));
+ // Blob staging and column writers belong to this segment, not the
whole rowset.
+ for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
+ RETURN_IF_ERROR(index_file_writer->begin_close());
+ inverted_index_size +=
index_file_writer->get_index_file_total_size();
+ }
+ for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
+ RETURN_IF_ERROR(index_file_writer->finish_close());
+ }
+ _index_column_writers.clear();
+ _index_file_writers.clear();
}
- size_t inverted_index_size = 0;
- for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
- RETURN_IF_ERROR(index_file_writer->begin_close());
- inverted_index_size += index_file_writer->get_index_file_total_size();
- }
- for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
- RETURN_IF_ERROR(index_file_writer->finish_close());
- }
- _index_column_writers.clear();
- _index_file_writers.clear();
output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size());
output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() +
inverted_index_size);
diff --git a/be/test/storage/predicate/block_column_predicate_test.cpp
b/be/test/storage/predicate/block_column_predicate_test.cpp
index 84cde001a07..1a71d50a569 100644
--- a/be/test/storage/predicate/block_column_predicate_test.cpp
+++ b/be/test/storage/predicate/block_column_predicate_test.cpp
@@ -320,6 +320,72 @@ void single_column_predicate_test_func(const
segment_v2::ZoneMap& zone_map_info,
EXPECT_EQ(matched, expect_match);
}
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueDeleteSign) {
+ const auto zero = Field::create_field<TYPE_TINYINT>(0);
+ const auto one = Field::create_field<TYPE_TINYINT>(1);
+ ComparisonPredicateBase<TYPE_TINYINT, PredicateType::EQ> predicate(0,
"delete_sign", zero);
+ segment_v2::ZoneMap zone_map {
+ .min_value = zero, .max_value = zero, .has_null = false,
.has_not_null = true};
+ EXPECT_TRUE(predicate.is_always_true(zone_map));
+
+ // A segment containing a tombstone must retain the delete-sign filter.
+ zone_map.max_value = one;
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+ zone_map.min_value = one;
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+ zone_map.min_value = Field::create_field<TYPE_TINYINT>(-1);
+ zone_map.max_value = zero;
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+
+ zone_map.min_value = zero;
+ zone_map.max_value = zero;
+ zone_map.has_null = true;
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+ zone_map.has_not_null = false;
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+}
+
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueOpposite) {
+ const auto zero = Field::create_field<TYPE_TINYINT>(0);
+ const auto one = Field::create_field<TYPE_TINYINT>(1);
+ ComparisonPredicateBase<TYPE_TINYINT, PredicateType::EQ> predicate(0,
"delete_sign", zero,
+ true);
+ segment_v2::ZoneMap zone_map {
+ .min_value = zero, .max_value = zero, .has_null = false,
.has_not_null = true};
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+
+ // Keep negated predicates conservative even when the non-negated equality
cannot match.
+ zone_map.min_value = one;
+ zone_map.max_value = one;
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+}
+
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueString) {
+ const auto value = Field::create_field<TYPE_STRING>("constant");
+ ComparisonPredicateBase<TYPE_STRING, PredicateType::EQ> predicate(0,
"value", value);
+ segment_v2::ZoneMap zone_map {
+ .min_value = value, .max_value = value, .has_null = false,
.has_not_null = true};
+ EXPECT_TRUE(predicate.is_always_true(zone_map));
+ zone_map.max_value = Field::create_field<TYPE_STRING>("other");
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+}
+
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueDouble) {
+ const auto value = Field::create_field<TYPE_DOUBLE>(1.0);
+ ComparisonPredicateBase<TYPE_DOUBLE, PredicateType::EQ> predicate(0,
"value", value);
+ segment_v2::ZoneMap zone_map {
+ .min_value = value, .max_value = value, .has_null = false,
.has_not_null = true};
+ EXPECT_TRUE(predicate.is_always_true(zone_map));
+
+ zone_map.max_value =
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN());
+ EXPECT_FALSE(predicate.is_always_true(zone_map));
+ ComparisonPredicateBase<TYPE_DOUBLE, PredicateType::EQ> nan_predicate(0,
"value",
+
zone_map.max_value);
+ zone_map.min_value = zone_map.max_value;
+ // Doris treats NaN as equal to NaN, including during row evaluation.
+ EXPECT_TRUE(nan_predicate.is_always_true(zone_map));
+}
+
// test zonemap index
TEST_F(BlockColumnPredicateTest, test_double_single_column_predicate) {
auto nan =
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]