This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.2 by this push:
     new 031bdd07052 [fix](storage) Backport ANN predicate pruning and index 
build cleanup to 4.2 (#70792)
031bdd07052 is described below

commit 031bdd0705250e9d73708991e7e66feee197ed67
Author: Mryange <[email protected]>
AuthorDate: Sat Nov 21 12:17:38 2026 +0800

    [fix](storage) Backport ANN predicate pruning and index build cleanup to 
4.2 (#70792)
    
    ### What problem does this PR solve?
    
    Issue Number: None
    
    Related PR: #70631, #70591
    
    Problem Summary:
    
    Backport two ANN-related storage fixes to branch-4.2.
    
    - Prune a non-negated equality predicate when both segment zone-map
    bounds equal its value and the segment has no NULLs. A constant
    delete-sign filter can otherwise force MOW ANN TopN queries to fall back
    to brute force. Carry the predicate unit tests. The snapshot wrapper and
    table-stream regression from #70631 are absent from branch-4.2.
    - Release ANN build resources and seal rebuilt index files after each
    segment in regular and SNII historical index builds. This limits
    accumulation across segments, including those that only carry inherited
    indexes. The backport preserves the 4.2 reader interfaces, close-error
    propagation, and the rowset publication boundary.
    
    ### Release note
    
    Avoid unnecessary brute-force fallback for MOW ANN queries with
    segment-wide constant equality predicates and reduce peak memory during
    historical ANN index builds.
    
    ### Testing
    
    - ASAN BE clean build and packaging passed (`sh build.sh --be --clean
    -j48`).
    - clang-format 16 and `git diff --check` passed.
    - Focused `BlockColumnPredicateTest.*` is running. Cluster regression
    tests were not run.
    
    ### Check List (For Author)
    
    - Behavior changed: Yes. See Problem Summary.
    - Documentation needed: No.
    
    ### Check List (For Reviewer who merge this PR)
    
    - [ ] Confirm the release note
    - [ ] Confirm test cases
    - [ ] Confirm document
    
    ### AI Disclosure
    
    - Tool: Codex
    - Generated: Branch-specific conflict resolution and PR description.
    - Human review: Pending.
---
 be/src/storage/index/ann/ann_index_writer.cpp      |  20 ++-
 be/src/storage/index/ann/ann_index_writer.h        |   5 +-
 be/src/storage/predicate/comparison_predicate.h    |   6 +-
 be/src/storage/task/index_builder.cpp              | 195 +++++++++++----------
 .../predicate/block_column_predicate_test.cpp      |  66 +++++++
 5 files changed, 189 insertions(+), 103 deletions(-)

diff --git a/be/src/storage/index/ann/ann_index_writer.cpp 
b/be/src/storage/index/ann/ann_index_writer.cpp
index ba93f6a733e..a5ba5d8bf82 100644
--- a/be/src/storage/index/ann/ann_index_writer.cpp
+++ b/be/src/storage/index/ann/ann_index_writer.cpp
@@ -26,6 +26,7 @@
 #include "common/config.h"
 #include "storage/index/ann/faiss_ann_index.h"
 #include "storage/index/index_fs_directory.h"
+#include "util/defer_op.h"
 
 namespace doris::segment_v2 {
 #include "common/compile_check_begin.h"
@@ -92,6 +93,11 @@ Status AnnIndexColumnWriter::add_values(const std::string 
fn, const void* values
 }
 
 void AnnIndexColumnWriter::close_on_error() {
+    _release_build_resources();
+}
+
+void AnnIndexColumnWriter::_release_build_resources() {
+    _vector_index.reset();
     PODArray<float> empty_buffered_vectors;
     _buffered_vectors.swap(empty_buffered_vectors);
 }
@@ -139,6 +145,7 @@ int64_t AnnIndexColumnWriter::size() const {
 Status AnnIndexColumnWriter::finish() {
     if (_total_rows == 0) {
         LOG_INFO("No data to train/add for ANN index. Skipping index 
building.");
+        _release_build_resources();
         return _index_file_writer->delete_index(_index_meta);
     }
 
@@ -150,8 +157,7 @@ Status AnnIndexColumnWriter::finish() {
                 "Total data size {} is less than minimum {} rows required for 
ANN index build. "
                 "Skipping index building for this segment.",
                 _total_rows, effective_min_rows);
-        PODArray<float> empty_buffered_vectors;
-        _buffered_vectors.swap(empty_buffered_vectors);
+        _release_build_resources();
         return _index_file_writer->delete_index(_index_meta);
     }
 
@@ -159,6 +165,10 @@ Status AnnIndexColumnWriter::finish() {
 }
 
 Status AnnIndexColumnWriter::_build_and_save(Int64 min_train_rows, Int64 
effective_min_rows) {
+    // Release the Faiss index and any remaining input buffer after saving, 
including
+    // on failure, rather than retaining them until this writer is destroyed.
+    Defer release_resources([this] { _release_build_resources(); });
+
     const size_t dim = _vector_index->get_dimension();
     DCHECK(_buffered_vectors.size() % dim == 0);
     const Int64 train_rows = cast_set<Int64>(_buffered_vectors.size() / dim);
@@ -170,8 +180,10 @@ Status AnnIndexColumnWriter::_build_and_save(Int64 
min_train_rows, Int64 effecti
     RETURN_IF_ERROR(_vector_index->add(train_rows, _buffered_vectors.data()));
     // PODArray::clear() keeps the allocated capacity. Swap with an empty 
array so the
     // full-segment build buffer is released before saving the index.
-    PODArray<float> empty_buffered_vectors;
-    _buffered_vectors.swap(empty_buffered_vectors);
+    {
+        PODArray<float> empty_buffered_vectors;
+        _buffered_vectors.swap(empty_buffered_vectors);
+    }
     Status status = _vector_index->save(_dir.get());
     if (!status.ok()) {
         // A failed save keeps whatever it managed to write, and under SNII 
that is
diff --git a/be/src/storage/index/ann/ann_index_writer.h 
b/be/src/storage/index/ann/ann_index_writer.h
index f95aba7d73b..eb9dbc053a3 100644
--- a/be/src/storage/index/ann/ann_index_writer.h
+++ b/be/src/storage/index/ann/ann_index_writer.h
@@ -73,14 +73,13 @@ public:
 
 private:
     Status _build_and_save(Int64 min_train_rows, Int64 effective_min_rows);
+    void _release_build_resources();
 
 #ifdef BE_TEST
     friend class TestAnnIndexColumnWriter;
 #endif
 
-    // VectorIndex shoule be managed by some cache.
-    // VectorIndex should be weak shared by AnnIndexWriter and 
VectorIndexReader
-    // This should be a weak_ptr
+    // The build index is owned by this writer and released when the segment 
is finished.
     std::shared_ptr<VectorIndex> _vector_index;
     PODArray<float> _buffered_vectors;
     int64_t _total_rows = 0;
diff --git a/be/src/storage/predicate/comparison_predicate.h 
b/be/src/storage/predicate/comparison_predicate.h
index 5e06953a536..0f8da5feb0a 100644
--- a/be/src/storage/predicate/comparison_predicate.h
+++ b/be/src/storage/predicate/comparison_predicate.h
@@ -246,7 +246,11 @@ public:
             return false;
         }
 
-        if constexpr (PT == PredicateType::LT) {
+        if constexpr (PT == PredicateType::EQ) {
+            // Equal bounds prove the equality is always true, unless the 
predicate is negated.
+            return !_opposite && Compare::equal(zone_map.min_value.template 
get<Type>(), _value) &&
+                   Compare::equal(zone_map.max_value.template get<Type>(), 
_value);
+        } else if constexpr (PT == PredicateType::LT) {
             return _value > zone_map.max_value.template get<Type>();
         } else if constexpr (PT == PredicateType::LE) {
             return _value >= zone_map.max_value.template get<Type>();
diff --git a/be/src/storage/task/index_builder.cpp 
b/be/src/storage/task/index_builder.cpp
index 22f933b623c..b1ecc6b040d 100644
--- a/be/src/storage/task/index_builder.cpp
+++ b/be/src/storage/task/index_builder.cpp
@@ -700,101 +700,105 @@ Status 
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
                     _index_file_writers.emplace(seg_ptr->id(), 
std::move(index_file_writer));
             DORIS_CHECK(inserted);
             DORIS_CHECK(index_file_writer_it->second != nullptr);
-            if (return_columns.empty()) {
-                // no columns to read
-                continue;
-            }
-            // create iterator for each segment
-            StorageReadOptions read_options;
-            OlapReaderStatistics stats;
-            read_options.stats = &stats;
-            read_options.tablet_schema = output_rowset_schema;
-            std::shared_ptr<Schema> schema =
-                    std::make_shared<Schema>(output_rowset_schema->columns(), 
return_columns);
-            std::unique_ptr<RowwiseIterator> iter;
-            auto res = seg_ptr->new_iterator(schema, read_options, &iter);
-            
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_create_iterator_error", {
-                res = Status::Error<ErrorCode::INTERNAL_ERROR>(
-                        "debug point: 
handle_single_rowset_create_iterator_error");
-            })
-            if (!res.ok()) {
-                LOG(WARNING) << "failed to create iterator[" << seg_ptr->id()
-                             << "]: " << res.to_string();
-                return 
Status::Error<ErrorCode::ROWSET_READER_INIT>(res.to_string());
-            }
+            // Even without columns to rebuild, the inherited indexes must be 
sealed below.
+            if (!return_columns.empty()) {
+                // create iterator for each segment
+                StorageReadOptions read_options;
+                OlapReaderStatistics stats;
+                read_options.stats = &stats;
+                read_options.tablet_schema = output_rowset_schema;
+                auto schema =
+                        
std::make_shared<Schema>(output_rowset_schema->columns(), return_columns);
+                std::unique_ptr<RowwiseIterator> iter;
+                auto res = seg_ptr->new_iterator(schema, read_options, &iter);
+                
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_create_iterator_error", {
+                    res = Status::Error<ErrorCode::INTERNAL_ERROR>(
+                            "debug point: 
handle_single_rowset_create_iterator_error");
+                })
+                if (!res.ok()) {
+                    LOG(WARNING) << "failed to create iterator[" << 
seg_ptr->id()
+                                 << "]: " << res.to_string();
+                    return 
Status::Error<ErrorCode::ROWSET_READER_INIT>(res.to_string());
+                }
 
-            auto block = 
Block::create_unique(output_rowset_schema->create_block(return_columns));
-            while (true) {
-                auto status = iter->next_batch(block.get());
-                
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_iterator_next_batch_error", 
{
-                    status = 
Status::Error<ErrorCode::SCHEMA_CHANGE_INFO_INVALID>(
-                            "next_batch fault injection");
-                });
-                if (!status.ok()) {
-                    if (status.is<ErrorCode::END_OF_FILE>()) {
-                        break;
+                auto block =
+                        
Block::create_unique(output_rowset_schema->create_block(return_columns));
+                while (true) {
+                    auto status = iter->next_batch(block.get());
+                    DBUG_EXECUTE_IF(
+                            
"IndexBuilder::handle_single_rowset_iterator_next_batch_error", {
+                                status = 
Status::Error<ErrorCode::SCHEMA_CHANGE_INFO_INVALID>(
+                                        "next_batch fault injection");
+                            });
+                    if (!status.ok()) {
+                        if (status.is<ErrorCode::END_OF_FILE>()) {
+                            break;
+                        }
+                        LOG(WARNING) << "failed to read next block when schema 
change for inverted "
+                                        "index."
+                                     << ", err=" << status.to_string();
+                        return status;
                     }
-                    LOG(WARNING)
-                            << "failed to read next block when schema change 
for inverted index."
-                            << ", err=" << status.to_string();
-                    return status;
-                }
 
-                // write inverted index data, or ann index data
-                status = _write_inverted_index_data(output_rowset_schema, 
iter->data_id(),
-                                                    block.get());
-                DBUG_EXECUTE_IF(
-                        
"IndexBuilder::handle_single_rowset_write_inverted_index_data_error", {
-                            status = Status::Error<ErrorCode::INTERNAL_ERROR>(
-                                    "debug point: "
-                                    
"handle_single_rowset_write_inverted_index_data_error");
-                        })
-                if (!status.ok()) {
-                    return status;
+                    // write inverted index data, or ann index data
+                    status = _write_inverted_index_data(output_rowset_schema, 
iter->data_id(),
+                                                        block.get());
+                    DBUG_EXECUTE_IF(
+                            
"IndexBuilder::handle_single_rowset_write_inverted_index_data_error", {
+                                status = 
Status::Error<ErrorCode::INTERNAL_ERROR>(
+                                        "debug point: "
+                                        
"handle_single_rowset_write_inverted_index_data_error");
+                            })
+                    if (!status.ok()) {
+                        return status;
+                    }
+                    block->clear_column_data();
                 }
-                block->clear_column_data();
-            }
 
-            // finish write inverted index, flush data to compound file
-            for (auto& writer_sign : inverted_index_writer_signs) {
-                try {
-                    auto index_column_writer_it = 
_index_column_writers.find(writer_sign);
-                    DORIS_CHECK(index_column_writer_it != 
_index_column_writers.end());
-                    DORIS_CHECK(index_column_writer_it->second != nullptr);
-                    RETURN_IF_ERROR(index_column_writer_it->second->finish());
-                    
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_index_build_finish_error", {
-                        _CLTHROWA(CL_ERR_IO,
-                                  "debug point: 
handle_single_rowset_index_build_finish_error");
-                    })
-                } catch (const std::exception& e) {
-                    return 
Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
-                            "CLuceneError occured: {}", e.what());
+                // finish write inverted index, flush data to compound file
+                for (auto& writer_sign : inverted_index_writer_signs) {
+                    try {
+                        auto index_column_writer_it = 
_index_column_writers.find(writer_sign);
+                        DORIS_CHECK(index_column_writer_it != 
_index_column_writers.end());
+                        DORIS_CHECK(index_column_writer_it->second != nullptr);
+                        
RETURN_IF_ERROR(index_column_writer_it->second->finish());
+                        DBUG_EXECUTE_IF(
+                                
"IndexBuilder::handle_single_rowset_index_build_finish_error", {
+                                    _CLTHROWA(CL_ERR_IO,
+                                              "debug point: "
+                                              
"handle_single_rowset_index_build_finish_error");
+                                })
+                    } catch (const std::exception& e) {
+                        return 
Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                                "CLuceneError occurred: {}", e.what());
+                    }
                 }
+                _olap_data_convertor->reset();
             }
-
-            _olap_data_convertor->reset();
-        }
-        for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
-            auto st = index_file_writer->begin_close();
-            
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_file_writer_close_error", {
-                st = Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
-                        "debug point: 
handle_single_rowset_file_writer_close_error");
-            })
-            if (!st.ok()) {
-                LOG(ERROR) << "close index_file_writer error:" << st;
-                return st;
+            // Each segment owns an independent index file. Seal it before 
building the
+            // next segment: otherwise every completed ANN file remains in its 
RAM directory.
+            for (auto&& [seg_id, completed_file_writer] : _index_file_writers) 
{
+                auto st = completed_file_writer->begin_close();
+                
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_file_writer_close_error", {
+                    st = 
Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                            "debug point: 
handle_single_rowset_file_writer_close_error");
+                })
+                if (!st.ok()) {
+                    LOG(ERROR) << "close completed_file_writer error:" << st;
+                    return st;
+                }
+                inverted_index_size += 
completed_file_writer->get_index_file_total_size();
             }
-            inverted_index_size += 
index_file_writer->get_index_file_total_size();
-        }
-        for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
-            auto st = index_file_writer->finish_close();
-            if (!st.ok()) {
-                LOG(ERROR) << "wait close index_file_writer error:" << st;
-                return st;
+            for (auto&& [seg_id, completed_file_writer] : _index_file_writers) 
{
+                auto st = completed_file_writer->finish_close();
+                if (!st.ok()) {
+                    LOG(ERROR) << "wait close completed_file_writer error:" << 
st;
+                    return st;
+                }
             }
+            _index_column_writers.clear();
+            _index_file_writers.clear();
         }
-        _index_column_writers.clear();
-        _index_file_writers.clear();
         
output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size());
         
output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() +
                                                 inverted_index_size);
@@ -813,21 +817,22 @@ Status IndexBuilder::_handle_single_rowset_snii(
     auto input_schema_it = _input_rowset_schemas.find(rowset_id);
     DORIS_CHECK(input_schema_it != _input_rowset_schemas.end());
 
+    size_t inverted_index_size = 0;
     for (auto& seg_ptr : segments) {
         RETURN_IF_ERROR(_rewrite_single_segment_snii(output_rowset_meta->fs(),
                                                      
output_rowset_meta->tablet_schema(),
                                                      *input_schema_it->second, 
rowset_id, seg_ptr));
+        // Blob staging and column writers belong to this segment, not the 
whole rowset.
+        for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
+            RETURN_IF_ERROR(index_file_writer->begin_close());
+            inverted_index_size += 
index_file_writer->get_index_file_total_size();
+        }
+        for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
+            RETURN_IF_ERROR(index_file_writer->finish_close());
+        }
+        _index_column_writers.clear();
+        _index_file_writers.clear();
     }
-    size_t inverted_index_size = 0;
-    for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
-        RETURN_IF_ERROR(index_file_writer->begin_close());
-        inverted_index_size += index_file_writer->get_index_file_total_size();
-    }
-    for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
-        RETURN_IF_ERROR(index_file_writer->finish_close());
-    }
-    _index_column_writers.clear();
-    _index_file_writers.clear();
     
output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size());
     
output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() +
                                             inverted_index_size);
diff --git a/be/test/storage/predicate/block_column_predicate_test.cpp 
b/be/test/storage/predicate/block_column_predicate_test.cpp
index 84cde001a07..1a71d50a569 100644
--- a/be/test/storage/predicate/block_column_predicate_test.cpp
+++ b/be/test/storage/predicate/block_column_predicate_test.cpp
@@ -320,6 +320,72 @@ void single_column_predicate_test_func(const 
segment_v2::ZoneMap& zone_map_info,
     EXPECT_EQ(matched, expect_match);
 }
 
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueDeleteSign) {
+    const auto zero = Field::create_field<TYPE_TINYINT>(0);
+    const auto one = Field::create_field<TYPE_TINYINT>(1);
+    ComparisonPredicateBase<TYPE_TINYINT, PredicateType::EQ> predicate(0, 
"delete_sign", zero);
+    segment_v2::ZoneMap zone_map {
+            .min_value = zero, .max_value = zero, .has_null = false, 
.has_not_null = true};
+    EXPECT_TRUE(predicate.is_always_true(zone_map));
+
+    // A segment containing a tombstone must retain the delete-sign filter.
+    zone_map.max_value = one;
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+    zone_map.min_value = one;
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+    zone_map.min_value = Field::create_field<TYPE_TINYINT>(-1);
+    zone_map.max_value = zero;
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+
+    zone_map.min_value = zero;
+    zone_map.max_value = zero;
+    zone_map.has_null = true;
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+    zone_map.has_not_null = false;
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+}
+
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueOpposite) {
+    const auto zero = Field::create_field<TYPE_TINYINT>(0);
+    const auto one = Field::create_field<TYPE_TINYINT>(1);
+    ComparisonPredicateBase<TYPE_TINYINT, PredicateType::EQ> predicate(0, 
"delete_sign", zero,
+                                                                       true);
+    segment_v2::ZoneMap zone_map {
+            .min_value = zero, .max_value = zero, .has_null = false, 
.has_not_null = true};
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+
+    // Keep negated predicates conservative even when the non-negated equality 
cannot match.
+    zone_map.min_value = one;
+    zone_map.max_value = one;
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+}
+
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueString) {
+    const auto value = Field::create_field<TYPE_STRING>("constant");
+    ComparisonPredicateBase<TYPE_STRING, PredicateType::EQ> predicate(0, 
"value", value);
+    segment_v2::ZoneMap zone_map {
+            .min_value = value, .max_value = value, .has_null = false, 
.has_not_null = true};
+    EXPECT_TRUE(predicate.is_always_true(zone_map));
+    zone_map.max_value = Field::create_field<TYPE_STRING>("other");
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+}
+
+TEST_F(BlockColumnPredicateTest, EqAlwaysTrueDouble) {
+    const auto value = Field::create_field<TYPE_DOUBLE>(1.0);
+    ComparisonPredicateBase<TYPE_DOUBLE, PredicateType::EQ> predicate(0, 
"value", value);
+    segment_v2::ZoneMap zone_map {
+            .min_value = value, .max_value = value, .has_null = false, 
.has_not_null = true};
+    EXPECT_TRUE(predicate.is_always_true(zone_map));
+
+    zone_map.max_value = 
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN());
+    EXPECT_FALSE(predicate.is_always_true(zone_map));
+    ComparisonPredicateBase<TYPE_DOUBLE, PredicateType::EQ> nan_predicate(0, 
"value",
+                                                                          
zone_map.max_value);
+    zone_map.min_value = zone_map.max_value;
+    // Doris treats NaN as equal to NaN, including during row evaluation.
+    EXPECT_TRUE(nan_predicate.is_always_true(zone_map));
+}
+
 // test zonemap index
 TEST_F(BlockColumnPredicateTest, test_double_single_column_predicate) {
     auto nan = 
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN());


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to