github-actions[bot] commented on code in PR #66360: URL: https://github.com/apache/doris/pull/66360#discussion_r3702214713
########## be/benchmark/parquet/benchmark_parquet_selection.hpp: ########## @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <benchmark/benchmark.h> + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <numeric> +#include <string> +#include <vector> + +#include "format_v2/parquet/selection_vector.h" +#include "parquet_benchmark_scenarios.h" + +namespace doris::parquet_benchmark::selection_detail { + +constexpr size_t SELECTION_ROWS = 1UL << 12; +constexpr int CASCADE_FIRST_SELECTIVITY = 90; + +inline std::vector<uint8_t> make_filter(size_t rows, int selectivity_percent, Pattern pattern) { + std::vector<uint8_t> filter(rows, 0); + const size_t selected_rows = + rows * static_cast<size_t>(std::clamp(selectivity_percent, 0, 100)) / 100; + if (pattern == Pattern::CLUSTERED) { + std::fill_n(filter.begin(), selected_rows, uint8_t {1}); + } else if (selected_rows != 0) { + for (size_t selected = 0; selected < selected_rows; ++selected) { + filter[selected * rows / selected_rows] = 1; + } + } + return filter; +} + +// Keep the benchmark source buildable on revisions before the bulk compaction helpers. The +// fallback mirrors the former Parquet scan loops so one named matrix can compare both revisions. +template <typename Selection> +size_t compact_with_row_filter(Selection* selection, const uint8_t* filter, size_t rows) { + if constexpr (requires { selection->compact_with_row_filter(filter, rows); }) { + return selection->compact_with_row_filter(filter, rows); + } else { + size_t output = 0; + for (size_t position = 0; position < rows; ++position) { + const auto row = selection->get_index(position); + if (filter[row] != 0) { + selection->set_index(output++, row); + } + } + return output; + } +} + +template <typename Selection> +size_t compact_with_selection_filter(Selection* selection, const uint8_t* filter, size_t rows) { + if constexpr (requires { selection->compact_with_selection_filter(filter, rows); }) { + return selection->compact_with_selection_filter(filter, rows); + } else { + size_t output = 0; + for (size_t position = 0; position < rows; ++position) { + if (filter[position] != 0) { + selection->set_index(output++, selection->get_index(position)); + } + } + return output; + } +} + +inline std::vector<format::parquet::SelectionVector::Index> expected_selection_rows( + const SelectionScenario& scenario, const std::vector<uint8_t>& row_filter, + const std::vector<uint8_t>& first_filter, const std::vector<uint8_t>& selection_filter) { + std::vector<format::parquet::SelectionVector::Index> rows(SELECTION_ROWS); + std::iota(rows.begin(), rows.end(), 0); + if (scenario.operation == SelectionOperation::RESIZE_IDENTITY) { + return rows; + } + if (scenario.operation == SelectionOperation::ROW_FILTER) { + std::erase_if(rows, [&](const auto row) { return row_filter[row] == 0; }); + return rows; + } + std::erase_if(rows, [&](const auto row) { return first_filter[row] == 0; }); + std::vector<format::parquet::SelectionVector::Index> cascaded; + cascaded.reserve(rows.size()); + for (size_t position = 0; position < rows.size(); ++position) { + if (selection_filter[position] != 0) { + cascaded.push_back(rows[position]); + } + } + return cascaded; +} + +inline void run_selection(benchmark::State& state, const SelectionScenario& scenario) { + format::parquet::SelectionVector selection; + const auto row_filter = + make_filter(SELECTION_ROWS, scenario.selectivity_percent, scenario.pattern); + const auto first_filter = + make_filter(SELECTION_ROWS, CASCADE_FIRST_SELECTIVITY, Pattern::ALTERNATING); + const size_t first_selected = + static_cast<size_t>(std::count(first_filter.begin(), first_filter.end(), uint8_t {1})); + const auto selection_filter = + make_filter(first_selected, scenario.selectivity_percent, scenario.pattern); + const auto expected_rows = + expected_selection_rows(scenario, row_filter, first_filter, selection_filter); + + size_t selected_rows = 0; + for (auto _ : state) { + selection.resize(SELECTION_ROWS); Review Comment: [P2] Make the resized SelectionVector state observable in the identity benchmark. A default vector already returns `get_index(i) == i` while its size is zero, and this branch only escapes the constant `selected_rows`, so the compiler can remove `resize()`'s otherwise-dead metadata stores and the post-loop oracle still passes. That can turn the reported identity-initialization speedup into loop overhead. Please use the standard benchmark escape/barrier pattern on the resized state and assert `selection.size() == SELECTION_ROWS` outside timing so a no-op or eliminated resize cannot pass. ########## be/benchmark/parquet/benchmark_file_scanner_expr.hpp: ########## @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <benchmark/benchmark.h> + +#include <array> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <string> + +#include "core/data_type/data_type_number.h" +#include "exprs/create_predicate_function.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" + +namespace doris::parquet_benchmark::file_scanner_expr_detail { + +inline TExprNode make_direct_in_node() { + TExprNode node; + node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::IN_PRED); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_num_children(1); + node.__set_is_nullable(false); + node.in_predicate.__set_is_not_in(false); + return node; +} + +inline void run_direct_in_clone_prepare_open(benchmark::State& state, size_t cardinality, + bool share_pruning_state) { + std::shared_ptr<HybridSetBase> filter(create_set(PrimitiveType::TYPE_INT, cardinality, false)); + for (size_t index = 0; index < cardinality; ++index) { + const int32_t value = static_cast<int32_t>(index); + filter->insert(&value); + } + auto root = VDirectInPredicate::create_shared(make_direct_in_node(), std::move(filter), true); + root->add_child(VSlotRef::create_shared(0, 0, -1, std::make_shared<DataTypeInt32>(), + "runtime_filter_key")); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + RowDescriptor row_desc; + VExprContext original(root); + auto status = original.prepare(&runtime_state, row_desc); + if (status.ok()) { + status = original.open(&runtime_state); + } + if (!status.ok()) { + const auto error = status.to_string(); + state.SkipWithError(error.c_str()); + return; + } + + for (auto _ : state) { + VExprSPtr cloned_root; + if (share_pruning_state) { + status = root->deep_clone(&cloned_root); + } else { + auto rematerialized = VDirectInPredicate::create_shared(make_direct_in_node(), + root->get_set_func(), true); + rematerialized->add_child(VSlotRef::create_shared( + 0, 0, -1, std::make_shared<DataTypeInt32>(), "runtime_filter_key")); + cloned_root = std::move(rematerialized); + status = Status::OK(); + } + if (status.ok()) { + VExprContext cloned(cloned_root); + status = cloned.prepare(&runtime_state, row_desc); + if (status.ok()) { + status = cloned.open(&runtime_state); + } + benchmark::DoNotOptimize(cloned_root); Review Comment: [P2] Validate the shared clone's exact filter semantics outside the timed loop. This path accepts prepare/open success and only passes `cloned_root` to `DoNotOptimize`, so a clone that drops or corrupts set entries can still report the same large speedup. The focused clone test only checks that `{1,30}` rejects `[10,20]`, which an empty set also satisfies. Please execute both shared and rematerialized clones on deterministic hit, miss, and boundary inputs for each cardinality and compare the exact result bitmap with an independently generated oracle. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
