This is an automated email from the ASF dual-hosted git repository.
wangbo pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new fa2746ffde6 [branch-3.0]count tablet meta's static memory load from
disk (#41429) (#45788)
fa2746ffde6 is described below
commit fa2746ffde697877ba96f04928da16d08bb20c4c
Author: wangbo <[email protected]>
AuthorDate: Tue Dec 24 10:13:28 2024 +0800
[branch-3.0]count tablet meta's static memory load from disk (#41429)
(#45788)
pick #41429
---
be/src/olap/metadata_adder.h | 227 +++++++++++++++++++++
be/src/olap/rowset/rowset_meta.cpp | 7 +
be/src/olap/rowset/rowset_meta.h | 5 +-
.../olap/rowset/segment_v2/bitmap_index_reader.h | 2 +-
.../segment_v2/bloom_filter_index_reader.cpp | 6 +
.../rowset/segment_v2/bloom_filter_index_reader.h | 4 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 13 +-
be/src/olap/rowset/segment_v2/column_reader.h | 4 +-
be/src/olap/rowset/segment_v2/index_page.cpp | 9 +
be/src/olap/rowset/segment_v2/index_page.h | 6 +-
.../rowset/segment_v2/indexed_column_reader.cpp | 12 +-
.../olap/rowset/segment_v2/indexed_column_reader.h | 4 +-
.../olap/rowset/segment_v2/inverted_index_reader.h | 3 +-
.../olap/rowset/segment_v2/ordinal_page_index.cpp | 21 +-
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 4 +-
be/src/olap/rowset/segment_v2/segment.cpp | 18 +-
be/src/olap/rowset/segment_v2/segment.h | 4 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 17 +-
be/src/olap/rowset/segment_v2/zone_map_index.h | 5 +-
be/src/olap/tablet_meta.cpp | 3 +-
be/src/olap/tablet_meta.h | 3 +-
be/src/olap/tablet_schema.cpp | 16 +-
be/src/olap/tablet_schema.h | 10 +-
23 files changed, 337 insertions(+), 66 deletions(-)
diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
new file mode 100644
index 00000000000..bdc9e7a398d
--- /dev/null
+++ b/be/src/olap/metadata_adder.h
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <bvar/bvar.h>
+#include <stdint.h>
+
+namespace doris {
+
+inline bvar::Adder<int64_t>
g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
+inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
+
+inline bvar::Adder<int64_t>
g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
+
+inline bvar::Adder<int64_t>
g_tablet_column_mem_bytes("doris_tablet_column_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_column_num("doris_tablet_column_num");
+
+inline bvar::Adder<int64_t>
g_tablet_index_mem_bytes("doris_tablet_index_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
+
+inline bvar::Adder<int64_t>
g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
+
+inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
+inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+
+inline bvar::Adder<int64_t>
g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
+
+inline bvar::Adder<int64_t>
g_bitmap_index_reader_mem_bytes("doris_bitmap_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_bitmap_index_reader_num("doris_bitmap_index_reader_num");
+
+inline bvar::Adder<int64_t> g_bloom_filter_index_reader_mem_bytes(
+ "doris_bloom_filter_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_bloom_filter_index_reader_num("doris_bloom_filter_index_reader_num");
+
+inline bvar::Adder<int64_t>
g_index_page_reader_mem_bytes("doris_index_page_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_index_page_reader_num("doris_index_page_reader_num");
+
+inline bvar::Adder<int64_t> g_indexed_column_reader_mem_bytes(
+ "doris_indexed_column_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_indexed_column_reader_num("doris_indexed_column_reader_num");
+
+inline bvar::Adder<int64_t> g_inverted_index_reader_mem_bytes(
+ "doris_inverted_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_inverted_index_reader_num("doris_inverted_index_reader_num");
+
+inline bvar::Adder<int64_t> g_ordinal_index_reader_mem_bytes(
+ "doris_ordinal_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_ordinal_index_reader_num("doris_ordinal_index_reader_num");
+
+inline bvar::Adder<int64_t> g_zone_map_index_reader_mem_bytes(
+ "doris_zone_map_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_zone_map_index_reader_num("doris_zone_map_index_reader_num");
+
+class RowsetMeta;
+class TabletMeta;
+class TabletColumn;
+class TabletIndex;
+class TabletSchema;
+
+namespace segment_v2 {
+class Segment;
+class ColumnReader;
+class BitmapIndexReader;
+class BloomFilterIndexReader;
+class IndexPageReader;
+class IndexedColumnReader;
+class InvertedIndexReader;
+class OrdinalIndexReader;
+class ZoneMapIndexReader;
+}; // namespace segment_v2
+
+/*
+ When a derived Class extends MetadataAdder, then the Class's number and
fixed length field's memory can be counted automatically.
+ But if the Class has variable length field, then you should overwrite
get_metadata_size and call update_metadata_size when the Class's memory changes.
+
+ There are some special situations that need to be noted:
+ 1. when the derived Class override copy constructor, you'd better update
memory size(call update_metadata_size) if derived class's
+ memory changed in its copy constructor or you not call MetadataAdder's
copy constructor.
+ 2. when the derived Class override operator=, you'd better update memory
size(call update_metadata_size) if the derived Class has variable length field;
+
+ Anyway, you should update mem size whenever derived Class's memory changes.
+*/
+
+template <typename T>
+class MetadataAdder {
+public:
+ MetadataAdder();
+
+protected:
+ MetadataAdder(const MetadataAdder& other);
+
+ virtual ~MetadataAdder();
+
+ virtual int64_t get_metadata_size() const { return sizeof(T); }
+
+ void update_metadata_size();
+
+ MetadataAdder<T>& operator=(const MetadataAdder<T>& other) = default;
+
+private:
+ int64_t _current_meta_size {0};
+
+ void add_mem_size(int64_t val);
+
+ void add_num(int64_t val);
+};
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder(const MetadataAdder<T>& other) {
+ this->_current_meta_size = other._current_meta_size;
+ add_num(1);
+ add_mem_size(this->_current_meta_size);
+}
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder() {
+ this->_current_meta_size = sizeof(T);
+ add_mem_size(this->_current_meta_size);
+ add_num(1);
+}
+
+template <typename T>
+MetadataAdder<T>::~MetadataAdder() {
+ add_mem_size(-_current_meta_size);
+ add_num(-1);
+}
+
+template <typename T>
+void MetadataAdder<T>::update_metadata_size() {
+ int64_t old_size = _current_meta_size;
+ _current_meta_size = get_metadata_size();
+ int64_t size_diff = _current_meta_size - old_size;
+
+ add_mem_size(size_diff);
+}
+
+template <typename T>
+void MetadataAdder<T>::add_mem_size(int64_t val) {
+ if (val == 0) {
+ return;
+ }
+ if constexpr (std::is_same_v<T, RowsetMeta>) {
+ g_rowset_meta_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletMeta>) {
+ g_tablet_meta_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletColumn>) {
+ g_tablet_column_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletIndex>) {
+ g_tablet_index_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletSchema>) {
+ g_tablet_schema_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+ g_segment_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+ g_column_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
+ g_bitmap_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T,
segment_v2::BloomFilterIndexReader>) {
+ g_bloom_filter_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexPageReader>) {
+ g_index_page_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
+ g_indexed_column_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ g_inverted_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
+ g_ordinal_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
+ g_zone_map_index_reader_mem_bytes << val;
+ }
+}
+
+template <typename T>
+void MetadataAdder<T>::add_num(int64_t val) {
+ if (val == 0) {
+ return;
+ }
+ if constexpr (std::is_same_v<T, RowsetMeta>) {
+ g_rowset_meta_num << val;
+ } else if constexpr (std::is_same_v<T, TabletMeta>) {
+ g_tablet_meta_num << val;
+ } else if constexpr (std::is_same_v<T, TabletColumn>) {
+ g_tablet_column_num << val;
+ } else if constexpr (std::is_same_v<T, TabletIndex>) {
+ g_tablet_index_num << val;
+ } else if constexpr (std::is_same_v<T, TabletSchema>) {
+ g_tablet_schema_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+ g_segment_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+ g_column_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
+ g_bitmap_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T,
segment_v2::BloomFilterIndexReader>) {
+ g_bloom_filter_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexPageReader>) {
+ g_index_page_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
+ g_indexed_column_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ g_inverted_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
+ g_ordinal_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
+ g_zone_map_index_reader_num << val;
+ }
+}
+
+}; // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/rowset_meta.cpp
b/be/src/olap/rowset/rowset_meta.cpp
index 3e175050cac..6bed5e800ed 100644
--- a/be/src/olap/rowset/rowset_meta.cpp
+++ b/be/src/olap/rowset/rowset_meta.cpp
@@ -199,6 +199,7 @@ void RowsetMeta::_init() {
} else {
_rowset_id.init(_rowset_meta_pb.rowset_id_v2());
}
+ update_metadata_size();
}
void RowsetMeta::add_segments_file_size(const std::vector<size_t>&
seg_file_size) {
@@ -256,6 +257,12 @@ void RowsetMeta::merge_rowset_meta(const RowsetMeta&
other) {
if (rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) {
set_rowset_state(RowsetStatePB::COMMITTED);
}
+
+ update_metadata_size();
+}
+
+int64_t RowsetMeta::get_metadata_size() const {
+ return sizeof(RowsetMeta) + _rowset_meta_pb.ByteSizeLong();
}
InvertedIndexFileInfo RowsetMeta::inverted_index_file_info(int seg_id) {
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index ce414fcc7c0..46121aeae2b 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -25,6 +25,7 @@
#include <vector>
#include "io/fs/file_system.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_fwd.h"
#include "olap/storage_policy.h"
@@ -33,7 +34,7 @@
namespace doris {
-class RowsetMeta {
+class RowsetMeta : public MetadataAdder<RowsetMeta> {
public:
RowsetMeta() = default;
~RowsetMeta();
@@ -366,6 +367,8 @@ public:
void add_inverted_index_files_info(
const std::vector<const InvertedIndexFileInfo*>& idx_file_info);
+ int64_t get_metadata_size() const override;
+
// Because the member field '_handle' is a raw pointer, use member func
'init' to replace copy ctor
RowsetMeta(const RowsetMeta&) = delete;
RowsetMeta operator=(const RowsetMeta&) = delete;
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
index 9753972583e..8d344e43ac7 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
@@ -41,7 +41,7 @@ namespace segment_v2 {
class BitmapIndexIterator;
class BitmapIndexPB;
-class BitmapIndexReader {
+class BitmapIndexReader : public MetadataAdder<BitmapIndexReader> {
public:
explicit BitmapIndexReader(io::FileReaderSPtr file_reader, const
BitmapIndexPB& index_meta)
: _file_reader(std::move(file_reader)),
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
index 3bf6d6ca80a..d81b18c7ed5 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
@@ -39,12 +39,18 @@ Status BloomFilterIndexReader::load(bool use_page_cache,
bool kept_in_memory,
});
}
+int64_t BloomFilterIndexReader::get_metadata_size() const {
+ return sizeof(BloomFilterIndexReader) +
+ (_bloom_filter_index_meta ?
_bloom_filter_index_meta->ByteSizeLong() : 0);
+}
+
Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats) {
const IndexedColumnMetaPB& bf_index_meta =
_bloom_filter_index_meta->bloom_filter();
_bloom_filter_reader.reset(new IndexedColumnReader(_file_reader,
bf_index_meta));
RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory,
index_load_stats));
+ update_metadata_size();
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
index 27643f74d86..fb53af89c0f 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
@@ -38,7 +38,7 @@ class BloomFilterIndexIterator;
class BloomFilter;
class BloomFilterIndexPB;
-class BloomFilterIndexReader {
+class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
public:
explicit BloomFilterIndexReader(io::FileReaderSPtr file_reader,
const BloomFilterIndexPB&
bloom_filter_index_meta)
@@ -61,6 +61,8 @@ public:
private:
Status _load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats);
+ int64_t get_metadata_size() const override;
+
private:
friend class BloomFilterIndexIterator;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 3a74405fe8a..4d7a281c656 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -85,8 +85,6 @@ inline bool read_as_string(PrimitiveType type) {
type == PrimitiveType::TYPE_OBJECT;
}
-static bvar::Adder<size_t>
g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
-static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
Status ColumnReader::create_array(const ColumnReaderOptions& opts, const
ColumnMetaPB& meta,
const io::FileReaderSPtr& file_reader,
std::unique_ptr<ColumnReader>* reader) {
@@ -265,14 +263,12 @@ ColumnReader::ColumnReader(const ColumnReaderOptions&
opts, const ColumnMetaPB&
_meta_is_nullable = meta.is_nullable();
_meta_dict_page = meta.dict_page();
_meta_compression = meta.compression();
-
- g_column_reader_memory_bytes << sizeof(*this);
- g_column_reader_num << 1;
}
-ColumnReader::~ColumnReader() {
- g_column_reader_memory_bytes << -sizeof(*this);
- g_column_reader_num << -1;
+ColumnReader::~ColumnReader() = default;
+
+int64_t ColumnReader::get_metadata_size() const {
+ return sizeof(ColumnReader) + (_segment_zone_map ?
_segment_zone_map->ByteSizeLong() : 0);
}
Status ColumnReader::init(const ColumnMetaPB* meta) {
@@ -312,6 +308,7 @@ Status ColumnReader::init(const ColumnMetaPB* meta) {
_file_reader->path().native(),
index_meta.type());
}
}
+ update_metadata_size();
// ArrayColumnWriter writes a single empty array and flushes. In this
scenario,
// the item writer doesn't write any data and the corresponding ordinal
index is empty.
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h
b/be/src/olap/rowset/segment_v2/column_reader.h
index 5fc31d7c03a..97fef8435ec 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -111,7 +111,7 @@ struct ColumnIteratorOptions {
// we should do our best to reduce resource usage through share
// same information, such as OrdinalPageIndex and Page data.
// This will cache data shared by all reader
-class ColumnReader {
+class ColumnReader : public MetadataAdder<ColumnReader> {
public:
// Create an initialized ColumnReader in *reader.
// This should be a lightweight operation without I/O.
@@ -252,6 +252,8 @@ private:
Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes,
RowRanges* row_ranges,
const ColumnIteratorOptions& iter_opts);
+ int64_t get_metadata_size() const override;
+
private:
int64_t _meta_length;
FieldType _meta_type;
diff --git a/be/src/olap/rowset/segment_v2/index_page.cpp
b/be/src/olap/rowset/segment_v2/index_page.cpp
index 9af7047c49b..1b033a9ff62 100644
--- a/be/src/olap/rowset/segment_v2/index_page.cpp
+++ b/be/src/olap/rowset/segment_v2/index_page.cpp
@@ -64,6 +64,10 @@ Status IndexPageBuilder::get_first_key(Slice* key) const {
///////////////////////////////////////////////////////////////////////////////
+int64_t IndexPageReader::get_metadata_size() const {
+ return sizeof(IndexPageReader) + _vl_field_mem_size;
+}
+
Status IndexPageReader::parse(const Slice& body, const IndexPageFooterPB&
footer) {
_footer = footer;
size_t num_entries = _footer.num_entries();
@@ -80,8 +84,13 @@ Status IndexPageReader::parse(const Slice& body, const
IndexPageFooterPB& footer
}
_keys.push_back(key);
_values.push_back(value);
+ _vl_field_mem_size += sizeof(char) * key.size;
}
+ _vl_field_mem_size +=
+ _keys.capacity() * sizeof(Slice) + _values.capacity() *
sizeof(PagePointer);
+ _vl_field_mem_size += _footer.ByteSizeLong();
+ update_metadata_size();
_parsed = true;
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/index_page.h
b/be/src/olap/rowset/segment_v2/index_page.h
index 7b15ef66391..0ebf425fc5c 100644
--- a/be/src/olap/rowset/segment_v2/index_page.h
+++ b/be/src/olap/rowset/segment_v2/index_page.h
@@ -26,6 +26,7 @@
#include <vector>
#include "common/status.h"
+#include "olap/metadata_adder.h"
#include "olap/rowset/segment_v2/page_pointer.h"
#include "util/faststring.h"
#include "util/slice.h"
@@ -79,7 +80,7 @@ private:
uint32_t _count = 0;
};
-class IndexPageReader {
+class IndexPageReader : public MetadataAdder<IndexPageReader> {
public:
IndexPageReader() : _parsed(false) {}
@@ -110,11 +111,14 @@ public:
void reset();
private:
+ int64_t get_metadata_size() const override;
+
bool _parsed;
IndexPageFooterPB _footer;
std::vector<Slice> _keys;
std::vector<PagePointer> _values;
+ int64_t _vl_field_mem_size {0};
};
class IndexPageIterator {
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 8a81ff1469a..a1a0aa15c6f 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -56,10 +56,12 @@ static bvar::Adder<uint64_t>
g_index_reader_pk_pages("doris_pk", "index_reader_p
static bvar::PerSecond<bvar::Adder<uint64_t>>
g_index_reader_pk_bytes_per_second(
"doris_pk", "index_reader_pk_pages_per_second",
&g_index_reader_pk_pages, 60);
-static bvar::Adder<uint64_t>
g_index_reader_memory_bytes("doris_index_reader_memory_bytes");
-
using strings::Substitute;
+int64_t IndexedColumnReader::get_metadata_size() const {
+ return sizeof(IndexedColumnReader) + _meta.ByteSizeLong();
+}
+
Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats) {
_use_page_cache = use_page_cache;
@@ -97,7 +99,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool
kept_in_memory,
}
_num_values = _meta.num_values();
- g_index_reader_memory_bytes << sizeof(*this);
+ update_metadata_size();
return Status::OK();
}
@@ -145,9 +147,7 @@ Status IndexedColumnReader::read_page(const PagePointer&
pp, PageHandle* handle,
return st;
}
-IndexedColumnReader::~IndexedColumnReader() {
- g_index_reader_memory_bytes << -sizeof(*this);
-}
+IndexedColumnReader::~IndexedColumnReader() = default;
///////////////////////////////////////////////////////////////////////////////
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index 2691874346d..23b1c070a81 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -47,7 +47,7 @@ namespace segment_v2 {
class EncodingInfo;
// thread-safe reader for IndexedColumn (see comments of `IndexedColumnWriter`
to understand what IndexedColumn is)
-class IndexedColumnReader {
+class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
public:
explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const
IndexedColumnMetaPB& meta)
: _file_reader(std::move(file_reader)), _meta(meta) {}
@@ -76,6 +76,8 @@ private:
Status load_index_page(const PagePointerPB& pp, PageHandle* handle,
IndexPageReader* reader,
OlapReaderStatistics* index_load_stats);
+ int64_t get_metadata_size() const override;
+
friend class IndexedColumnIterator;
io::FileReaderSPtr _file_reader;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index 57975ab7fec..a1445603286 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -171,7 +171,8 @@ public:
bool is_empty() const { return (_data_bitmap == nullptr && _null_bitmap ==
nullptr); }
};
-class InvertedIndexReader : public
std::enable_shared_from_this<InvertedIndexReader> {
+class InvertedIndexReader : public
std::enable_shared_from_this<InvertedIndexReader>,
+ public MetadataAdder<InvertedIndexReader> {
public:
explicit InvertedIndexReader(
const TabletIndex* index_meta,
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
index 196c5f45687..4995e779892 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
@@ -34,8 +34,6 @@
namespace doris {
-static bvar::Adder<size_t>
g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes");
-
namespace segment_v2 {
void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer&
data_pp) {
@@ -120,10 +118,6 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
_ordinals.resize(_num_pages + 1);
_pages.resize(_num_pages);
- g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() *
sizeof(ordinal_t) +
- _pages.size() *
sizeof(PagePointer) +
- sizeof(OrdinalIndexReader);
-
for (int i = 0; i < _num_pages; i++) {
Slice key = reader.get_key(i);
ordinal_t ordinal = 0;
@@ -136,9 +130,16 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
}
_ordinals[_num_pages] = _num_values;
+ update_metadata_size();
+
return Status::OK();
}
+int64_t OrdinalIndexReader::get_metadata_size() const {
+ return sizeof(OrdinalIndexReader) + _ordinals.capacity() *
sizeof(ordinal_t) +
+ _pages.capacity() * sizeof(PagePointer);
+}
+
OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t
ordinal) {
int32_t left = 0;
int32_t right = _num_pages - 1;
@@ -160,13 +161,7 @@ OrdinalPageIndexIterator
OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal
return OrdinalPageIndexIterator(this, left);
}
-OrdinalIndexReader::~OrdinalIndexReader() {
- if (_ordinals.size() > 0) {
- g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() *
sizeof(ordinal_t) -
- _pages.size() *
sizeof(PagePointer) -
- sizeof(OrdinalIndexReader);
- }
-}
+OrdinalIndexReader::~OrdinalIndexReader() = default;
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
index 68a84768558..df60edb12d1 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
@@ -64,7 +64,7 @@ private:
class OrdinalPageIndexIterator;
-class OrdinalIndexReader {
+class OrdinalIndexReader : public MetadataAdder<OrdinalIndexReader> {
public:
explicit OrdinalIndexReader(io::FileReaderSPtr file_reader, ordinal_t
num_values,
const OrdinalIndexPB& meta_pb)
@@ -97,6 +97,8 @@ private:
std::unique_ptr<OrdinalIndexPB> index_meta,
OlapReaderStatistics* index_load_stats);
+ int64_t get_metadata_size() const override;
+
private:
friend OrdinalPageIndexIterator;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index bc0e6e671e2..f7c43ea4df4 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -74,7 +74,7 @@
#include "vec/olap/vgeneric_iterators.h"
namespace doris::segment_v2 {
-static bvar::Adder<size_t> g_total_segment_num("doris_total_segment_num");
+
class InvertedIndexIterator;
io::UInt128Wrapper file_cache_key_from_path(const std::string& seg_path) {
@@ -161,18 +161,19 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id,
TabletSchemaSPtr table
_meta_mem_usage(0),
_rowset_id(rowset_id),
_tablet_schema(std::move(tablet_schema)),
- _idx_file_info(idx_file_info) {
- g_total_segment_num << 1;
-}
+ _idx_file_info(idx_file_info) {}
-Segment::~Segment() {
- g_total_segment_num << -1;
-}
+Segment::~Segment() = default;
io::UInt128Wrapper Segment::file_cache_key(std::string_view rowset_id,
uint32_t seg_id) {
return io::BlockFileCache::hash(fmt::format("{}_{}.dat", rowset_id,
seg_id));
}
+int64_t Segment::get_metadata_size() const {
+ return sizeof(Segment) + (_footer_pb ? _footer_pb->ByteSizeLong() : 0) +
+ (_pk_index_meta ? _pk_index_meta->ByteSizeLong() : 0);
+}
+
Status Segment::_open() {
_footer_pb = std::make_unique<SegmentFooterPB>();
RETURN_IF_ERROR(_parse_footer(_footer_pb.get()));
@@ -189,6 +190,9 @@ Status Segment::_open() {
if (_pk_index_meta != nullptr) {
_meta_mem_usage += _pk_index_meta->ByteSizeLong();
}
+
+ update_metadata_size();
+
_meta_mem_usage += sizeof(*this);
_meta_mem_usage += _tablet_schema->num_columns() *
config::estimated_mem_per_column_reader;
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index 96409bff343..931ddd31c56 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -78,7 +78,7 @@ using SegmentSharedPtr = std::shared_ptr<Segment>;
// NOTE: This segment is used to a specified TabletSchema, when TabletSchema
// is changed, this segment can not be used any more. For example, after a
schema
// change finished, client should disable all cached Segment for old
TabletSchema.
-class Segment : public std::enable_shared_from_this<Segment> {
+class Segment : public std::enable_shared_from_this<Segment>, public
MetadataAdder<Segment> {
public:
static Status open(io::FileSystemSPtr fs, const std::string& path, int64_t
tablet_id,
uint32_t segment_id, RowsetId rowset_id,
TabletSchemaSPtr tablet_schema,
@@ -92,6 +92,8 @@ public:
~Segment();
+ int64_t get_metadata_size() const override;
+
Status new_iterator(SchemaSPtr schema, const StorageReadOptions&
read_options,
std::unique_ptr<RowwiseIterator>* iter);
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
index ea7d74e520a..9249c82aedf 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
@@ -39,8 +39,6 @@
namespace doris {
struct uint24_t;
-static bvar::Adder<size_t>
g_zone_map_memory_bytes("doris_zone_map_memory_bytes");
-
namespace segment_v2 {
template <PrimitiveType Type>
@@ -160,9 +158,6 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
_page_zone_maps.resize(reader.num_values());
- g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) *
_page_zone_maps.size() +
- sizeof(IndexedColumnMetaPB);
-
// read and cache all page zone maps
for (int i = 0; i < reader.num_values(); ++i) {
size_t num_to_read = 1;
@@ -180,18 +175,18 @@ Status ZoneMapIndexReader::_load(bool use_page_cache,
bool kept_in_memory,
column->get_data_at(0).size)) {
return Status::Corruption("Failed to parse zone map");
}
+ _pb_meta_size += _page_zone_maps[i].ByteSizeLong();
}
+ update_metadata_size();
return Status::OK();
}
-ZoneMapIndexReader::~ZoneMapIndexReader() {
- // Maybe wrong due to load failures.
- if (_page_zone_maps.size() > 0) {
- g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) *
_page_zone_maps.size() -
- sizeof(IndexedColumnMetaPB);
- }
+int64_t ZoneMapIndexReader::get_metadata_size() const {
+ return sizeof(ZoneMapIndexReader) + _pb_meta_size;
}
+
+ZoneMapIndexReader::~ZoneMapIndexReader() = default;
#define APPLY_FOR_PRIMITITYPE(M) \
M(TYPE_TINYINT) \
M(TYPE_SMALLINT) \
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h
b/be/src/olap/rowset/segment_v2/zone_map_index.h
index 07cf845fd97..04cae12975c 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.h
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.h
@@ -143,7 +143,7 @@ private:
uint64_t _estimated_size = 0;
};
-class ZoneMapIndexReader {
+class ZoneMapIndexReader : public MetadataAdder<ZoneMapIndexReader> {
public:
explicit ZoneMapIndexReader(io::FileReaderSPtr file_reader,
const IndexedColumnMetaPB& page_zone_maps)
@@ -165,12 +165,15 @@ private:
Status _load(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<IndexedColumnMetaPB>,
OlapReaderStatistics* index_load_stats);
+ int64_t get_metadata_size() const override;
+
private:
DorisCallOnce<Status> _load_once;
// TODO: yyq, we shoud remove file_reader from here.
io::FileReaderSPtr _file_reader;
std::unique_ptr<IndexedColumnMetaPB> _page_zone_maps_meta;
std::vector<ZoneMapPB> _page_zone_maps;
+ int64_t _pb_meta_size {0};
};
} // namespace segment_v2
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index a8dd0e56273..237617c47f0 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -347,7 +347,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
}
TabletMeta::TabletMeta(const TabletMeta& b)
- : _table_id(b._table_id),
+ : MetadataAdder(b),
+ _table_id(b._table_id),
_index_id(b._index_id),
_partition_id(b._partition_id),
_tablet_id(b._tablet_id),
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 9f47ef4913c..550c478694f 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -43,6 +43,7 @@
#include "io/fs/file_system.h"
#include "olap/binlog_config.h"
#include "olap/lru_cache.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/tablet_schema.h"
@@ -91,7 +92,7 @@ class TBinlogConfig;
// Class encapsulates meta of tablet.
// The concurrency control is handled in Tablet Class, not in this class.
-class TabletMeta {
+class TabletMeta : public MetadataAdder<TabletMeta> {
public:
static TabletMetaSharedPtr create(
const TCreateTabletReq& request, const TabletUid& tablet_uid,
uint64_t shard_id,
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index e668d74b53d..ec357dbcdb4 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -55,8 +55,6 @@
namespace doris {
-static bvar::Adder<size_t>
g_total_tablet_schema_num("doris_total_tablet_schema_num");
-
FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
switch (primitiveType) {
case PrimitiveType::INVALID_TYPE:
@@ -855,15 +853,16 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index)
const {
}
}
-TabletSchema::TabletSchema() {
- g_total_tablet_schema_num << 1;
-}
+TabletSchema::TabletSchema() = default;
TabletSchema::~TabletSchema() {
- g_total_tablet_schema_num << -1;
clear_column_cache_handlers();
}
+int64_t TabletSchema::get_metadata_size() const {
+ return sizeof(TabletSchema) + _vl_field_mem_size;
+}
+
void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
if (column.is_key()) {
_num_key_columns++;
@@ -1005,7 +1004,10 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
_cols.emplace_back(std::move(column));
if (!_cols.back()->is_extracted_column()) {
+ _vl_field_mem_size += sizeof(StringRef) + sizeof(char) *
_cols.back()->name().size() +
+ sizeof(int32_t);
_field_name_to_index.emplace(StringRef(_cols.back()->name()),
_num_columns);
+ _vl_field_mem_size += sizeof(int32_t) * 2;
_field_id_to_index[_cols.back()->unique_id()] = _num_columns;
}
_num_columns++;
@@ -1049,6 +1051,8 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
_row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(),
schema.row_store_column_unique_ids().end());
_variant_enable_flatten_nested = schema.variant_enable_flatten_nested();
+ _vl_field_mem_size += _row_store_column_unique_ids.capacity() *
sizeof(int32_t);
+ update_metadata_size();
}
void TabletSchema::copy_from(const TabletSchema& tablet_schema) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 24b0f7b4fe3..223e65790b0 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -35,6 +35,7 @@
#include "common/consts.h"
#include "common/status.h"
#include "gutil/stringprintf.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/options.h"
#include "runtime/define_primitive_type.h"
@@ -61,7 +62,7 @@ class TabletColumn;
using TabletColumnPtr = std::shared_ptr<TabletColumn>;
-class TabletColumn {
+class TabletColumn : public MetadataAdder<TabletColumn> {
public:
TabletColumn();
TabletColumn(const ColumnPB& column);
@@ -252,7 +253,7 @@ bool operator!=(const TabletColumn& a, const TabletColumn&
b);
class TabletSchema;
-class TabletIndex {
+class TabletIndex : public MetadataAdder<TabletIndex> {
public:
TabletIndex() = default;
void init_from_thrift(const TOlapTableIndex& index, const TabletSchema&
tablet_schema);
@@ -294,7 +295,7 @@ private:
std::map<string, string> _properties;
};
-class TabletSchema {
+class TabletSchema : public MetadataAdder<TabletSchema> {
public:
enum ColumnType { NORMAL = 0, DROPPED = 1, VARIANT = 2 };
// TODO(yingchun): better to make constructor as private to avoid
@@ -524,6 +525,8 @@ public:
const std::vector<int32_t>& row_columns_uids() const { return
_row_store_column_unique_ids; }
+ int64_t get_metadata_size() const override;
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -575,6 +578,7 @@ private:
// ATTN: For compability reason empty cids means all columns of tablet
schema are encoded to row column
std::vector<int32_t> _row_store_column_unique_ids;
bool _variant_enable_flatten_nested = false;
+ int64_t _vl_field_mem_size {0}; // variable length field
};
bool operator==(const TabletSchema& a, const TabletSchema& b);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]