[Fix]count tablet meta's static memory load from disk (#41429)
## Proposed changes
Count segment and tableMeta's static memory which load from disk, and
add the memory usage to bvar.
diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
new file mode 100644
index 0000000..bdc9e7a
--- /dev/null
+++ b/be/src/olap/metadata_adder.h
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <bvar/bvar.h>
+#include <stdint.h>
+
+namespace doris {
+
+inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
+inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
+
+inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
+
+inline bvar::Adder<int64_t> g_tablet_column_mem_bytes("doris_tablet_column_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_column_num("doris_tablet_column_num");
+
+inline bvar::Adder<int64_t> g_tablet_index_mem_bytes("doris_tablet_index_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
+
+inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
+
+inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
+inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+
+inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
+
+inline bvar::Adder<int64_t> g_bitmap_index_reader_mem_bytes("doris_bitmap_index_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_bitmap_index_reader_num("doris_bitmap_index_reader_num");
+
+inline bvar::Adder<int64_t> g_bloom_filter_index_reader_mem_bytes(
+ "doris_bloom_filter_index_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_bloom_filter_index_reader_num("doris_bloom_filter_index_reader_num");
+
+inline bvar::Adder<int64_t> g_index_page_reader_mem_bytes("doris_index_page_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_index_page_reader_num("doris_index_page_reader_num");
+
+inline bvar::Adder<int64_t> g_indexed_column_reader_mem_bytes(
+ "doris_indexed_column_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_indexed_column_reader_num("doris_indexed_column_reader_num");
+
+inline bvar::Adder<int64_t> g_inverted_index_reader_mem_bytes(
+ "doris_inverted_index_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_inverted_index_reader_num("doris_inverted_index_reader_num");
+
+inline bvar::Adder<int64_t> g_ordinal_index_reader_mem_bytes(
+ "doris_ordinal_index_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_ordinal_index_reader_num("doris_ordinal_index_reader_num");
+
+inline bvar::Adder<int64_t> g_zone_map_index_reader_mem_bytes(
+ "doris_zone_map_index_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_zone_map_index_reader_num("doris_zone_map_index_reader_num");
+
+class RowsetMeta;
+class TabletMeta;
+class TabletColumn;
+class TabletIndex;
+class TabletSchema;
+
+namespace segment_v2 {
+class Segment;
+class ColumnReader;
+class BitmapIndexReader;
+class BloomFilterIndexReader;
+class IndexPageReader;
+class IndexedColumnReader;
+class InvertedIndexReader;
+class OrdinalIndexReader;
+class ZoneMapIndexReader;
+}; // namespace segment_v2
+
+/*
+ When a derived Class extends MetadataAdder, then the Class's number and fixed length field's memory can be counted automatically.
+ But if the Class has variable length field, then you should overwrite get_metadata_size and call update_metadata_size when the Class's memory changes.
+
+ There are some special situations that need to be noted:
+ 1. when the derived Class override copy constructor, you'd better update memory size(call update_metadata_size) if derived class's
+ memory changed in its copy constructor or you not call MetadataAdder's copy constructor.
+ 2. when the derived Class override operator=, you'd better update memory size(call update_metadata_size) if the derived Class has variable length field;
+
+ Anyway, you should update mem size whenever derived Class's memory changes.
+*/
+
+template <typename T>
+class MetadataAdder {
+public:
+ MetadataAdder();
+
+protected:
+ MetadataAdder(const MetadataAdder& other);
+
+ virtual ~MetadataAdder();
+
+ virtual int64_t get_metadata_size() const { return sizeof(T); }
+
+ void update_metadata_size();
+
+ MetadataAdder<T>& operator=(const MetadataAdder<T>& other) = default;
+
+private:
+ int64_t _current_meta_size {0};
+
+ void add_mem_size(int64_t val);
+
+ void add_num(int64_t val);
+};
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder(const MetadataAdder<T>& other) {
+ this->_current_meta_size = other._current_meta_size;
+ add_num(1);
+ add_mem_size(this->_current_meta_size);
+}
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder() {
+ this->_current_meta_size = sizeof(T);
+ add_mem_size(this->_current_meta_size);
+ add_num(1);
+}
+
+template <typename T>
+MetadataAdder<T>::~MetadataAdder() {
+ add_mem_size(-_current_meta_size);
+ add_num(-1);
+}
+
+template <typename T>
+void MetadataAdder<T>::update_metadata_size() {
+ int64_t old_size = _current_meta_size;
+ _current_meta_size = get_metadata_size();
+ int64_t size_diff = _current_meta_size - old_size;
+
+ add_mem_size(size_diff);
+}
+
+template <typename T>
+void MetadataAdder<T>::add_mem_size(int64_t val) {
+ if (val == 0) {
+ return;
+ }
+ if constexpr (std::is_same_v<T, RowsetMeta>) {
+ g_rowset_meta_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletMeta>) {
+ g_tablet_meta_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletColumn>) {
+ g_tablet_column_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletIndex>) {
+ g_tablet_index_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletSchema>) {
+ g_tablet_schema_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+ g_segment_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+ g_column_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
+ g_bitmap_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BloomFilterIndexReader>) {
+ g_bloom_filter_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexPageReader>) {
+ g_index_page_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
+ g_indexed_column_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ g_inverted_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
+ g_ordinal_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
+ g_zone_map_index_reader_mem_bytes << val;
+ }
+}
+
+template <typename T>
+void MetadataAdder<T>::add_num(int64_t val) {
+ if (val == 0) {
+ return;
+ }
+ if constexpr (std::is_same_v<T, RowsetMeta>) {
+ g_rowset_meta_num << val;
+ } else if constexpr (std::is_same_v<T, TabletMeta>) {
+ g_tablet_meta_num << val;
+ } else if constexpr (std::is_same_v<T, TabletColumn>) {
+ g_tablet_column_num << val;
+ } else if constexpr (std::is_same_v<T, TabletIndex>) {
+ g_tablet_index_num << val;
+ } else if constexpr (std::is_same_v<T, TabletSchema>) {
+ g_tablet_schema_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+ g_segment_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+ g_column_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
+ g_bitmap_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BloomFilterIndexReader>) {
+ g_bloom_filter_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexPageReader>) {
+ g_index_page_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
+ g_indexed_column_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ g_inverted_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
+ g_ordinal_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
+ g_zone_map_index_reader_num << val;
+ }
+}
+
+}; // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp
index f053ad2..1843fb8 100644
--- a/be/src/olap/rowset/rowset_meta.cpp
+++ b/be/src/olap/rowset/rowset_meta.cpp
@@ -199,6 +199,7 @@
} else {
_rowset_id.init(_rowset_meta_pb.rowset_id_v2());
}
+ update_metadata_size();
}
void RowsetMeta::add_segments_file_size(const std::vector<size_t>& seg_file_size) {
@@ -255,6 +256,12 @@
if (rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) {
set_rowset_state(RowsetStatePB::COMMITTED);
}
+
+ update_metadata_size();
+}
+
+int64_t RowsetMeta::get_metadata_size() const {
+ return sizeof(RowsetMeta) + _rowset_meta_pb.ByteSizeLong();
}
InvertedIndexFileInfo RowsetMeta::inverted_index_file_info(int seg_id) {
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index 4f25c67..164d42c 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -25,6 +25,7 @@
#include <vector>
#include "io/fs/file_system.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_fwd.h"
#include "olap/storage_policy.h"
@@ -33,7 +34,7 @@
namespace doris {
-class RowsetMeta {
+class RowsetMeta : public MetadataAdder<RowsetMeta> {
public:
RowsetMeta() = default;
~RowsetMeta();
@@ -367,6 +368,8 @@
void update_inverted_index_files_info(const std::vector<InvertedIndexFileInfo>& idx_file_info);
+ int64_t get_metadata_size() const override;
+
// Because the member field '_handle' is a raw pointer, use member func 'init' to replace copy ctor
RowsetMeta(const RowsetMeta&) = delete;
RowsetMeta operator=(const RowsetMeta&) = delete;
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
index 9753972..8d344e4 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
@@ -41,7 +41,7 @@
class BitmapIndexIterator;
class BitmapIndexPB;
-class BitmapIndexReader {
+class BitmapIndexReader : public MetadataAdder<BitmapIndexReader> {
public:
explicit BitmapIndexReader(io::FileReaderSPtr file_reader, const BitmapIndexPB& index_meta)
: _file_reader(std::move(file_reader)),
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
index 0857c18..3a1c9f5 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
@@ -38,11 +38,17 @@
});
}
+int64_t BloomFilterIndexReader::get_metadata_size() const {
+ return sizeof(BloomFilterIndexReader) +
+ (_bloom_filter_index_meta ? _bloom_filter_index_meta->ByteSizeLong() : 0);
+}
+
Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory) {
const IndexedColumnMetaPB& bf_index_meta = _bloom_filter_index_meta->bloom_filter();
_bloom_filter_reader.reset(new IndexedColumnReader(_file_reader, bf_index_meta));
RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory));
+ update_metadata_size();
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
index c2617ef..a10a910 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
@@ -38,7 +38,7 @@
class BloomFilter;
class BloomFilterIndexPB;
-class BloomFilterIndexReader {
+class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
public:
explicit BloomFilterIndexReader(io::FileReaderSPtr file_reader,
const BloomFilterIndexPB& bloom_filter_index_meta)
@@ -59,6 +59,8 @@
private:
Status _load(bool use_page_cache, bool kept_in_memory);
+ int64_t get_metadata_size() const override;
+
private:
friend class BloomFilterIndexIterator;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 3c9b5b7..6924235 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -86,8 +86,6 @@
type == PrimitiveType::TYPE_OBJECT;
}
-static bvar::Adder<size_t> g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
-static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
const io::FileReaderSPtr& file_reader,
std::unique_ptr<ColumnReader>* reader) {
@@ -276,14 +274,12 @@
_meta_is_nullable = meta.is_nullable();
_meta_dict_page = meta.dict_page();
_meta_compression = meta.compression();
-
- g_column_reader_memory_bytes << sizeof(*this);
- g_column_reader_num << 1;
}
-ColumnReader::~ColumnReader() {
- g_column_reader_memory_bytes << -sizeof(*this);
- g_column_reader_num << -1;
+ColumnReader::~ColumnReader() = default;
+
+int64_t ColumnReader::get_metadata_size() const {
+ return sizeof(ColumnReader) + (_segment_zone_map ? _segment_zone_map->ByteSizeLong() : 0);
}
Status ColumnReader::init(const ColumnMetaPB* meta) {
@@ -323,6 +319,7 @@
_file_reader->path().native(), index_meta.type());
}
}
+ update_metadata_size();
// ArrayColumnWriter writes a single empty array and flushes. In this scenario,
// the item writer doesn't write any data and the corresponding ordinal index is empty.
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 6727ea7..b013c71 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -111,7 +111,7 @@
// we should do our best to reduce resource usage through share
// same information, such as OrdinalPageIndex and Page data.
// This will cache data shared by all reader
-class ColumnReader {
+class ColumnReader : public MetadataAdder<ColumnReader> {
public:
// Create an initialized ColumnReader in *reader.
// This should be a lightweight operation without I/O.
@@ -244,6 +244,8 @@
Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes, RowRanges* row_ranges);
+ int64_t get_metadata_size() const override;
+
private:
int64_t _meta_length;
FieldType _meta_type;
diff --git a/be/src/olap/rowset/segment_v2/index_page.cpp b/be/src/olap/rowset/segment_v2/index_page.cpp
index 9af7047..1b033a9 100644
--- a/be/src/olap/rowset/segment_v2/index_page.cpp
+++ b/be/src/olap/rowset/segment_v2/index_page.cpp
@@ -64,6 +64,10 @@
///////////////////////////////////////////////////////////////////////////////
+int64_t IndexPageReader::get_metadata_size() const {
+ return sizeof(IndexPageReader) + _vl_field_mem_size;
+}
+
Status IndexPageReader::parse(const Slice& body, const IndexPageFooterPB& footer) {
_footer = footer;
size_t num_entries = _footer.num_entries();
@@ -80,8 +84,13 @@
}
_keys.push_back(key);
_values.push_back(value);
+ _vl_field_mem_size += sizeof(char) * key.size;
}
+ _vl_field_mem_size +=
+ _keys.capacity() * sizeof(Slice) + _values.capacity() * sizeof(PagePointer);
+ _vl_field_mem_size += _footer.ByteSizeLong();
+ update_metadata_size();
_parsed = true;
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/index_page.h b/be/src/olap/rowset/segment_v2/index_page.h
index 7b15ef6..0ebf425 100644
--- a/be/src/olap/rowset/segment_v2/index_page.h
+++ b/be/src/olap/rowset/segment_v2/index_page.h
@@ -26,6 +26,7 @@
#include <vector>
#include "common/status.h"
+#include "olap/metadata_adder.h"
#include "olap/rowset/segment_v2/page_pointer.h"
#include "util/faststring.h"
#include "util/slice.h"
@@ -79,7 +80,7 @@
uint32_t _count = 0;
};
-class IndexPageReader {
+class IndexPageReader : public MetadataAdder<IndexPageReader> {
public:
IndexPageReader() : _parsed(false) {}
@@ -110,11 +111,14 @@
void reset();
private:
+ int64_t get_metadata_size() const override;
+
bool _parsed;
IndexPageFooterPB _footer;
std::vector<Slice> _keys;
std::vector<PagePointer> _values;
+ int64_t _vl_field_mem_size {0};
};
class IndexPageIterator {
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 59251b5..cce35d0 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -56,10 +56,12 @@
static bvar::PerSecond<bvar::Adder<uint64_t>> g_index_reader_pk_bytes_per_second(
"doris_pk", "index_reader_pk_pages_per_second", &g_index_reader_pk_pages, 60);
-static bvar::Adder<uint64_t> g_index_reader_memory_bytes("doris_index_reader_memory_bytes");
-
using strings::Substitute;
+int64_t IndexedColumnReader::get_metadata_size() const {
+ return sizeof(IndexedColumnReader) + _meta.ByteSizeLong();
+}
+
Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
_use_page_cache = use_page_cache;
_kept_in_memory = kept_in_memory;
@@ -94,7 +96,7 @@
}
_num_values = _meta.num_values();
- g_index_reader_memory_bytes << sizeof(*this);
+ update_metadata_size();
return Status::OK();
}
@@ -138,9 +140,7 @@
return st;
}
-IndexedColumnReader::~IndexedColumnReader() {
- g_index_reader_memory_bytes << -sizeof(*this);
-}
+IndexedColumnReader::~IndexedColumnReader() = default;
///////////////////////////////////////////////////////////////////////////////
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index d156643..8a57383 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -46,7 +46,7 @@
class EncodingInfo;
// thread-safe reader for IndexedColumn (see comments of `IndexedColumnWriter` to understand what IndexedColumn is)
-class IndexedColumnReader {
+class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
public:
explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta)
: _file_reader(std::move(file_reader)), _meta(meta) {}
@@ -72,6 +72,8 @@
private:
Status load_index_page(const PagePointerPB& pp, PageHandle* handle, IndexPageReader* reader);
+ int64_t get_metadata_size() const override;
+
friend class IndexedColumnIterator;
io::FileReaderSPtr _file_reader;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index ab143da..3756988 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -171,7 +171,8 @@
bool is_empty() const { return (_data_bitmap == nullptr && _null_bitmap == nullptr); }
};
-class InvertedIndexReader : public std::enable_shared_from_this<InvertedIndexReader> {
+class InvertedIndexReader : public std::enable_shared_from_this<InvertedIndexReader>,
+ public MetadataAdder<InvertedIndexReader> {
public:
explicit InvertedIndexReader(
const TabletIndex* index_meta,
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
index 24b2e33..9ee82ba 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
@@ -34,8 +34,6 @@
namespace doris {
-static bvar::Adder<size_t> g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes");
-
namespace segment_v2 {
void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer& data_pp) {
@@ -116,10 +114,6 @@
_ordinals.resize(_num_pages + 1);
_pages.resize(_num_pages);
- g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() * sizeof(ordinal_t) +
- _pages.size() * sizeof(PagePointer) +
- sizeof(OrdinalIndexReader);
-
for (int i = 0; i < _num_pages; i++) {
Slice key = reader.get_key(i);
ordinal_t ordinal = 0;
@@ -132,9 +126,16 @@
}
_ordinals[_num_pages] = _num_values;
+ update_metadata_size();
+
return Status::OK();
}
+int64_t OrdinalIndexReader::get_metadata_size() const {
+ return sizeof(OrdinalIndexReader) + _ordinals.capacity() * sizeof(ordinal_t) +
+ _pages.capacity() * sizeof(PagePointer);
+}
+
OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal) {
int32_t left = 0;
int32_t right = _num_pages - 1;
@@ -156,13 +157,7 @@
return OrdinalPageIndexIterator(this, left);
}
-OrdinalIndexReader::~OrdinalIndexReader() {
- if (_ordinals.size() > 0) {
- g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() * sizeof(ordinal_t) -
- _pages.size() * sizeof(PagePointer) -
- sizeof(OrdinalIndexReader);
- }
-}
+OrdinalIndexReader::~OrdinalIndexReader() = default;
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
index 8f9e0af..1d74cf9 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
@@ -64,7 +64,7 @@
class OrdinalPageIndexIterator;
-class OrdinalIndexReader {
+class OrdinalIndexReader : public MetadataAdder<OrdinalIndexReader> {
public:
explicit OrdinalIndexReader(io::FileReaderSPtr file_reader, ordinal_t num_values,
const OrdinalIndexPB& meta_pb)
@@ -96,6 +96,8 @@
Status _load(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<OrdinalIndexPB> index_meta);
+ int64_t get_metadata_size() const override;
+
private:
friend OrdinalPageIndexIterator;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 0fa6979..68fe319 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -74,7 +74,7 @@
#include "vec/olap/vgeneric_iterators.h"
namespace doris::segment_v2 {
-static bvar::Adder<size_t> g_total_segment_num("doris_total_segment_num");
+
class InvertedIndexIterator;
io::UInt128Wrapper file_cache_key_from_path(const std::string& seg_path) {
@@ -141,18 +141,19 @@
_meta_mem_usage(0),
_rowset_id(rowset_id),
_tablet_schema(std::move(tablet_schema)),
- _idx_file_info(idx_file_info) {
- g_total_segment_num << 1;
-}
+ _idx_file_info(idx_file_info) {}
-Segment::~Segment() {
- g_total_segment_num << -1;
-}
+Segment::~Segment() = default;
io::UInt128Wrapper Segment::file_cache_key(std::string_view rowset_id, uint32_t seg_id) {
return io::BlockFileCache::hash(fmt::format("{}_{}.dat", rowset_id, seg_id));
}
+int64_t Segment::get_metadata_size() const {
+ return sizeof(Segment) + (_footer_pb ? _footer_pb->ByteSizeLong() : 0) +
+ (_pk_index_meta ? _pk_index_meta->ByteSizeLong() : 0);
+}
+
Status Segment::_open() {
_footer_pb = std::make_unique<SegmentFooterPB>();
RETURN_IF_ERROR(_parse_footer(_footer_pb.get()));
@@ -169,6 +170,9 @@
if (_pk_index_meta != nullptr) {
_meta_mem_usage += _pk_index_meta->ByteSizeLong();
}
+
+ update_metadata_size();
+
_meta_mem_usage += sizeof(*this);
_meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index 4184e67..13c8c86 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -78,7 +78,7 @@
// NOTE: This segment is used to a specified TabletSchema, when TabletSchema
// is changed, this segment can not be used any more. For example, after a schema
// change finished, client should disable all cached Segment for old TabletSchema.
-class Segment : public std::enable_shared_from_this<Segment> {
+class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdder<Segment> {
public:
static Status open(io::FileSystemSPtr fs, const std::string& path, uint32_t segment_id,
RowsetId rowset_id, TabletSchemaSPtr tablet_schema,
@@ -92,6 +92,8 @@
~Segment();
+ int64_t get_metadata_size() const override;
+
Status new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options,
std::unique_ptr<RowwiseIterator>* iter);
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
index 991df2f..c2139ff 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
@@ -39,8 +39,6 @@
namespace doris {
struct uint24_t;
-static bvar::Adder<size_t> g_zone_map_memory_bytes("doris_zone_map_memory_bytes");
-
namespace segment_v2 {
template <PrimitiveType Type>
@@ -157,9 +155,6 @@
_page_zone_maps.resize(reader.num_values());
- g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) * _page_zone_maps.size() +
- sizeof(IndexedColumnMetaPB);
-
// read and cache all page zone maps
for (int i = 0; i < reader.num_values(); ++i) {
size_t num_to_read = 1;
@@ -177,18 +172,18 @@
column->get_data_at(0).size)) {
return Status::Corruption("Failed to parse zone map");
}
+ _pb_meta_size += _page_zone_maps[i].ByteSizeLong();
}
+ update_metadata_size();
return Status::OK();
}
-ZoneMapIndexReader::~ZoneMapIndexReader() {
- // Maybe wrong due to load failures.
- if (_page_zone_maps.size() > 0) {
- g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() -
- sizeof(IndexedColumnMetaPB);
- }
+int64_t ZoneMapIndexReader::get_metadata_size() const {
+ return sizeof(ZoneMapIndexReader) + _pb_meta_size;
}
+
+ZoneMapIndexReader::~ZoneMapIndexReader() = default;
#define APPLY_FOR_PRIMITITYPE(M) \
M(TYPE_TINYINT) \
M(TYPE_SMALLINT) \
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h b/be/src/olap/rowset/segment_v2/zone_map_index.h
index 923bd2c..34869bb 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.h
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.h
@@ -143,7 +143,7 @@
uint64_t _estimated_size = 0;
};
-class ZoneMapIndexReader {
+class ZoneMapIndexReader : public MetadataAdder<ZoneMapIndexReader> {
public:
explicit ZoneMapIndexReader(io::FileReaderSPtr file_reader,
const IndexedColumnMetaPB& page_zone_maps)
@@ -163,12 +163,15 @@
private:
Status _load(bool use_page_cache, bool kept_in_memory, std::unique_ptr<IndexedColumnMetaPB>);
+ int64_t get_metadata_size() const override;
+
private:
DorisCallOnce<Status> _load_once;
// TODO: yyq, we shoud remove file_reader from here.
io::FileReaderSPtr _file_reader;
std::unique_ptr<IndexedColumnMetaPB> _page_zone_maps_meta;
std::vector<ZoneMapPB> _page_zone_maps;
+ int64_t _pb_meta_size {0};
};
} // namespace segment_v2
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 6123dc6..12f4b5d 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -345,7 +345,8 @@
}
TabletMeta::TabletMeta(const TabletMeta& b)
- : _table_id(b._table_id),
+ : MetadataAdder(b),
+ _table_id(b._table_id),
_index_id(b._index_id),
_partition_id(b._partition_id),
_tablet_id(b._tablet_id),
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index b99a3ca..3c87fec 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -43,6 +43,7 @@
#include "io/fs/file_system.h"
#include "olap/binlog_config.h"
#include "olap/lru_cache.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/tablet_schema.h"
@@ -90,7 +91,7 @@
// Class encapsulates meta of tablet.
// The concurrency control is handled in Tablet Class, not in this class.
-class TabletMeta {
+class TabletMeta : public MetadataAdder<TabletMeta> {
public:
static TabletMetaSharedPtr create(
const TCreateTabletReq& request, const TabletUid& tablet_uid, uint64_t shard_id,
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 7909e31..c88a23a 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -53,8 +53,6 @@
namespace doris {
-static bvar::Adder<size_t> g_total_tablet_schema_num("doris_total_tablet_schema_num");
-
FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
switch (primitiveType) {
case PrimitiveType::INVALID_TYPE:
@@ -845,12 +843,12 @@
}
}
-TabletSchema::TabletSchema() {
- g_total_tablet_schema_num << 1;
-}
+TabletSchema::TabletSchema() = default;
-TabletSchema::~TabletSchema() {
- g_total_tablet_schema_num << -1;
+TabletSchema::~TabletSchema() = default;
+
+int64_t TabletSchema::get_metadata_size() const {
+ return sizeof(TabletSchema) + _vl_field_mem_size;
}
void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
@@ -974,7 +972,10 @@
++_num_variant_columns;
}
_cols.emplace_back(std::make_shared<TabletColumn>(std::move(column)));
+ _vl_field_mem_size +=
+ sizeof(StringRef) + sizeof(char) * _cols.back()->name().size() + sizeof(size_t);
_field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns);
+ _vl_field_mem_size += sizeof(int32_t) * 2;
_field_id_to_index[_cols.back()->unique_id()] = _num_columns;
_num_columns++;
}
@@ -1018,6 +1019,8 @@
_row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(),
schema.row_store_column_unique_ids().end());
_variant_enable_flatten_nested = schema.variant_enable_flatten_nested();
+ _vl_field_mem_size += _row_store_column_unique_ids.capacity() * sizeof(int32_t);
+ update_metadata_size();
}
void TabletSchema::copy_from(const TabletSchema& tablet_schema) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 3fca9c5..ebe2c63 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -35,6 +35,7 @@
#include "common/consts.h"
#include "common/status.h"
#include "gutil/stringprintf.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/options.h"
#include "runtime/define_primitive_type.h"
@@ -60,7 +61,7 @@
using TabletColumnPtr = std::shared_ptr<TabletColumn>;
-class TabletColumn {
+class TabletColumn : public MetadataAdder<TabletColumn> {
public:
TabletColumn();
TabletColumn(const ColumnPB& column);
@@ -246,7 +247,7 @@
class TabletSchema;
-class TabletIndex {
+class TabletIndex : public MetadataAdder<TabletIndex> {
public:
TabletIndex() = default;
void init_from_thrift(const TOlapTableIndex& index, const TabletSchema& tablet_schema);
@@ -288,7 +289,7 @@
std::map<string, string> _properties;
};
-class TabletSchema {
+class TabletSchema : public MetadataAdder<TabletSchema> {
public:
enum ColumnType { NORMAL = 0, DROPPED = 1, VARIANT = 2 };
// TODO(yingchun): better to make constructor as private to avoid
@@ -500,6 +501,8 @@
const std::vector<int32_t>& row_columns_uids() const { return _row_store_column_unique_ids; }
+ int64_t get_metadata_size() const override;
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -548,6 +551,7 @@
// ATTN: For compability reason empty cids means all columns of tablet schema are encoded to row column
std::vector<int32_t> _row_store_column_unique_ids;
bool _variant_enable_flatten_nested = false;
+ int64_t _vl_field_mem_size {0}; // variable length field
};
bool operator==(const TabletSchema& a, const TabletSchema& b);
