This is an automated email from the ASF dual-hosted git repository. colinlee pushed a commit to branch colin_fix_deserialize in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 7dd882c34b9c2c4151a1b415bf1525d9b63aba80 Author: ColinLee <[email protected]> AuthorDate: Wed Apr 9 01:41:37 2025 +0800 fix deserialize. --- cpp/src/common/tsfile_common.cc | 4 ++-- cpp/src/common/tsfile_common.h | 19 ++++++--------- cpp/src/file/tsfile_io_reader.cc | 43 +++++++++++++++++----------------- cpp/src/file/tsfile_io_reader.h | 11 +++++---- cpp/src/reader/device_meta_iterator.cc | 4 ++-- cpp/test/common/tsfile_common_test.cc | 28 ++++++++++------------ 6 files changed, 50 insertions(+), 59 deletions(-) diff --git a/cpp/src/common/tsfile_common.cc b/cpp/src/common/tsfile_common.cc index 89a007ba..17c3d479 100644 --- a/cpp/src/common/tsfile_common.cc +++ b/cpp/src/common/tsfile_common.cc @@ -260,7 +260,7 @@ int TsFileMeta::deserialize_from(common::ByteStream &in) { /* ================ MetaIndexNode ================ */ int MetaIndexNode::binary_search_children(std::shared_ptr<IComparable> key, bool exact_search, - IMetaIndexEntry &ret_index_entry, + std::shared_ptr<IMetaIndexEntry> &ret_index_entry, int64_t &ret_end_offset) { #if DEBUG_SE std::cout << "MetaIndexNode::binary_search_children start, name=" << key @@ -311,7 +311,7 @@ int MetaIndexNode::binary_search_children(std::shared_ptr<IComparable> key, bool return E_NOT_EXIST; } } - ret_index_entry.clone(children_[l], pa_); + ret_index_entry = children_[l]->clone(pa_); if (l == (int)children_.size() - 1) { ret_end_offset = this->end_offset_; } else { diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h index 9fca0168..878fc4d2 100644 --- a/cpp/src/common/tsfile_common.h +++ b/cpp/src/common/tsfile_common.h @@ -738,15 +738,14 @@ struct IMetaIndexEntry { common::PageArena *pa) { return common::E_NOT_SUPPORT; } - virtual int64_t get_offset() const { return 0; } + virtual int64_t get_offset() const = 0; virtual bool is_device_level() const { return false; } virtual std::shared_ptr<IComparable> get_compare_key() const { return std::shared_ptr<IComparable>(); } virtual common::String get_name() const { return {}; } virtual std::shared_ptr<IDeviceID> get_device_id() const { return nullptr; } - virtual void clone(std::shared_ptr<IMetaIndexEntry> entry, - common::PageArena *pa) {} + virtual std::shared_ptr<IMetaIndexEntry> clone(common::PageArena *pa) = 0; #ifndef NDEBUG virtual void print(std::ostream &os) const {} friend std::ostream &operator<<(std::ostream &os, @@ -801,10 +800,8 @@ struct DeviceMetaIndexEntry : IMetaIndexEntry { std::shared_ptr<IDeviceID> get_device_id() const override { return device_id_; } - void clone(std::shared_ptr<IMetaIndexEntry> entry, - common::PageArena *pa) override { - offset_ = entry->get_offset(); - device_id_ = entry->get_device_id(); + std::shared_ptr<IMetaIndexEntry> clone(common::PageArena *pa) override { + return std::make_shared<DeviceMetaIndexEntry>(device_id_, offset_); } #ifndef NDEBUG void print(std::ostream &os) const override { @@ -862,10 +859,8 @@ struct MeasurementMetaIndexEntry : IMetaIndexEntry { std::shared_ptr<IDeviceID> get_device_id() const override { return nullptr; } - void clone(std::shared_ptr<IMetaIndexEntry> entry, - common::PageArena *pa) override { - offset_ = entry->get_offset(); - name_.dup_from(entry->get_name(), *pa); + std::shared_ptr<IMetaIndexEntry> clone(common::PageArena *pa) override { + return std::make_shared<MeasurementMetaIndexEntry>(name_, offset_, *pa); } #ifndef NDEBUG void print(std::ostream &os) const override { @@ -915,7 +910,7 @@ struct MetaIndexNode { int binary_search_children(std::shared_ptr<IComparable> key, bool exact_search, - IMetaIndexEntry &ret_index_entry, + std::shared_ptr<IMetaIndexEntry> &ret_index_entry, int64_t &ret_end_offset); int serialize_to(common::ByteStream &out) { diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index 11113fc0..a2566959 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -93,7 +93,7 @@ int TsFileIOReader::get_device_timeseries_meta_without_chunk_meta( PageArena &pa) { int ret = E_OK; load_tsfile_meta_if_necessary(); - DeviceMetaIndexEntry meta_index_entry; + std::shared_ptr<IMetaIndexEntry> meta_index_entry; int64_t end_offset; std::vector<std::pair<std::shared_ptr<IMetaIndexEntry>, int64_t> > meta_index_entry_list; @@ -101,7 +101,7 @@ int TsFileIOReader::get_device_timeseries_meta_without_chunk_meta( load_device_index_entry(std::make_shared<DeviceIDComparable>(device_id), meta_index_entry, end_offset))) { } else if (RET_FAIL(load_all_measurement_index_entry( - meta_index_entry.offset_, end_offset, pa, + meta_index_entry->get_offset(), end_offset, pa, meta_index_entry_list))) { } else if (RET_FAIL(do_load_all_timeseries_index(meta_index_entry_list, pa, timeseries_indexs))) { @@ -215,20 +215,20 @@ int TsFileIOReader::load_timeseries_index_for_ssi( std::shared_ptr<IDeviceID> device_id, const std::string &measurement_name, TsFileSeriesScanIterator *&ssi) { int ret = E_OK; - DeviceMetaIndexEntry device_index_entry; + std::shared_ptr<IMetaIndexEntry> device_index_entry; int64_t device_ie_end_offset = 0; - MeasurementMetaIndexEntry measurement_index_entry; + std::shared_ptr<IMetaIndexEntry> measurement_index_entry; int64_t measurement_ie_end_offset = 0; // bool is_aligned = false; if (RET_FAIL(load_device_index_entry( std::make_shared<DeviceIDComparable>(device_id), device_index_entry, device_ie_end_offset))) { } else if (RET_FAIL(load_measurement_index_entry( - measurement_name, device_index_entry.offset_, + measurement_name, device_index_entry->get_offset(), device_ie_end_offset, measurement_index_entry, measurement_ie_end_offset))) { } else if (RET_FAIL(do_load_timeseries_index( - measurement_name, measurement_index_entry.offset_, + measurement_name, measurement_index_entry->get_offset(), measurement_ie_end_offset, ssi->timeseries_index_pa_, ssi->itimeseries_index_))) { } else { @@ -249,7 +249,7 @@ int TsFileIOReader::load_timeseries_index_for_ssi( int TsFileIOReader::load_device_index_entry( std::shared_ptr<IComparable> device_name, - IMetaIndexEntry &device_index_entry, int64_t &end_offset) { + std::shared_ptr<IMetaIndexEntry> &device_index_entry, int64_t &end_offset) { int ret = E_OK; std::shared_ptr<DeviceIDComparable> device_id_comparable = std::dynamic_pointer_cast<DeviceIDComparable>(device_name); @@ -281,7 +281,7 @@ int TsFileIOReader::load_device_index_entry( int TsFileIOReader::load_measurement_index_entry( const std::string &measurement_name_str, int64_t start_offset, - int64_t end_offset, IMetaIndexEntry &ret_measurement_index_entry, + int64_t end_offset, std::shared_ptr<IMetaIndexEntry> &ret_measurement_index_entry, int64_t &ret_end_offset) { #if DEBUG_SE std::cout << "load_measurement_index_entry: measurement_name_str=" @@ -380,7 +380,8 @@ int TsFileIOReader::load_all_measurement_index_entry( int TsFileIOReader::read_device_meta_index(int32_t start_offset, int32_t end_offset, common::PageArena &pa, - MetaIndexNode *&device_meta_index) { + MetaIndexNode *&device_meta_index, + bool leaf) { int ret = E_OK; ASSERT(start_offset < end_offset); const int32_t read_size = (int32_t)(end_offset - start_offset); @@ -393,9 +394,8 @@ int TsFileIOReader::read_device_meta_index(int32_t start_offset, device_meta_index = new (m_idx_node_buf) MetaIndexNode(&pa); if (RET_FAIL(read_file_->read(start_offset, data_buf, read_size, ret_read_len))) { - } else if (RET_FAIL( - device_meta_index->deserialize_from(data_buf, read_size))) { } + ret = device_meta_index->device_deserialize_from(data_buf, read_size); return ret; } @@ -404,9 +404,9 @@ int TsFileIOReader::get_timeseries_indexes(std::shared_ptr<IDeviceID> device_id, std::vector<ITimeseriesIndex *> ×eries_indexs, common::PageArena &pa) { int ret = E_OK; - DeviceMetaIndexEntry device_index_entry; + std::shared_ptr<IMetaIndexEntry> device_index_entry; int64_t device_ie_end_offset = 0; - MeasurementMetaIndexEntry measurement_index_entry; + std::shared_ptr<IMetaIndexEntry> measurement_index_entry; int64_t measurement_ie_end_offset = 0; if (RET_FAIL(load_device_index_entry( std::make_shared<DeviceIDComparable>(device_id), device_index_entry, @@ -416,11 +416,11 @@ int TsFileIOReader::get_timeseries_indexes(std::shared_ptr<IDeviceID> device_id, int64_t idx = 0; for (const auto &measurement_name : measurement_names) { if (RET_FAIL(load_measurement_index_entry( - measurement_name, device_index_entry.offset_, + measurement_name, device_index_entry->get_offset(), device_ie_end_offset, measurement_index_entry, measurement_ie_end_offset))) { } else if (RET_FAIL(do_load_timeseries_index( - measurement_name, measurement_index_entry.offset_, + measurement_name, measurement_index_entry->get_offset(), measurement_ie_end_offset, pa, timeseries_indexs[idx++]))) { } @@ -435,7 +435,7 @@ int TsFileIOReader::get_timeseries_indexes(std::shared_ptr<IDeviceID> device_id, int TsFileIOReader::search_from_leaf_node( std::shared_ptr<IComparable> target_name, std::shared_ptr<MetaIndexNode> index_node, - IMetaIndexEntry &ret_index_entry, int64_t &ret_end_offset) { + std::shared_ptr<IMetaIndexEntry> &ret_index_entry, int64_t &ret_end_offset) { int ret = E_OK; ret = index_node->binary_search_children(target_name, true, ret_index_entry, ret_end_offset); @@ -444,15 +444,14 @@ int TsFileIOReader::search_from_leaf_node( int TsFileIOReader::search_from_internal_node( std::shared_ptr<IComparable> target_name, - std::shared_ptr<MetaIndexNode> index_node, IMetaIndexEntry &ret_index_entry, + std::shared_ptr<MetaIndexNode> index_node, std::shared_ptr<IMetaIndexEntry> &ret_index_entry, int64_t &ret_end_offset) { int ret = E_OK; - IMetaIndexEntry index_entry; + std::shared_ptr<IMetaIndexEntry> index_entry; int64_t end_offset = 0; ASSERT(index_node->node_type_ == INTERNAL_MEASUREMENT || index_node->node_type_ == INTERNAL_DEVICE); - if (RET_FAIL(index_node->binary_search_children( target_name, /*exact=*/false, index_entry, end_offset))) { return ret; @@ -460,7 +459,7 @@ int TsFileIOReader::search_from_internal_node( while (IS_SUCC(ret)) { // reader next level index node - const int read_size = end_offset - index_entry.get_offset(); + const int read_size = end_offset - index_entry->get_offset(); #if DEBUG_SE std::cout << "search_from_internal_node, end_offset=" << end_offset << ", index_entry.offset_=" << index_entry.get_offset() @@ -476,11 +475,11 @@ int TsFileIOReader::search_from_internal_node( MetaIndexNode *cur_level_index_node = new(buf) MetaIndexNode(&cur_level_index_node_pa); int32_t ret_read_len = 0; - if (RET_FAIL(read_file_->read(index_entry.get_offset(), data_buf, read_size, + if (RET_FAIL(read_file_->read(index_entry->get_offset(), data_buf, read_size, ret_read_len))) { } else if (read_size != ret_read_len) { ret = E_TSFILE_CORRUPTED; - } else if (RET_FAIL(cur_level_index_node->deserialize_from( + } else if (RET_FAIL(cur_level_index_node->device_deserialize_from( data_buf, read_size))) { } else { if (cur_level_index_node->node_type_ == LEAF_DEVICE) { diff --git a/cpp/src/file/tsfile_io_reader.h b/cpp/src/file/tsfile_io_reader.h index d32f690f..bd4a293a 100644 --- a/cpp/src/file/tsfile_io_reader.h +++ b/cpp/src/file/tsfile_io_reader.h @@ -77,7 +77,8 @@ class TsFileIOReader { std::vector<ChunkMeta *> &chunk_meta_list); int read_device_meta_index(int32_t start_offset, int32_t end_offset, common::PageArena &pa, - MetaIndexNode *&device_meta_index); + MetaIndexNode *&device_meta_index, + bool leaf); int get_timeseries_indexes( std::shared_ptr<IDeviceID> device_id, const std::unordered_set<std::string> &measurement_names, @@ -92,12 +93,12 @@ class TsFileIOReader { int load_tsfile_meta_if_necessary(); int load_device_index_entry(std::shared_ptr<IComparable> target_name, - IMetaIndexEntry &device_index_entry, + std::shared_ptr<IMetaIndexEntry> &device_index_entry, int64_t &end_offset); int load_measurement_index_entry( const std::string &measurement_name, int64_t start_offset, - int64_t end_offset, IMetaIndexEntry &ret_measurement_index_entry, + int64_t end_offset, std::shared_ptr<IMetaIndexEntry> &ret_measurement_index_entry, int64_t &ret_end_offset); int load_all_measurement_index_entry( @@ -122,12 +123,12 @@ class TsFileIOReader { int search_from_leaf_node(std::shared_ptr<IComparable> target_name, std::shared_ptr<MetaIndexNode> index_node, - IMetaIndexEntry &ret_index_entry, + std::shared_ptr<IMetaIndexEntry> &ret_index_entry, int64_t &ret_end_offset); int search_from_internal_node(std::shared_ptr<IComparable> target_name, std::shared_ptr<MetaIndexNode> index_node, - IMetaIndexEntry &ret_index_entry, + std::shared_ptr<IMetaIndexEntry> &ret_index_entry, int64_t &ret_end_offset); bool filter_stasify(ITimeseriesIndex *ts_index, Filter *time_filter); diff --git a/cpp/src/reader/device_meta_iterator.cc b/cpp/src/reader/device_meta_iterator.cc index 90bcdc10..d4512263 100644 --- a/cpp/src/reader/device_meta_iterator.cc +++ b/cpp/src/reader/device_meta_iterator.cc @@ -78,7 +78,7 @@ int DeviceMetaIterator::load_leaf_device(MetaIndexNode* meta_index_node) { : meta_index_node->end_offset_; MetaIndexNode* child_node = nullptr; if (RET_FAIL(io_reader_->read_device_meta_index( - start_offset, end_offset, pa_, child_node))) { + start_offset, end_offset, pa_, child_node, false))) { return ret; } else { result_cache_.push( @@ -101,7 +101,7 @@ int DeviceMetaIterator::load_internal_node(MetaIndexNode* meta_index_node) { MetaIndexNode* child_node = nullptr; if (RET_FAIL(io_reader_->read_device_meta_index( - start_offset, end_offset, pa_, child_node))) { + start_offset, end_offset, pa_, child_node, false))) { return ret; } else { meta_index_nodes_.push(child_node); diff --git a/cpp/test/common/tsfile_common_test.cc b/cpp/test/common/tsfile_common_test.cc index e5eebd49..dc9cebf0 100644 --- a/cpp/test/common/tsfile_common_test.cc +++ b/cpp/test/common/tsfile_common_test.cc @@ -368,54 +368,50 @@ protected: TEST_F(MetaIndexNodeSearchTest, ExactSearchFound) { const std::string ret_entry_name(""); - std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared< - MeasurementMetaIndexEntry>(); - ret_entry->init(ret_entry_name, 0, arena_); + std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared< + MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_); int64_t ret_offset = 0; int result = node_.binary_search_children( std::make_shared<StringComparable>("banana"), - true, *ret_entry, ret_offset); + true, ret_entry, ret_offset); ASSERT_EQ(result, 0); ASSERT_EQ(ret_offset, 30); } TEST_F(MetaIndexNodeSearchTest, ExactSearchNotFound) { const std::string ret_entry_name(""); - std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared< - MeasurementMetaIndexEntry>(); - ret_entry->init(ret_entry_name, 0, arena_); + std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared< + MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_); int64_t ret_offset = 0; char search_name[] = "grape"; int result = node_.binary_search_children( std::make_shared<StringComparable>(search_name), - true, *ret_entry, ret_offset); + true, ret_entry, ret_offset); ASSERT_EQ(result, common::E_NOT_EXIST); } TEST_F(MetaIndexNodeSearchTest, NonExactSearchFound) { const std::string ret_entry_name(""); - std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared< - MeasurementMetaIndexEntry>(); - ret_entry->init(ret_entry_name, 0, arena_); + std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared< + MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_); int64_t ret_offset = 0; char search_name[] = "blueberry"; int result = node_.binary_search_children( std::make_shared<StringComparable>(search_name), - false, *ret_entry, ret_offset); + false, ret_entry, ret_offset); ASSERT_EQ(result, 0); ASSERT_EQ(ret_offset, 30); } TEST_F(MetaIndexNodeSearchTest, NonExactSearchNotFound) { const std::string ret_entry_name(""); - std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared< - MeasurementMetaIndexEntry>(); - ret_entry->init(ret_entry_name, 0, arena_); + std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared< + MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_); int64_t ret_offset = 0; char search_name[] = "aardvark"; int result = node_.binary_search_children( std::make_shared<StringComparable>(search_name), - false, *ret_entry, ret_offset); + false, ret_entry, ret_offset); ASSERT_EQ(result, common::E_NOT_EXIST); }
