This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 1bc0a652 fix deserialize. (#456)
1bc0a652 is described below
commit 1bc0a652f250cbd90705ad786ecf7d5efadf6b6e
Author: Colin Lee <[email protected]>
AuthorDate: Tue Apr 22 14:34:44 2025 +0800
fix deserialize. (#456)
---
cpp/src/common/tsfile_common.cc | 4 ++--
cpp/src/common/tsfile_common.h | 19 ++++++---------
cpp/src/file/tsfile_io_reader.cc | 43 +++++++++++++++++-----------------
cpp/src/file/tsfile_io_reader.h | 11 +++++----
cpp/src/reader/device_meta_iterator.cc | 4 ++--
cpp/test/common/tsfile_common_test.cc | 28 ++++++++++------------
6 files changed, 50 insertions(+), 59 deletions(-)
diff --git a/cpp/src/common/tsfile_common.cc b/cpp/src/common/tsfile_common.cc
index 89a007ba..17c3d479 100644
--- a/cpp/src/common/tsfile_common.cc
+++ b/cpp/src/common/tsfile_common.cc
@@ -260,7 +260,7 @@ int TsFileMeta::deserialize_from(common::ByteStream &in) {
/* ================ MetaIndexNode ================ */
int MetaIndexNode::binary_search_children(std::shared_ptr<IComparable> key,
bool exact_search,
- IMetaIndexEntry &ret_index_entry,
+ std::shared_ptr<IMetaIndexEntry>
&ret_index_entry,
int64_t &ret_end_offset) {
#if DEBUG_SE
std::cout << "MetaIndexNode::binary_search_children start, name=" << key
@@ -311,7 +311,7 @@ int
MetaIndexNode::binary_search_children(std::shared_ptr<IComparable> key, bool
return E_NOT_EXIST;
}
}
- ret_index_entry.clone(children_[l], pa_);
+ ret_index_entry = children_[l]->clone(pa_);
if (l == (int)children_.size() - 1) {
ret_end_offset = this->end_offset_;
} else {
diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h
index 9fca0168..878fc4d2 100644
--- a/cpp/src/common/tsfile_common.h
+++ b/cpp/src/common/tsfile_common.h
@@ -738,15 +738,14 @@ struct IMetaIndexEntry {
common::PageArena *pa) {
return common::E_NOT_SUPPORT;
}
- virtual int64_t get_offset() const { return 0; }
+ virtual int64_t get_offset() const = 0;
virtual bool is_device_level() const { return false; }
virtual std::shared_ptr<IComparable> get_compare_key() const {
return std::shared_ptr<IComparable>();
}
virtual common::String get_name() const { return {}; }
virtual std::shared_ptr<IDeviceID> get_device_id() const { return nullptr;
}
- virtual void clone(std::shared_ptr<IMetaIndexEntry> entry,
- common::PageArena *pa) {}
+ virtual std::shared_ptr<IMetaIndexEntry> clone(common::PageArena *pa) = 0;
#ifndef NDEBUG
virtual void print(std::ostream &os) const {}
friend std::ostream &operator<<(std::ostream &os,
@@ -801,10 +800,8 @@ struct DeviceMetaIndexEntry : IMetaIndexEntry {
std::shared_ptr<IDeviceID> get_device_id() const override {
return device_id_;
}
- void clone(std::shared_ptr<IMetaIndexEntry> entry,
- common::PageArena *pa) override {
- offset_ = entry->get_offset();
- device_id_ = entry->get_device_id();
+ std::shared_ptr<IMetaIndexEntry> clone(common::PageArena *pa) override {
+ return std::make_shared<DeviceMetaIndexEntry>(device_id_, offset_);
}
#ifndef NDEBUG
void print(std::ostream &os) const override {
@@ -862,10 +859,8 @@ struct MeasurementMetaIndexEntry : IMetaIndexEntry {
std::shared_ptr<IDeviceID> get_device_id() const override {
return nullptr;
}
- void clone(std::shared_ptr<IMetaIndexEntry> entry,
- common::PageArena *pa) override {
- offset_ = entry->get_offset();
- name_.dup_from(entry->get_name(), *pa);
+ std::shared_ptr<IMetaIndexEntry> clone(common::PageArena *pa) override {
+ return std::make_shared<MeasurementMetaIndexEntry>(name_, offset_,
*pa);
}
#ifndef NDEBUG
void print(std::ostream &os) const override {
@@ -915,7 +910,7 @@ struct MetaIndexNode {
int binary_search_children(std::shared_ptr<IComparable> key,
bool exact_search,
- IMetaIndexEntry &ret_index_entry,
+ std::shared_ptr<IMetaIndexEntry>
&ret_index_entry,
int64_t &ret_end_offset);
int serialize_to(common::ByteStream &out) {
diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc
index 11113fc0..a2566959 100644
--- a/cpp/src/file/tsfile_io_reader.cc
+++ b/cpp/src/file/tsfile_io_reader.cc
@@ -93,7 +93,7 @@ int
TsFileIOReader::get_device_timeseries_meta_without_chunk_meta(
PageArena &pa) {
int ret = E_OK;
load_tsfile_meta_if_necessary();
- DeviceMetaIndexEntry meta_index_entry;
+ std::shared_ptr<IMetaIndexEntry> meta_index_entry;
int64_t end_offset;
std::vector<std::pair<std::shared_ptr<IMetaIndexEntry>, int64_t> >
meta_index_entry_list;
@@ -101,7 +101,7 @@ int
TsFileIOReader::get_device_timeseries_meta_without_chunk_meta(
load_device_index_entry(std::make_shared<DeviceIDComparable>(device_id),
meta_index_entry, end_offset))) {
} else if (RET_FAIL(load_all_measurement_index_entry(
- meta_index_entry.offset_, end_offset, pa,
+ meta_index_entry->get_offset(), end_offset, pa,
meta_index_entry_list))) {
} else if (RET_FAIL(do_load_all_timeseries_index(meta_index_entry_list, pa,
timeseries_indexs))) {
@@ -215,20 +215,20 @@ int TsFileIOReader::load_timeseries_index_for_ssi(
std::shared_ptr<IDeviceID> device_id, const std::string &measurement_name,
TsFileSeriesScanIterator *&ssi) {
int ret = E_OK;
- DeviceMetaIndexEntry device_index_entry;
+ std::shared_ptr<IMetaIndexEntry> device_index_entry;
int64_t device_ie_end_offset = 0;
- MeasurementMetaIndexEntry measurement_index_entry;
+ std::shared_ptr<IMetaIndexEntry> measurement_index_entry;
int64_t measurement_ie_end_offset = 0;
// bool is_aligned = false;
if (RET_FAIL(load_device_index_entry(
std::make_shared<DeviceIDComparable>(device_id), device_index_entry,
device_ie_end_offset))) {
} else if (RET_FAIL(load_measurement_index_entry(
- measurement_name, device_index_entry.offset_,
+ measurement_name, device_index_entry->get_offset(),
device_ie_end_offset, measurement_index_entry,
measurement_ie_end_offset))) {
} else if (RET_FAIL(do_load_timeseries_index(
- measurement_name, measurement_index_entry.offset_,
+ measurement_name, measurement_index_entry->get_offset(),
measurement_ie_end_offset, ssi->timeseries_index_pa_,
ssi->itimeseries_index_))) {
} else {
@@ -249,7 +249,7 @@ int TsFileIOReader::load_timeseries_index_for_ssi(
int TsFileIOReader::load_device_index_entry(
std::shared_ptr<IComparable> device_name,
- IMetaIndexEntry &device_index_entry, int64_t &end_offset) {
+ std::shared_ptr<IMetaIndexEntry> &device_index_entry, int64_t &end_offset)
{
int ret = E_OK;
std::shared_ptr<DeviceIDComparable> device_id_comparable =
std::dynamic_pointer_cast<DeviceIDComparable>(device_name);
@@ -281,7 +281,7 @@ int TsFileIOReader::load_device_index_entry(
int TsFileIOReader::load_measurement_index_entry(
const std::string &measurement_name_str, int64_t start_offset,
- int64_t end_offset, IMetaIndexEntry &ret_measurement_index_entry,
+ int64_t end_offset, std::shared_ptr<IMetaIndexEntry>
&ret_measurement_index_entry,
int64_t &ret_end_offset) {
#if DEBUG_SE
std::cout << "load_measurement_index_entry: measurement_name_str="
@@ -380,7 +380,8 @@ int TsFileIOReader::load_all_measurement_index_entry(
int TsFileIOReader::read_device_meta_index(int32_t start_offset,
int32_t end_offset,
common::PageArena &pa,
- MetaIndexNode *&device_meta_index) {
+ MetaIndexNode *&device_meta_index,
+ bool leaf) {
int ret = E_OK;
ASSERT(start_offset < end_offset);
const int32_t read_size = (int32_t)(end_offset - start_offset);
@@ -393,9 +394,8 @@ int TsFileIOReader::read_device_meta_index(int32_t
start_offset,
device_meta_index = new (m_idx_node_buf) MetaIndexNode(&pa);
if (RET_FAIL(read_file_->read(start_offset, data_buf, read_size,
ret_read_len))) {
- } else if (RET_FAIL(
- device_meta_index->deserialize_from(data_buf, read_size))) {
}
+ ret = device_meta_index->device_deserialize_from(data_buf, read_size);
return ret;
}
@@ -404,9 +404,9 @@ int
TsFileIOReader::get_timeseries_indexes(std::shared_ptr<IDeviceID> device_id,
std::vector<ITimeseriesIndex *>
×eries_indexs,
common::PageArena &pa) {
int ret = E_OK;
- DeviceMetaIndexEntry device_index_entry;
+ std::shared_ptr<IMetaIndexEntry> device_index_entry;
int64_t device_ie_end_offset = 0;
- MeasurementMetaIndexEntry measurement_index_entry;
+ std::shared_ptr<IMetaIndexEntry> measurement_index_entry;
int64_t measurement_ie_end_offset = 0;
if (RET_FAIL(load_device_index_entry(
std::make_shared<DeviceIDComparable>(device_id), device_index_entry,
@@ -416,11 +416,11 @@ int
TsFileIOReader::get_timeseries_indexes(std::shared_ptr<IDeviceID> device_id,
int64_t idx = 0;
for (const auto &measurement_name : measurement_names) {
if (RET_FAIL(load_measurement_index_entry(
- measurement_name, device_index_entry.offset_,
+ measurement_name, device_index_entry->get_offset(),
device_ie_end_offset, measurement_index_entry,
measurement_ie_end_offset))) {
} else if (RET_FAIL(do_load_timeseries_index(
- measurement_name, measurement_index_entry.offset_,
+ measurement_name, measurement_index_entry->get_offset(),
measurement_ie_end_offset, pa,
timeseries_indexs[idx++]))) {
}
@@ -435,7 +435,7 @@ int
TsFileIOReader::get_timeseries_indexes(std::shared_ptr<IDeviceID> device_id,
int TsFileIOReader::search_from_leaf_node(
std::shared_ptr<IComparable> target_name,
std::shared_ptr<MetaIndexNode> index_node,
- IMetaIndexEntry &ret_index_entry, int64_t &ret_end_offset) {
+ std::shared_ptr<IMetaIndexEntry> &ret_index_entry, int64_t
&ret_end_offset) {
int ret = E_OK;
ret = index_node->binary_search_children(target_name, true, ret_index_entry,
ret_end_offset);
@@ -444,15 +444,14 @@ int TsFileIOReader::search_from_leaf_node(
int TsFileIOReader::search_from_internal_node(
std::shared_ptr<IComparable> target_name,
- std::shared_ptr<MetaIndexNode> index_node, IMetaIndexEntry
&ret_index_entry,
+ std::shared_ptr<MetaIndexNode> index_node,
std::shared_ptr<IMetaIndexEntry> &ret_index_entry,
int64_t &ret_end_offset) {
int ret = E_OK;
- IMetaIndexEntry index_entry;
+ std::shared_ptr<IMetaIndexEntry> index_entry;
int64_t end_offset = 0;
ASSERT(index_node->node_type_ == INTERNAL_MEASUREMENT ||
index_node->node_type_ == INTERNAL_DEVICE);
-
if (RET_FAIL(index_node->binary_search_children(
target_name, /*exact=*/false, index_entry, end_offset))) {
return ret;
@@ -460,7 +459,7 @@ int TsFileIOReader::search_from_internal_node(
while (IS_SUCC(ret)) {
// reader next level index node
- const int read_size = end_offset - index_entry.get_offset();
+ const int read_size = end_offset - index_entry->get_offset();
#if DEBUG_SE
std::cout << "search_from_internal_node, end_offset=" << end_offset
<< ", index_entry.offset_=" << index_entry.get_offset()
@@ -476,11 +475,11 @@ int TsFileIOReader::search_from_internal_node(
MetaIndexNode *cur_level_index_node =
new(buf) MetaIndexNode(&cur_level_index_node_pa);
int32_t ret_read_len = 0;
- if (RET_FAIL(read_file_->read(index_entry.get_offset(), data_buf,
read_size,
+ if (RET_FAIL(read_file_->read(index_entry->get_offset(), data_buf,
read_size,
ret_read_len))) {
} else if (read_size != ret_read_len) {
ret = E_TSFILE_CORRUPTED;
- } else if (RET_FAIL(cur_level_index_node->deserialize_from(
+ } else if (RET_FAIL(cur_level_index_node->device_deserialize_from(
data_buf, read_size))) {
} else {
if (cur_level_index_node->node_type_ == LEAF_DEVICE) {
diff --git a/cpp/src/file/tsfile_io_reader.h b/cpp/src/file/tsfile_io_reader.h
index d32f690f..bd4a293a 100644
--- a/cpp/src/file/tsfile_io_reader.h
+++ b/cpp/src/file/tsfile_io_reader.h
@@ -77,7 +77,8 @@ class TsFileIOReader {
std::vector<ChunkMeta *> &chunk_meta_list);
int read_device_meta_index(int32_t start_offset, int32_t end_offset,
common::PageArena &pa,
- MetaIndexNode *&device_meta_index);
+ MetaIndexNode *&device_meta_index,
+ bool leaf);
int get_timeseries_indexes(
std::shared_ptr<IDeviceID> device_id,
const std::unordered_set<std::string> &measurement_names,
@@ -92,12 +93,12 @@ class TsFileIOReader {
int load_tsfile_meta_if_necessary();
int load_device_index_entry(std::shared_ptr<IComparable> target_name,
- IMetaIndexEntry &device_index_entry,
+ std::shared_ptr<IMetaIndexEntry>
&device_index_entry,
int64_t &end_offset);
int load_measurement_index_entry(
const std::string &measurement_name, int64_t start_offset,
- int64_t end_offset, IMetaIndexEntry &ret_measurement_index_entry,
+ int64_t end_offset, std::shared_ptr<IMetaIndexEntry>
&ret_measurement_index_entry,
int64_t &ret_end_offset);
int load_all_measurement_index_entry(
@@ -122,12 +123,12 @@ class TsFileIOReader {
int search_from_leaf_node(std::shared_ptr<IComparable> target_name,
std::shared_ptr<MetaIndexNode> index_node,
- IMetaIndexEntry &ret_index_entry,
+ std::shared_ptr<IMetaIndexEntry>
&ret_index_entry,
int64_t &ret_end_offset);
int search_from_internal_node(std::shared_ptr<IComparable> target_name,
std::shared_ptr<MetaIndexNode> index_node,
- IMetaIndexEntry &ret_index_entry,
+ std::shared_ptr<IMetaIndexEntry>
&ret_index_entry,
int64_t &ret_end_offset);
bool filter_stasify(ITimeseriesIndex *ts_index, Filter *time_filter);
diff --git a/cpp/src/reader/device_meta_iterator.cc
b/cpp/src/reader/device_meta_iterator.cc
index 90bcdc10..d4512263 100644
--- a/cpp/src/reader/device_meta_iterator.cc
+++ b/cpp/src/reader/device_meta_iterator.cc
@@ -78,7 +78,7 @@ int DeviceMetaIterator::load_leaf_device(MetaIndexNode*
meta_index_node) {
: meta_index_node->end_offset_;
MetaIndexNode* child_node = nullptr;
if (RET_FAIL(io_reader_->read_device_meta_index(
- start_offset, end_offset, pa_, child_node))) {
+ start_offset, end_offset, pa_, child_node, false))) {
return ret;
} else {
result_cache_.push(
@@ -101,7 +101,7 @@ int DeviceMetaIterator::load_internal_node(MetaIndexNode*
meta_index_node) {
MetaIndexNode* child_node = nullptr;
if (RET_FAIL(io_reader_->read_device_meta_index(
- start_offset, end_offset, pa_, child_node))) {
+ start_offset, end_offset, pa_, child_node, false))) {
return ret;
} else {
meta_index_nodes_.push(child_node);
diff --git a/cpp/test/common/tsfile_common_test.cc
b/cpp/test/common/tsfile_common_test.cc
index e5eebd49..dc9cebf0 100644
--- a/cpp/test/common/tsfile_common_test.cc
+++ b/cpp/test/common/tsfile_common_test.cc
@@ -368,54 +368,50 @@ protected:
TEST_F(MetaIndexNodeSearchTest, ExactSearchFound) {
const std::string ret_entry_name("");
- std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared<
- MeasurementMetaIndexEntry>();
- ret_entry->init(ret_entry_name, 0, arena_);
+ std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared<
+ MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_);
int64_t ret_offset = 0;
int result = node_.binary_search_children(
std::make_shared<StringComparable>("banana"),
- true, *ret_entry, ret_offset);
+ true, ret_entry, ret_offset);
ASSERT_EQ(result, 0);
ASSERT_EQ(ret_offset, 30);
}
TEST_F(MetaIndexNodeSearchTest, ExactSearchNotFound) {
const std::string ret_entry_name("");
- std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared<
- MeasurementMetaIndexEntry>();
- ret_entry->init(ret_entry_name, 0, arena_);
+ std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared<
+ MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_);
int64_t ret_offset = 0;
char search_name[] = "grape";
int result = node_.binary_search_children(
std::make_shared<StringComparable>(search_name),
- true, *ret_entry, ret_offset);
+ true, ret_entry, ret_offset);
ASSERT_EQ(result, common::E_NOT_EXIST);
}
TEST_F(MetaIndexNodeSearchTest, NonExactSearchFound) {
const std::string ret_entry_name("");
- std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared<
- MeasurementMetaIndexEntry>();
- ret_entry->init(ret_entry_name, 0, arena_);
+ std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared<
+ MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_);
int64_t ret_offset = 0;
char search_name[] = "blueberry";
int result = node_.binary_search_children(
std::make_shared<StringComparable>(search_name),
- false, *ret_entry, ret_offset);
+ false, ret_entry, ret_offset);
ASSERT_EQ(result, 0);
ASSERT_EQ(ret_offset, 30);
}
TEST_F(MetaIndexNodeSearchTest, NonExactSearchNotFound) {
const std::string ret_entry_name("");
- std::shared_ptr<MeasurementMetaIndexEntry> ret_entry = std::make_shared<
- MeasurementMetaIndexEntry>();
- ret_entry->init(ret_entry_name, 0, arena_);
+ std::shared_ptr<IMetaIndexEntry> ret_entry = std::make_shared<
+ MeasurementMetaIndexEntry>(ret_entry_name, 0, arena_);
int64_t ret_offset = 0;
char search_name[] = "aardvark";
int result = node_.binary_search_children(
std::make_shared<StringComparable>(search_name),
- false, *ret_entry, ret_offset);
+ false, ret_entry, ret_offset);
ASSERT_EQ(result, common::E_NOT_EXIST);
}
