This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch fix_large_page_read_and_bitmap_index in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit b4cd0509931ca587f70e8164a5dfc374c6dd8453 Author: Tian Jiang <[email protected]> AuthorDate: Mon Mar 3 14:29:47 2025 +0800 Fix large page read and bitmap index --- cpp/src/reader/aligned_chunk_reader.cc | 23 +++++++++++++--------- cpp/src/reader/aligned_chunk_reader.h | 2 +- .../reader/table_view/tsfile_reader_table_test.cc | 15 ++++++++++++-- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index a6ce3e7b..3daca292 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -289,7 +289,7 @@ int AlignedChunkReader::get_cur_page_header(ChunkMeta *&chunk_meta, // @in_stream_ int AlignedChunkReader::read_from_file_and_rewrap( common::ByteStream &in_stream_, ChunkMeta *&chunk_meta, - uint32_t &chunk_visit_offset, int32_t file_data_buf_size, int want_size) { + uint32_t &chunk_visit_offset, int32_t &file_data_buf_size, int want_size) { int ret = E_OK; const int DEFAULT_READ_SIZE = 4096; // may use page_size + page_header_size char *file_data_buf = in_stream_.get_wrapped_buf(); @@ -350,8 +350,8 @@ int AlignedChunkReader::decode_cur_time_page_data() { // << cur_page_header_.compressed_size_ << std::endl; if (RET_FAIL(read_from_file_and_rewrap( time_in_stream_, time_chunk_meta_, time_chunk_visit_offset_, - cur_time_page_header_.compressed_size_, - file_data_time_buf_size_))) { + file_data_time_buf_size_, + cur_value_page_header_.compressed_size_))) { } } @@ -429,8 +429,8 @@ int AlignedChunkReader::decode_cur_value_page_data() { // << cur_page_header_.compressed_size_ << std::endl; if (RET_FAIL(read_from_file_and_rewrap( value_in_stream_, value_chunk_meta_, value_chunk_visit_offset_, - cur_value_page_header_.compressed_size_, - file_data_value_buf_size_))) { + file_data_value_buf_size_, + cur_value_page_header_.compressed_size_))) { } } @@ -529,17 +529,22 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock( int64_t time = 0; \ CppType value; \ while ((time_decoder_->has_remaining() || time_in.has_remaining()) \ - && (value_decoder_->has_remaining() || \ - value_in.has_remaining())){ \ - cur_value_index++; \ + && (value_decoder_->has_remaining() || \ + value_in.has_remaining())){ \ if (((value_page_col_notnull_bitmap_[cur_value_index / 8] & \ 0xFF) & \ (mask >> (cur_value_index % 8))) == 0) { \ - RET_FAIL(time_decoder_->read_int64(time, time_in)); \ + ret = time_decoder_->read_int64(time, time_in); \ + if (ret != E_OK) { \ + break; \ + } \ + ret = value_decoder_->read_##ReadType(value, \ + value_in); \ if (ret != E_OK) { \ break; \ } \ } \ + cur_value_index++; \ if (UNLIKELY(!row_appender.add_row())) { \ ret = E_OVERFLOW; \ break; \ diff --git a/cpp/src/reader/aligned_chunk_reader.h b/cpp/src/reader/aligned_chunk_reader.h index 365e9efe..becca806 100644 --- a/cpp/src/reader/aligned_chunk_reader.h +++ b/cpp/src/reader/aligned_chunk_reader.h @@ -94,7 +94,7 @@ class AlignedChunkReader : public IChunkReader { int read_from_file_and_rewrap(common::ByteStream &in_stream_, ChunkMeta *&chunk_meta, uint32_t &chunk_visit_offset, - int32_t file_data_buf_size, + int32_t &file_data_buf_size, int want_size = 0); bool cur_page_statisify_filter(Filter *filter); int skip_cur_page(); diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc index c633b405..45e3d502 100644 --- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc +++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc @@ -102,7 +102,8 @@ class TsFileTableReaderTest : public ::testing::Test { storage::Tablet tablet(table_schema->get_table_name(), table_schema->get_measurement_names(), table_schema->get_data_types(), - table_schema->get_column_categories()); + table_schema->get_column_categories(), + device_num * num_timestamp_per_device); char* literal = new char[std::strlen("device_id") + 1]; std::strcpy(literal, "device_id"); @@ -197,8 +198,18 @@ class TsFileTableReaderTest : public ::testing::Test { TEST_F(TsFileTableReaderTest, TableModelQuery) { test_table_model_query(); } -TEST_F(TsFileTableReaderTest, TableModelQueryOnePage) { +TEST_F(TsFileTableReaderTest, TableModelQueryOneSmallPage) { + int prev_config = g_config_value_.page_writer_max_point_num_; + g_config_value_.page_writer_max_point_num_ = 5; test_table_model_query(g_config_value_.page_writer_max_point_num_); + g_config_value_.page_writer_max_point_num_ = prev_config; +} + +TEST_F(TsFileTableReaderTest, TableModelQueryOneLargePage) { + int prev_config = g_config_value_.page_writer_max_point_num_; + g_config_value_.page_writer_max_point_num_ = 10000; + test_table_model_query(g_config_value_.page_writer_max_point_num_); + g_config_value_.page_writer_max_point_num_ = prev_config; } TEST_F(TsFileTableReaderTest, TableModelResultMetadata) {
