This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 0adbc26b fix for reset bug and add ut (#435)
0adbc26b is described below
commit 0adbc26b4c7bea843b2a94b42313dc5ceb94fb8d
Author: Yukim1 <[email protected]>
AuthorDate: Tue Mar 11 09:57:47 2025 +0800
fix for reset bug and add ut (#435)
* fix for reset bug and add ut
* fix for num_of_pages_
* fix mem leak{
---
cpp/CMakeLists.txt | 4 +-
cpp/src/common/statistic.h | 59 +++++++++++++++++++++-
cpp/src/compress/lz4_compressor.h | 2 +-
cpp/src/compress/lzo_compressor.h | 2 +-
cpp/src/compress/snappy_compressor.h | 2 -
cpp/src/file/tsfile_io_writer.cc | 3 ++
cpp/src/file/write_file.cc | 1 +
cpp/src/file/write_file.h | 1 +
cpp/src/reader/table_result_set.cc | 2 +-
.../reader/table_view/tsfile_reader_table_test.cc | 56 +++++++++++++++++++-
.../writer/table_view/tsfile_writer_table_test.cc | 2 -
11 files changed, 122 insertions(+), 12 deletions(-)
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d33605bc..8ee4f17a 100755
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -52,8 +52,8 @@ endif()
if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build."
FORCE)
endif ()
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
message("CMAKE BUILD TYPE " ${CMAKE_BUILD_TYPE})
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -69,7 +69,7 @@ endif ()
message("CMAKE DEBUG: CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
# disable asan by default.
-option(ENABLE_ASAN "Enable Address Sanitizer" OFF)
+option(ENABLE_ASAN "Enable Address Sanitizer" ON)
if (NOT WIN32)
if (ENABLE_ASAN)
diff --git a/cpp/src/common/statistic.h b/cpp/src/common/statistic.h
index 77c62223..8eb866e2 100644
--- a/cpp/src/common/statistic.h
+++ b/cpp/src/common/statistic.h
@@ -127,7 +127,7 @@ class Statistic {
public:
Statistic() : count_(0), start_time_(0), end_time_(0) {}
virtual void destroy() {}
- FORCE_INLINE void reset() { count_ = 0; }
+ virtual FORCE_INLINE void reset() { count_ = 0; }
virtual FORCE_INLINE void update(int64_t time, bool value) {
ASSERT(false);
@@ -430,6 +430,13 @@ class BooleanStatistic : public Statistic {
last_value_ = that.last_value_;
}
+ FORCE_INLINE void reset() {
+ count_ = 0;
+ sum_value_ = 0;
+ first_value_ = false;
+ last_value_ = false;
+ }
+
FORCE_INLINE void update(int64_t time, bool value) {
BOOL_STAT_UPDATE(time, value);
}
@@ -494,6 +501,15 @@ class Int32Statistic : public Statistic {
last_value_ = that.last_value_;
}
+ FORCE_INLINE void reset() {
+ count_ = 0;
+ sum_value_ = 0;
+ min_value_ = 0;
+ max_value_ = 0;
+ first_value_ = 0;
+ last_value_ = 0;
+ }
+
FORCE_INLINE void update(int64_t time, int32_t value) {
NUM_STAT_UPDATE(time, value);
}
@@ -586,6 +602,14 @@ class Int64Statistic : public Statistic {
last_value_ = that.last_value_;
}
+ FORCE_INLINE void reset() {
+ count_ = 0;
+ sum_value_ = 0;
+ min_value_ = 0;
+ max_value_ = 0;
+ first_value_ = 0;
+ last_value_ = 0;
+ }
FORCE_INLINE void update(int64_t time, int64_t value) {
NUM_STAT_UPDATE(time, value);
}
@@ -670,6 +694,15 @@ class FloatStatistic : public Statistic {
first_value_ = that.first_value_;
last_value_ = that.last_value_;
}
+
+ FORCE_INLINE void reset() {
+ count_ = 0;
+ sum_value_ = 0;
+ min_value_ = 0;
+ max_value_ = 0;
+ first_value_ = 0;
+ last_value_ = 0;
+ }
FORCE_INLINE void update(int64_t time, float value) {
NUM_STAT_UPDATE(time, value);
}
@@ -738,6 +771,15 @@ class DoubleStatistic : public Statistic {
first_value_ = that.first_value_;
last_value_ = that.last_value_;
}
+
+ FORCE_INLINE void reset() {
+ count_ = 0;
+ sum_value_ = 0;
+ min_value_ = 0;
+ max_value_ = 0;
+ first_value_ = 0;
+ last_value_ = 0;
+ }
FORCE_INLINE void update(int64_t time, double value) {
NUM_STAT_UPDATE(time, value);
}
@@ -798,6 +840,12 @@ class TimeStatistic : public Statistic {
end_time_ = that.end_time_;
}
+ FORCE_INLINE void reset() {
+ count_ = 0;
+ start_time_ = 0;
+ end_time_ = 0;
+ }
+
FORCE_INLINE void update(int64_t time) {
TIME_STAT_UPDATE((time));
count_++;
@@ -849,6 +897,15 @@ class StringStatistic : public Statistic {
}
}
+ FORCE_INLINE void reset() {
+ count_ = 0;
+ start_time_ = 0;
+ end_time_ = 0;
+ min_value_ = common::String();
+ max_value_ = common::String();
+ first_value_ = common::String();
+ last_value_ = common::String();
+ }
void clone_from(const StringStatistic &that) {
count_ = that.count_;
start_time_ = that.start_time_;
diff --git a/cpp/src/compress/lz4_compressor.h
b/cpp/src/compress/lz4_compressor.h
index 74d6c697..e7f47a87 100644
--- a/cpp/src/compress/lz4_compressor.h
+++ b/cpp/src/compress/lz4_compressor.h
@@ -32,7 +32,7 @@
#include "utils/errno_define.h"
#include "utils/util_define.h"
-#define UNCOMPRESSED_TIME 4
+#define UNCOMPRESSED_TIME 5
namespace storage {
diff --git a/cpp/src/compress/lzo_compressor.h
b/cpp/src/compress/lzo_compressor.h
index aba0e208..95510032 100644
--- a/cpp/src/compress/lzo_compressor.h
+++ b/cpp/src/compress/lzo_compressor.h
@@ -32,7 +32,7 @@
#include "utils/errno_define.h"
#include "utils/util_define.h"
-#define UNCOMPRESSED_TIME 4
+#define UNCOMPRESSED_TIME 5
namespace storage {
diff --git a/cpp/src/compress/snappy_compressor.h
b/cpp/src/compress/snappy_compressor.h
index f94a208a..86b3bca5 100644
--- a/cpp/src/compress/snappy_compressor.h
+++ b/cpp/src/compress/snappy_compressor.h
@@ -32,8 +32,6 @@
#include "utils/errno_define.h"
#include "utils/util_define.h"
-#define UNCOMPRESSED_TIME 4
-
namespace storage {
class SnappyCompressor : public Compressor {
diff --git a/cpp/src/file/tsfile_io_writer.cc b/cpp/src/file/tsfile_io_writer.cc
index 58ee8442..6364adf8 100644
--- a/cpp/src/file/tsfile_io_writer.cc
+++ b/cpp/src/file/tsfile_io_writer.cc
@@ -246,6 +246,9 @@ int TsFileIOWriter::end_flush_chunk_group(bool is_aligned) {
int TsFileIOWriter::end_file() {
int ret = E_OK;
+ if (file_->get_fd() == -1) {
+ return E_OK;
+ }
OFFSET_DEBUG("before end file");
if (RET_FAIL(write_log_index_range())) {
std::cout << "writer range index error, ret =" << ret << std::endl;
diff --git a/cpp/src/file/write_file.cc b/cpp/src/file/write_file.cc
index 71cb553e..c8c2cbba 100644
--- a/cpp/src/file/write_file.cc
+++ b/cpp/src/file/write_file.cc
@@ -127,6 +127,7 @@ int WriteFile::close() {
// log_err("file close error, path=%s, errno=%d", path_.c_str(),
errno);
return E_FILE_CLOSE_ERR;
}
+ fd_ = -1;
#ifdef DEBUG_SE
std::cout << "close finish" << std::endl;
#endif
diff --git a/cpp/src/file/write_file.h b/cpp/src/file/write_file.h
index ddb1f2bc..3346d7ad 100644
--- a/cpp/src/file/write_file.h
+++ b/cpp/src/file/write_file.h
@@ -34,6 +34,7 @@ class WriteFile {
#ifndef LIBTSFILE_SDK
WriteFile() : path_(), file_id_(), fd_(-1) {}
FORCE_INLINE common::FileID get_file_id() const { return file_id_; }
+ FORCE_INLINE int get_fd() const { return fd_; }
int create(const common::FileID &file_id, int flags, mode_t mode);
#else
WriteFile() : path_(), fd_(-1) {}
diff --git a/cpp/src/reader/table_result_set.cc
b/cpp/src/reader/table_result_set.cc
index 8eb5d9dd..6fb8a58a 100644
--- a/cpp/src/reader/table_result_set.cc
+++ b/cpp/src/reader/table_result_set.cc
@@ -62,7 +62,7 @@ int TableResultSet::next(bool& has_next) {
if (row_iterator_ == nullptr || !row_iterator_->has_next()) {
has_next = false;
}
-
+
if (has_next && IS_SUCC(ret)) {
uint32_t len = 0;
bool null = false;
diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
index 33e9efa8..58288a23 100644
--- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
+++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
@@ -48,8 +48,6 @@ class TsFileTableReaderTest : public ::testing::Test {
write_file_.create(file_name_, flags, mode);
}
void TearDown() override {
- write_file_.sync();
- write_file_.close();
remove(file_name_.c_str());
}
std::string file_name_;
@@ -324,3 +322,57 @@ TEST_F(TsFileTableReaderTest, TableModelGetSchema) {
ASSERT_EQ(reader.close(), common::E_OK);
delete tmp_table_schema;
}
+
+TEST_F(TsFileTableReaderTest, TableModelQueryWithMultiTabletsMultiFlush) {
+ auto tmp_table_schema = gen_table_schema(0);
+ auto tsfile_table_writer_ =
+ std::make_shared<TsFileTableWriter>(&write_file_, tmp_table_schema);
+ int max_rows = 100000;
+ int tablet_size = 10000;
+ int cur_row = 0;
+ for (; cur_row < max_rows;) {
+ if (cur_row + tablet_size > max_rows) {
+ tablet_size = max_rows - cur_row;
+ }
+ auto tablet = gen_tablet(tmp_table_schema, cur_row, 1, tablet_size);
+ ASSERT_EQ(tsfile_table_writer_->write_table(tablet), common::E_OK);
+ cur_row += tablet_size;
+ std::cout << "finish writing " << cur_row << " rows" << std::endl;
+ }
+ ASSERT_EQ(tsfile_table_writer_->flush(), common::E_OK);
+ ASSERT_EQ(tsfile_table_writer_->close(), common::E_OK);
+ common::init_config_value();
+ storage::TsFileReader reader;
+ int ret = reader.open(file_name_);
+ ASSERT_EQ(ret, common::E_OK);
+ storage::ResultSet* tmp_result_set = nullptr;
+ ret = reader.query("testtable0",
+ tmp_table_schema->get_measurement_names(), 0,
1000000000000,
+ tmp_result_set);
+ std::cout << "begin to dump data from tsfile ---" << std::endl;
+ auto* table_result_set = (storage::TableResultSet*)tmp_result_set;
+ bool has_next = false;
+ char* literal = new char[std::strlen("device_id") + 1];
+ std::strcpy(literal, "device_id");
+ String literal_str(literal, std::strlen("device_id"));
+ while (IS_SUCC(table_result_set->next(has_next)) && has_next) {
+ for (int i = 0; i < 1; i++) {
+ auto column_schemas = tmp_table_schema->get_measurement_schemas();
+ for (int j = 0; j < column_schemas.size(); j++) {
+ switch (column_schemas[j]->data_type_) {
+ case TSDataType::INT64:
+ ASSERT_EQ(table_result_set->get_value<int64_t>(j + 2),
i);
+ break;
+ case TSDataType::STRING:
+
ASSERT_EQ(table_result_set->get_value<common::String*>(j +
2)->compare(literal_str), 0);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ }
+ reader.destroy_query_data_set(table_result_set);
+ delete[] literal;
+ delete tmp_table_schema;
+}
diff --git a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
index bc152571..78d9c31b 100644
--- a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
+++ b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
@@ -46,8 +46,6 @@ class TsFileWriterTableTest : public ::testing::Test {
write_file_.create(file_name_, flags, mode);
}
void TearDown() override {
- write_file_.sync();
- write_file_.close();
remove(file_name_.c_str());
}
std::string file_name_;
