This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new a5424d37 Same name (case insensitive) in tablet / table. (#490)
a5424d37 is described below
commit a5424d3718154ac02b4710cc0daaf244c2bacf63
Author: Colin Lee <[email protected]>
AuthorDate: Wed Jun 4 16:27:59 2025 +0800
Same name (case insensitive) in tablet / table. (#490)
* tmp code.
* add duplicate column name check.
* fix memory leak.
* fix memory leak.
* fix memory leak.
---
cpp/src/common/schema.h | 4 +++
cpp/src/common/tablet.cc | 16 +++++++--
cpp/src/common/tablet.h | 13 ++++----
cpp/src/cwrapper/tsfile_cwrapper.cc | 7 ++--
cpp/src/writer/tsfile_table_writer.cc | 4 +++
cpp/src/writer/tsfile_table_writer.h | 6 +++-
cpp/src/writer/tsfile_writer.cc | 25 ++++++++++++--
.../writer/table_view/tsfile_writer_table_test.cc | 38 ++++++++++++++++++++++
8 files changed, 97 insertions(+), 16 deletions(-)
diff --git a/cpp/src/common/schema.h b/cpp/src/common/schema.h
index 5deffad9..0227aa50 100644
--- a/cpp/src/common/schema.h
+++ b/cpp/src/common/schema.h
@@ -331,6 +331,10 @@ class TableSchema {
}
}
+ size_t get_column_pos_index_num() const {
+ return column_pos_index_.size();
+ }
+
void update(ChunkGroupMeta *chunk_group_meta) {
for (auto iter = chunk_group_meta->chunk_meta_list_.begin();
iter != chunk_group_meta->chunk_meta_list_.end(); iter++) {
diff --git a/cpp/src/common/tablet.cc b/cpp/src/common/tablet.cc
index ac4a2708..8bc801cf 100644
--- a/cpp/src/common/tablet.cc
+++ b/cpp/src/common/tablet.cc
@@ -36,9 +36,8 @@ int Tablet::init() {
std::pair<std::map<std::string, int>::iterator, bool> ins_res;
for (size_t c = 0; c < schema_count; c++) {
ins_res = schema_map_.insert(
- std::make_pair(schema_vec_->at(c).measurement_name_, c));
+ std::make_pair(to_lower(schema_vec_->at(c).measurement_name_), c));
if (!ins_res.second) {
- ASSERT(false);
// maybe dup measurement_name
return E_INVALID_ARG;
}
@@ -131,6 +130,9 @@ void Tablet::destroy() {
}
int Tablet::add_timestamp(uint32_t row_index, int64_t timestamp) {
+ if (err_code_ != E_OK) {
+ return err_code_;
+ }
ASSERT(timestamps_ != NULL);
if (UNLIKELY(row_index >= static_cast<uint32_t>(max_row_num_))) {
ASSERT(false);
@@ -223,6 +225,9 @@ void Tablet::process_val(uint32_t row_index, uint32_t
schema_index, T val) {
template <typename T>
int Tablet::add_value(uint32_t row_index, uint32_t schema_index, T val) {
+ if (err_code_ != E_OK) {
+ return err_code_;
+ }
int ret = common::E_OK;
if (UNLIKELY(schema_index >= schema_vec_->size())) {
ASSERT(false);
@@ -250,6 +255,9 @@ int Tablet::add_value(uint32_t row_index, uint32_t
schema_index, T val) {
template <>
int Tablet::add_value(uint32_t row_index, uint32_t schema_index,
common::String val) {
+ if (err_code_ != E_OK) {
+ return err_code_;
+ }
int ret = common::E_OK;
if (UNLIKELY(schema_index >= schema_vec_->size())) {
ASSERT(false);
@@ -269,9 +277,11 @@ template <typename T>
int Tablet::add_value(uint32_t row_index, const std::string &measurement_name,
T val) {
int ret = common::E_OK;
+ if (err_code_ != E_OK) {
+ return err_code_;
+ }
SchemaMapIterator find_iter = schema_map_.find(measurement_name);
if (LIKELY(find_iter == schema_map_.end())) {
- ASSERT(false);
ret = E_INVALID_ARG;
} else {
ret = add_value(row_index, find_iter->second, val);
diff --git a/cpp/src/common/tablet.h b/cpp/src/common/tablet.h
index e69d477e..b036cca9 100644
--- a/cpp/src/common/tablet.h
+++ b/cpp/src/common/tablet.h
@@ -57,6 +57,7 @@ class Tablet {
public:
static const uint32_t DEFAULT_MAX_ROWS = 1024;
+ int err_code_ = common::E_OK;
public:
Tablet(const std::string &device_id,
@@ -75,7 +76,7 @@ class Tablet {
ASSERT(false);
max_row_num_ = DEFAULT_MAX_ROWS;
}
- init();
+ err_code_ = init();
}
Tablet(const std::string &device_id,
@@ -106,7 +107,7 @@ class Tablet {
return MeasurementSchema(name, type);
});
schema_vec_ =
std::make_shared<std::vector<MeasurementSchema>>(measurement_vec);
- init();
+ err_code_ = init();
}
Tablet(const std::string &insert_target_name,
@@ -127,7 +128,7 @@ class Tablet {
common::get_default_compressor()));
}
set_column_categories(column_categories);
- init();
+ err_code_ = init();
}
/**
@@ -150,10 +151,10 @@ class Tablet {
schema_vec_ = std::make_shared<std::vector<MeasurementSchema>>();
for (size_t i = 0; i < column_names.size(); i++) {
schema_vec_->emplace_back(
- MeasurementSchema(column_names[i], data_types[i],
common::get_value_encoder(data_types[i]),
- common::get_default_compressor()));
+ column_names[i], data_types[i],
common::get_value_encoder(data_types[i]),
+ common::get_default_compressor());
}
- init();
+ err_code_ = init();
}
~Tablet() { destroy(); }
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc
b/cpp/src/cwrapper/tsfile_cwrapper.cc
index 0997dc42..656737df 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.cc
+++ b/cpp/src/cwrapper/tsfile_cwrapper.cc
@@ -174,7 +174,7 @@ Tablet tablet_new(char **column_name_list, TSDataType
*data_types,
std::vector<std::string> measurement_list;
std::vector<common::TSDataType> data_type_list;
for (uint32_t i = 0; i < column_num; i++) {
- measurement_list.emplace_back(column_name_list[i]);
+ measurement_list.emplace_back(storage::to_lower(column_name_list[i]));
data_type_list.push_back(
static_cast<common::TSDataType>(*(data_types + i)));
}
@@ -196,7 +196,7 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t
row_index,
const char *column_name, \
const type value) { \
return static_cast<storage::Tablet *>(tablet)->add_value( \
- row_index, column_name, value); \
+ row_index, storage::to_lower(column_name), value); \
}
TABLET_ADD_VALUE_BY_NAME_DEF(int32_t);
TABLET_ADD_VALUE_BY_NAME_DEF(int64_t);
@@ -208,7 +208,7 @@ ERRNO tablet_add_value_by_name_string(Tablet tablet,
uint32_t row_index,
const char *column_name,
const char *value) {
return static_cast<storage::Tablet *>(tablet)->add_value(
- row_index, column_name, common::String(value));
+ row_index, storage::to_lower(column_name), common::String(value));
}
#define TABLE_ADD_VALUE_BY_INDEX_DEF(type) \
@@ -688,7 +688,6 @@ ERRNO _tsfile_writer_flush(TsFileWriter writer) {
return w->flush();
}
-
ResultSet _tsfile_reader_query_device(TsFileReader reader,
const char *device_name,
char **sensor_name, uint32_t sensor_num,
diff --git a/cpp/src/writer/tsfile_table_writer.cc
b/cpp/src/writer/tsfile_table_writer.cc
index 6cbb0c9b..942b15b5 100644
--- a/cpp/src/writer/tsfile_table_writer.cc
+++ b/cpp/src/writer/tsfile_table_writer.cc
@@ -29,6 +29,10 @@ int storage::TsFileTableWriter::register_table(const
std::shared_ptr<TableSchema
}
int storage::TsFileTableWriter::write_table(storage::Tablet& tablet) const {
+ // DIRTY CODE...
+ if (common::E_OK != error_number) {
+ return error_number;
+ }
if (tablet.get_table_name().empty()) {
tablet.set_table_name(exclusive_table_name_);
} else if (!exclusive_table_name_.empty() && tablet.get_table_name() !=
exclusive_table_name_) {
diff --git a/cpp/src/writer/tsfile_table_writer.h
b/cpp/src/writer/tsfile_table_writer.h
index 72d2833e..d3fc918b 100644
--- a/cpp/src/writer/tsfile_table_writer.h
+++ b/cpp/src/writer/tsfile_table_writer.h
@@ -61,7 +61,7 @@ class TsFileTableWriter {
// Perform a deep copy. The source TableSchema object may be
// stack/heap-allocated.
auto table_schema_ptr = std::make_shared<TableSchema>(*table_schema);
- tsfile_writer_->register_table(table_schema_ptr);
+ error_number = tsfile_writer_->register_table(table_schema_ptr);
exclusive_table_name_ = table_schema->get_table_name();
common::g_config_value_.chunk_group_size_threshold_ = memory_threshold;
}
@@ -106,6 +106,10 @@ class TsFileTableWriter {
// if this TsFile only contains one table, this will be its name,
otherwise,
// it will be an empty string
std::string exclusive_table_name_;
+
+ // Some errors may not be conveyed during the construction phase, so it's
+ // necessary to maintain an internal error code.
+ int error_number = common::E_OK;
};
} // namespace storage
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index a570fc54..0f47756d 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -117,6 +117,27 @@ void TsFileWriter::set_generate_table_schema(bool
generate_table_schema) {
int TsFileWriter::register_table(
const std::shared_ptr<TableSchema> &table_schema) {
if (!table_schema) return E_INVALID_ARG;
+
+ // Empty table name or column name is not allowed.
+ if (table_schema->get_table_name().empty()) {
+ return E_INVALID_ARG;
+ }
+ for (const auto &name : table_schema->get_measurement_names()) {
+ if (name.empty()) {
+ return E_INVALID_ARG;
+ }
+ }
+
+ // Because it is not possible to return an error code for duplicate name
+ // checks during the construction phase of TabletSchema, the duplicate name
+ // check has been moved to the table registration stage.
+
+ // TODO: Add Debug INFO if ErrorCode is not enough to describe problems.
+ if (table_schema->get_column_pos_index_num() !=
+ table_schema->get_measurement_names().size()) {
+ return E_INVALID_ARG;
+ }
+
if (io_writer_->get_schema()->table_schema_map_.find(
table_schema->get_table_name()) !=
io_writer_->get_schema()->table_schema_map_.end()) {
@@ -671,7 +692,7 @@ int TsFileWriter::write_tablet_aligned(const Tablet
&tablet) {
continue;
}
if (RET_FAIL(value_write_column(value_chunk_writer, tablet, c, 0,
- tablet.get_cur_row_size()))) {
+ tablet.get_cur_row_size()))) {
return ret;
}
}
@@ -764,7 +785,7 @@ int TsFileWriter::write_table(Tablet &tablet) {
continue;
}
if (RET_FAIL(write_column(chunk_writer, tablet, c, start_idx,
- device_id_end_index_pair.second))) {
+ device_id_end_index_pair.second))) {
return ret;
}
}
diff --git a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
index 85ddcbce..a13c20be 100644
--- a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
+++ b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
@@ -351,4 +351,42 @@ TEST_F(TsFileWriterTableTest, WriteAndReadSimple) {
reader.close();
delete table_schema;
+}
+
+TEST_F(TsFileWriterTableTest, DuplicateColumnName) {
+ std::vector<MeasurementSchema*> measurement_schemas;
+ std::vector<ColumnCategory> column_categories;
+ measurement_schemas.resize(3);
+ measurement_schemas[0] = new MeasurementSchema("device", STRING);
+ column_categories.emplace_back(ColumnCategory::TAG);
+ measurement_schemas[1] = new MeasurementSchema("Device", STRING);
+ column_categories.emplace_back(ColumnCategory::TAG);
+ measurement_schemas[2] = new MeasurementSchema("value", DOUBLE);
+ column_categories.emplace_back(ColumnCategory::FIELD);
+ TableSchema* table_schema =
+ new TableSchema("test_table", measurement_schemas, column_categories);
+ auto tsfile_table_writer =
+ std::make_shared<TsFileTableWriter>(&write_file_, table_schema);
+ Tablet tablet = Tablet(table_schema->get_measurement_names(),
+ table_schema->get_data_types());
+ tablet.set_table_name("test_table");
+ ASSERT_EQ(E_INVALID_ARG, tablet.add_timestamp(0, 10));
+ ASSERT_EQ(E_INVALID_ARG, tablet.add_value(1, 1, 10));
+ ASSERT_EQ(E_INVALID_ARG, tablet.add_value(1, "test", 10));
+ std::vector<MeasurementSchema> measurement_schemas2;
+ for (int i = 0; i < 2; i++) {
+ measurement_schemas2.push_back(*measurement_schemas[i]);
+ }
+ Tablet tablet1 = Tablet(
+ "test_table",
+
std::make_shared<std::vector<MeasurementSchema>>(measurement_schemas2));
+ tablet1.set_table_name("test_table");
+ ASSERT_EQ(E_INVALID_ARG, tablet1.add_timestamp(0, 10));
+ ASSERT_EQ(E_INVALID_ARG, tablet1.add_value(1, 1, 10));
+ ASSERT_EQ(E_INVALID_ARG, tablet1.add_value(1, "test", 10));
+
+ ASSERT_EQ(E_INVALID_ARG, tsfile_table_writer->write_table(tablet));
+ ASSERT_EQ(E_INVALID_ARG, tsfile_table_writer->register_table(
+
std::make_shared<TableSchema>(*table_schema)));
+ delete table_schema;
}
\ No newline at end of file
