This is an automated email from the ASF dual-hosted git repository.
colinlee pushed a commit to branch fix_err
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/fix_err by this push:
new 0377f58e fix length.
0377f58e is described below
commit 0377f58e900f0333182de948cc02430cd7ba3558
Author: ColinLee <[email protected]>
AuthorDate: Mon Jun 23 18:06:42 2025 +0800
fix length.
---
cpp/src/common/allocator/alloc_base.h | 41 -----------
cpp/src/common/allocator/byte_stream.h | 28 ++++----
cpp/src/common/allocator/mem_alloc.cc | 41 +++++------
cpp/src/common/allocator/my_string.h | 120 ++++++++++++++++++++-------------
cpp/src/common/error_info/error_info.h | 45 +++++++------
cpp/src/reader/bloom_filter.cc | 21 +++---
cpp/src/reader/bloom_filter.h | 4 +-
7 files changed, 145 insertions(+), 155 deletions(-)
diff --git a/cpp/src/common/allocator/alloc_base.h
b/cpp/src/common/allocator/alloc_base.h
index e9824b11..153516fc 100644
--- a/cpp/src/common/allocator/alloc_base.h
+++ b/cpp/src/common/allocator/alloc_base.h
@@ -79,47 +79,6 @@ void *mem_alloc(uint32_t size, AllocModID mid);
void mem_free(void *ptr);
void *mem_realloc(void *ptr, uint32_t size);
-class ModStat {
- public:
- ModStat() : stat_arr_(NULL) {}
-
- static ModStat &get_instance() {
- /*
- * This is the singleton of Mod Memory Statistic.
- * gms is short for Global Mod Statistic
- */
- static ModStat gms;
- return gms;
- }
- void init();
- void destroy();
- INLINE void update_alloc(AllocModID mid, int32_t size) {
- // ASSERT(mid < __LAST_MOD_ID);
- // ATOMIC_FAA(get_item(mid), size);
- }
- void update_free(AllocModID mid, uint32_t size) {
- // ASSERT(mid < __LAST_MOD_ID);
- // ATOMIC_FAA(get_item(mid), 0 - size);
- }
- void print_stat();
-
-#ifdef ENABLE_TEST
- int32_t TEST_get_stat(int8_t mid) { return ATOMIC_FAA(get_item(mid), 0); }
-#endif
-
- private:
- INLINE int32_t *get_item(int8_t mid) {
- return &(stat_arr_[mid * (ITEM_SIZE / sizeof(int32_t))]);
- }
-
- private:
- static const int32_t ITEM_SIZE = CACHE_LINE_SIZE;
- static const int32_t ITEM_COUNT = __LAST_MOD_ID;
- int32_t *stat_arr_;
-
- STATIC_ASSERT((ITEM_SIZE % sizeof(int32_t) == 0), ModStat_ITEM_SIZE_ERROR);
-};
-
/* base allocator */
class BaseAllocator {
public:
diff --git a/cpp/src/common/allocator/byte_stream.h
b/cpp/src/common/allocator/byte_stream.h
index d4047f83..e0a1be89 100644
--- a/cpp/src/common/allocator/byte_stream.h
+++ b/cpp/src/common/allocator/byte_stream.h
@@ -236,7 +236,7 @@ class ByteStream {
private:
struct Page {
OptionalAtomic<Page *> next_; // 9 bytes
- uint8_t *buf_; // 8 bytes
+ const uint8_t *buf_; // 8 bytes
explicit Page(bool enable_atomic) : next_(nullptr, enable_atomic) {
buf_ = (uint8_t
@@ -287,9 +287,9 @@ class ByteStream {
/* ================ Part 0: wrap from outside buffer ================ */
// if you wrap a buffer as a ByteStream, you should
// manage the outside buffer yourself.
- void wrap_from(const char *buf, int32_t buf_len) {
+ void wrap_from(const char *buf, const size_t buf_len) {
wrapped_page_.next_.store(nullptr);
- wrapped_page_.buf_ = (uint8_t *)buf;
+ wrapped_page_.buf_ = reinterpret_cast<uint8_t const *>(buf);
page_size_ = buf_len;
head_.store(&wrapped_page_);
@@ -302,7 +302,7 @@ class ByteStream {
FORCE_INLINE bool is_wrapped() const {
return head_.load() == &wrapped_page_;
}
- char *get_wrapped_buf() { return (char *)wrapped_page_.buf_; }
+ char *get_wrapped_buf() const { return (char *)wrapped_page_.buf_; }
void clear_wrapped_buf() { wrapped_page_.buf_ = nullptr; }
/* ================ Part 1: basic ================ */
@@ -358,16 +358,16 @@ class ByteStream {
/* ================ Part 2: write_xxx and read_xxx ================ */
// writer @buf with length @len into this bytestream
- int write_buf(const uint8_t *buf, const uint32_t len) {
+ int write_buf(const uint8_t *buf, const size_t len) {
int ret = error_info::E_OK;
- uint32_t write_len = 0;
+ size_t write_len = 0;
while (write_len < len) {
if (RET_FAIL(prepare_space())) {
- std::cout << "write_buf error " << ret << std::endl;
- return ret;
+ RETURN_ERR(error_info::E_OOM, "prepare spec failed");
}
- uint32_t remainder = page_size_ - (total_size_.load() %
page_size_);
- uint32_t copy_len =
+ const size_t remainder =
+ page_size_ - (total_size_.load() % page_size_);
+ const size_t copy_len =
remainder < (len - write_len) ? remainder : (len - write_len);
memcpy(tail_.load()->buf_ + total_size_.load() % page_size_,
buf + write_len, copy_len);
@@ -643,10 +643,10 @@ class ByteStream {
OptionalAtomic<Page *> head_;
OptionalAtomic<Page *> tail_;
Page *read_page_; // only one thread is allow to reader this ByteStream
- OptionalAtomic<int64_t> total_size_; // total size in byte
- int64_t read_pos_; // current reader position
- int64_t marked_read_pos_; // current reader position
- int64_t page_size_;
+ OptionalAtomic<size_t> total_size_; // total size in byte
+ size_t read_pos_; // current reader position
+ size_t marked_read_pos_; // current reader position
+ size_t page_size_;
AllocModID mid_;
Page wrapped_page_;
};
diff --git a/cpp/src/common/allocator/mem_alloc.cc
b/cpp/src/common/allocator/mem_alloc.cc
index a1e16d9b..e84f5867 100644
--- a/cpp/src/common/allocator/mem_alloc.cc
+++ b/cpp/src/common/allocator/mem_alloc.cc
@@ -24,6 +24,7 @@
#include <iostream>
+#include "../../../cmake-build-debug/include/common/error_info/error_info.h"
#include "alloc_base.h"
#include "common/logger/elog.h"
#include "stdio.h"
@@ -63,42 +64,40 @@ const char *g_mod_names[__LAST_MOD_ID] = {
/* 27 */ "HASH_TABLE",
};
-const uint32_t HEADER_SIZE_4B = 4;
-const uint32_t HEADER_SIZE_8B = 8;
+constexpr uint32_t HEADER_SIZE_4B = 4;
+constexpr uint32_t HEADER_SIZE_8B = 8;
-void *mem_alloc(uint32_t size, AllocModID mid) {
+void *mem_alloc(const uint32_t size, const AllocModID mid) {
// use 7bit at most
ASSERT(mid <= 127);
if (size <= 0xFFFFFF) {
// use 3B size + 1B mod
- char *p = (char *)malloc(size + HEADER_SIZE_4B);
+ auto *p = static_cast<char *>(malloc(size + HEADER_SIZE_4B));
if (UNLIKELY(p == nullptr)) {
return nullptr;
} else {
uint32_t header = (size << 8) | ((uint32_t)mid);
*((uint32_t *)p) = header;
- ModStat::get_instance().update_alloc(mid, size);
- // cppcheck-suppress memleak
- // cppcheck-suppress unmatchedSuppression
return p + HEADER_SIZE_4B;
}
} else {
- char *p = (char *)malloc(size + HEADER_SIZE_8B);
+ if (size > UINT32_MAX - HEADER_SIZE_4B) {
+ set_err_info(error_info::E_OOM, "Too large spec to allocate");
+ return nullptr;
+ }
+ auto *p = static_cast<char *>(malloc(size + HEADER_SIZE_8B));
if (UNLIKELY(p == nullptr)) {
- std::cout << "alloc big filed for size " << size + HEADER_SIZE_4B
- << std::endl;
+ set_err_info(error_info::E_OOM, "allocate failed");
return nullptr;
} else {
- uint64_t large_size = size;
- uint64_t header = ((large_size) << 8) | (((uint32_t)mid) | (0x80));
- uint32_t low4b = (uint32_t)(header & 0xFFFFFFFF);
- uint32_t high4b = (uint32_t)(header >> 32);
- *(uint32_t *)p = high4b;
- *(uint32_t *)(p + 4) = low4b;
- ModStat::get_instance().update_alloc(mid, size);
- // cppcheck-suppress unmatchedSuppression
- // cppcheck-suppress memleak
+ const uint64_t large_size = size;
+ const uint64_t header =
+ ((large_size) << 8) | (((uint32_t)mid) | (0x80));
+ const auto low4b = static_cast<uint32_t>(header & 0xFFFFFFFF);
+ const auto high4b = static_cast<uint32_t>(header >> 32);
+ *reinterpret_cast<uint32_t *>(p) = high4b;
+ *reinterpret_cast<uint32_t *>(p + 4) = low4b;
return p + HEADER_SIZE_8B;
}
}
@@ -247,10 +246,6 @@ void ModStat::init() {
void ModStat::destroy() { ::free(stat_arr_); }
-// TODO return to SQL
-void ModStat::print_stat() {
- //
-}
BaseAllocator g_base_allocator;
diff --git a/cpp/src/common/allocator/my_string.h
b/cpp/src/common/allocator/my_string.h
index 68f18635..82c7edf2 100644
--- a/cpp/src/common/allocator/my_string.h
+++ b/cpp/src/common/allocator/my_string.h
@@ -19,9 +19,9 @@
#ifndef COMMON_ALLOCATOR_MY_STRING_H
#define COMMON_ALLOCATOR_MY_STRING_H
-#include <string.h>
+#include <common/error_info/error_info.h>
+#include <cstring>
-#include <iostream>
#include <numeric>
#include "common/allocator/page_arena.h"
@@ -29,37 +29,45 @@
namespace common {
-// String that use PageArena
+// String that use PageArena or system spec.
struct String {
char *buf_;
+ // if len_ == 0: equal to "";
+ // if len_ == -1: equal to nullptr;
int32_t len_;
+ // if owned is true, buf_ should be free.
+ bool owned_;
- String() : buf_(nullptr), len_(0) {}
- String(char *buf, uint32_t len) : buf_(buf), len_(len) {}
+ String() : buf_(nullptr), len_(0), owned_(false) {}
+ String(char *buf, int32_t len) : buf_(buf), len_(len), owned_(false) {}
String(const std::string &str, common::PageArena &pa)
- : buf_(nullptr), len_(0) {
+ : buf_(nullptr), len_(0), owned_(false) {
dup_from(str, pa);
}
- String(const std::string &str) {
- buf_ = (char *)str.c_str();
- len_ = str.size();
+ explicit String(const std::string &str) {
+ buf_ = const_cast<char *>(str.c_str());
+ len_ = static_cast<int32_t>(str.size());
+ owned_ = false;
}
- FORCE_INLINE bool is_null() const { return buf_ == nullptr && len_ == 0; }
+ FORCE_INLINE bool is_null() const { return len_ == -1; }
+ FORCE_INLINE bool is_empty() const { return len_ == 0; }
FORCE_INLINE void reset() {
len_ = 0;
buf_ = nullptr;
}
FORCE_INLINE int dup_from(const std::string &str, common::PageArena &pa) {
- ASSERT(str.length() <=
static_cast<int32_t>(std::numeric_limits<int32_t>::max()));
- len_ = str.size();
+ ASSERT(str.length() <=
+ static_cast<int32_t>(std::numeric_limits<int32_t>::max()));
+ len_ = static_cast<int32_t>(str.size());
if (UNLIKELY(len_ == 0)) {
return error_info::E_OK;
}
- buf_ = pa.alloc(len_);
+ // len_ must >= 0;
+ buf_ = pa.alloc(static_cast<uint32_t>(len_));
if (IS_NULL(buf_)) {
- return common::E_OOM;
+ RETURN_ERR(error_info::E_OOM, "oom when dup from str: " + str);
}
- memcpy(buf_, str.c_str(), len_);
+ memcpy(buf_, str.c_str(), static_cast<size_t>(len_));
return error_info::E_OK;
}
@@ -73,22 +81,25 @@ struct String {
buf_ = nullptr;
return error_info::E_OK;
}
- buf_ = pa.alloc(len_);
+ buf_ = pa.alloc(static_cast<uint32_t>(len_));
if (IS_NULL(buf_)) {
- return common::E_OOM;
+ RETURN_ERR(error_info::E_OOM,
+ "oom when dup from str: " + str.to_std_string());
}
- memcpy(buf_, str.buf_, len_);
+ memcpy(buf_, str.buf_, static_cast<uint32_t>(len_));
return error_info::E_OK;
}
FORCE_INLINE int build_from(const String &s1, const String &s2,
common::PageArena &pa) {
len_ = s1.len_ + s2.len_;
- buf_ = pa.alloc(len_);
+ buf_ = pa.alloc(static_cast<uint32_t>(len_));
if (IS_NULL(buf_)) {
- return common::E_OOM;
+ RETURN_ERR(error_info::E_OOM,
+ "oom when build from str1: " + s1.to_std_string() +
+ ", str2: " + s2.to_std_string());
}
- memcpy(buf_, s1.buf_, s1.len_);
- memcpy(buf_ + s1.len_, s2.buf_, s2.len_);
+ memcpy(buf_, s1.buf_, static_cast<size_t>(s1.len_));
+ memcpy(buf_ + s1.len_, s2.buf_, static_cast<size_t>(s2.len_));
return error_info::E_OK;
}
FORCE_INLINE void shallow_copy_from(const String &src) {
@@ -97,23 +108,28 @@ struct String {
}
FORCE_INLINE bool equal_to(const String &that) const {
return (len_ == 0 && that.len_ == 0) ||
- ((len_ == that.len_) && (0 == memcmp(buf_, that.buf_, len_)));
+ (len_ == -1 && that.len_ == -1) ||
+ ((len_ == that.len_) && (0 == memcmp(buf_, that.buf_,
static_cast<size_t>(len_))));
}
-
- // FORCE_INLINE bool equal_to(const std::string &that)
- // {
- // return (len_ == 0 && that.size() == 0)
- // || ((len_ == that.len_) && (0 == memcmp(buf_, that.c_str(),
- // len_)));
- // }
-
// strict less than. If @this equals to @that, return false.
FORCE_INLINE bool less_than(const String &that) const {
- if (len_ == 0 || that.len_ == 0) {
+ if (len_ == -1) {
+ return that.len_ != -1;
+ }
+ if (that.len_ == -1) {
+ return false;
+ }
+
+ if (len_ == 0) {
+ return that.len_ > 0;
+ }
+
+ if (that.len_ == 0) {
return false;
}
- uint32_t min_len = std::min(len_, that.len_);
- int cmp_res = memcmp(buf_, that.buf_, min_len);
+
+ const int32_t min_len = std::min(len_, that.len_);
+ const int cmp_res = memcmp(buf_, that.buf_,
static_cast<size_t>(min_len));
if (cmp_res < 0) {
return true;
} else if (cmp_res > 0) {
@@ -127,16 +143,26 @@ struct String {
// return < 0, if this < that
// return > 0, if this > that
FORCE_INLINE int compare(const String &that) const {
- if (len_ == 0 || that.len_ == 0) {
- return 0;
- }
- uint32_t min_len = std::min(len_, that.len_);
- int cmp_res = memcmp(buf_, that.buf_, min_len);
- if (cmp_res == 0) {
- return len_ - that.len_;
- } else {
- return cmp_res;
+ // Rule 1: nullptr is less than everything
+ if (len_ == -1 && that.len_ == -1) return 0;
+ if (len_ == -1) return -1;
+ if (that.len_ == -1) return 1;
+
+ // Rule 2: empty is less than normal string
+ if (len_ == 0 && that.len_ == 0) return 0;
+ if (len_ == 0) return -1;
+ if (that.len_ == 0) return 1;
+
+ // Rule 3: normal string compare (lexicographical)
+ const int32_t min_len = std::min(len_, that.len_);
+
+ const int cmp_res = memcmp(buf_, that.buf_,
static_cast<size_t>(min_len));
+ if (cmp_res != 0) {
+ return cmp_res; // >0 / <0
}
+
+ // Prefix equal, shorter one is less
+ return len_ - that.len_;
}
FORCE_INLINE void max(const String &that, common::PageArena &pa) {
@@ -162,15 +188,17 @@ struct String {
return false;
}
- int min_len = std::min(this->len_, other.len_);
- int cmp = std::memcmp(this->buf_, other.buf_, min_len);
+ const int min_len = std::min(this->len_, other.len_);
+ const int cmp = std::memcmp(this->buf_, other.buf_,
static_cast<size_t>(min_len));
if (cmp != 0) {
return cmp < 0;
}
return this->len_ < other.len_;
}
- std::string to_std_string() const { return std::string(buf_, len_); }
+ std::string to_std_string() const {
+ return {buf_, static_cast<size_t>(len_)};
+ }
#ifndef NDEBUG
friend std::ostream &operator<<(std::ostream &os, const String &s) {
diff --git a/cpp/src/common/error_info/error_info.h
b/cpp/src/common/error_info/error_info.h
index 7ae2a5b1..e1a20cc9 100644
--- a/cpp/src/common/error_info/error_info.h
+++ b/cpp/src/common/error_info/error_info.h
@@ -1,21 +1,21 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* License); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
#ifndef COMMON_ERROR_INFO_H
#define COMMON_ERROR_INFO_H
@@ -25,8 +25,9 @@
namespace error_info {
enum E_CODE {
-#define ERRNO(name, val, desc) name = val
+#define ERRNO(name, val, desc) name = val,
#include "error_define.inc"
+
#undef ERRNO
};
@@ -35,6 +36,12 @@ void print_err_info();
void set_err_no(int error);
void set_err_msg(const std::string& msg);
void set_err_info(int error, const std::string& msg);
-}
+
+#define RETURN_ERR(code, msg) \
+ do { \
+ ::error_info::set_err_info((code), msg); \
+ return (code); \
+ } while (0)
+} // namespace error_info
#endif
\ No newline at end of file
diff --git a/cpp/src/reader/bloom_filter.cc b/cpp/src/reader/bloom_filter.cc
index 174791dd..5093615c 100644
--- a/cpp/src/reader/bloom_filter.cc
+++ b/cpp/src/reader/bloom_filter.cc
@@ -70,7 +70,7 @@ double math_log(double in) {
#endif
/* ================ BitSet ================ */
-void BitSet::to_bytes(uint8_t *&ret_bytes, int32_t &ret_len) const {
+void BitSet::to_bytes(uint8_t *&ret_bytes, size_t &ret_len) const {
int32_t words_in_use = get_words_in_use();
if (words_in_use == 0) {
return;
@@ -83,20 +83,21 @@ void BitSet::to_bytes(uint8_t *&ret_bytes, int32_t
&ret_len) const {
x = x >> 8;
}
- uint8_t *res =
- (uint8_t *)mem_alloc(sizeof(uint8_t) * len, MOD_BLOOM_FILTER);
+ auto *res =
+ static_cast<uint8_t *>(
+ mem_alloc(sizeof(uint8_t) * len, MOD_BLOOM_FILTER));
int32_t res_pos = 0;
for (int32_t w = 0; w < words_in_use - 1; w++) {
uint64_t word = words_[w];
for (int b = 0; b < 8; b++) {
- *(res + res_pos) = (uint8_t)(word & 0xFF);
+ *(res + res_pos) = static_cast<uint8_t>(word & 0xFF);
word = word >> 8;
res_pos++;
}
}
uint64_t last_word = words_[words_in_use - 1];
for (; res_pos < len; res_pos++) {
- *(res + res_pos) = (uint8_t)(last_word & 0xFF);
+ *(res + res_pos) = static_cast<uint8_t>(last_word & 0xFF);
last_word = last_word >> 8;
}
@@ -195,7 +196,7 @@ int BloomFilter::add_path_entry(const String &device_name,
String entry = get_entry_string(device_name, measurement_name);
if (IS_NULL(entry.buf_)) {
- return E_OOM;
+ return error_info::E_OOM;
}
for (uint32_t i = 0; i < hash_func_count_; i++) {
@@ -204,13 +205,13 @@ int BloomFilter::add_path_entry(const String &device_name,
bitset_.set(hv);
}
free_entry_buf(entry.buf_);
- return E_OK;
+ return error_info::E_OK;
}
int BloomFilter::serialize_to(ByteStream &out) {
- int ret = E_OK;
+ int ret = error_info::E_OK;
uint8_t *filter_data_bytes = nullptr;
- int32_t filter_data_bytes_len = 0;
+ size_t filter_data_bytes_len = 0;
bitset_.to_bytes(filter_data_bytes, filter_data_bytes_len);
if (RET_FAIL(
SerializationUtil::write_var_uint(filter_data_bytes_len, out))) {
@@ -227,7 +228,7 @@ int BloomFilter::serialize_to(ByteStream &out) {
}
int BloomFilter::deserialize_from(ByteStream &in) {
- int ret = E_OK;
+ int ret = error_info::E_OK;
uint32_t filter_data_bytes_len = 0;
uint32_t ret_read_len = 0;
uint8_t *filter_data = nullptr;
diff --git a/cpp/src/reader/bloom_filter.h b/cpp/src/reader/bloom_filter.h
index 805551a8..ba3394ce 100644
--- a/cpp/src/reader/bloom_filter.h
+++ b/cpp/src/reader/bloom_filter.h
@@ -82,9 +82,9 @@ class BitSet {
}
return 0;
}
- void to_bytes(uint8_t *&ret_bytes, int32_t &ret_len) const;
+ void to_bytes(uint8_t *&ret_bytes, size_t &ret_len) const;
void revert_bytes(uint8_t *bytes) { common::mem_free(bytes); }
- int from_bytes(uint8_t *filter_data, uint32_t filter_data_bytes_len);
+ int from_bytes(uint8_t *filter_data, size_t filter_data_bytes_len);
private:
uint64_t *words_;
