This is an automated email from the ASF dual-hosted git repository. colinlee pushed a commit to branch rc/2.2.0_bak in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 5eaf356a67194bf66977b63bee6258e61ac46ad1 Author: Hongzhi Gao <[email protected]> AuthorDate: Tue Dec 2 12:26:36 2025 +0800 Feature/cpp removed antlr4 (#648) * Supports compilation without antlr4 * Adjusted build tools * fix pom * remove compressor depencency * Supports compilation without compressor algorithm * Supports compilation without compressor algorithm * fix some issue * fix libtsfile_init() * spotless apply --- cpp/CMakeLists.txt | 30 +- cpp/build.sh | 66 +++- cpp/pom.xml | 12 + cpp/src/CMakeLists.txt | 110 +++++- cpp/src/common/device_id.cc | 404 +++++++++++++++++++++ cpp/src/common/device_id.h | 320 ++++++---------- cpp/src/common/global.cc | 8 + cpp/src/common/path.h | 7 + cpp/src/compress/CMakeLists.txt | 55 ++- cpp/src/compress/{compressor.c => compressor.cc} | 0 cpp/src/compress/compressor_factory.h | 40 +- cpp/src/cwrapper/tsfile_cwrapper.cc | 2 + cpp/src/reader/meta_data_querier.h | 2 + cpp/src/writer/tsfile_writer.cc | 16 +- cpp/test/CMakeLists.txt | 57 ++- .../reader/table_view/tsfile_reader_table_test.cc | 5 +- .../reader/tree_view/tsfile_reader_tree_test.cc | 8 +- cpp/test/reader/tsfile_reader_test.cc | 1 + .../writer/table_view/tsfile_writer_table_test.cc | 7 +- cpp/test/writer/tsfile_writer_test.cc | 4 + cpp/third_party/CMakeLists.txt | 26 +- 21 files changed, 886 insertions(+), 294 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 51f69e83..e5af0432 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -88,6 +88,23 @@ if (NOT WIN32) endif () endif () +option(BUILD_TEST "Build tests" OFF) +message("cmake using: BUILD_TEST=${BUILD_TEST}") + +option(ENABLE_ANTLR4 "Enable ANTLR4 runtime" ON) +message("cmake using: ENABLE_ANTLR4=${ENABLE_ANTLR4}") + +option(ENABLE_SNAPPY "Enable Google Snappy compression" ON) +message("cmake using: ENABLE_SNAPPY=${ENABLE_SNAPPY}") + +option(ENABLE_LZ4 "Enable LZ4 compression" ON) +message("cmake using: ENABLE_LZ4=${ENABLE_LZ4}") + +option(ENABLE_LZOKAY "Enable LZOKAY compression" ON) +message("cmake using: ENABLE_LZOKAY=${ENABLE_LZOKAY}") + +option(ENABLE_ZLIB "Enable Zlib compression" ON) +message("cmake using: ENABLE_ZLIB=${ENABLE_ZLIB}") # All libs will be stored here, including libtsfile, compress-encoding lib. set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) @@ -106,9 +123,14 @@ add_subdirectory(third_party) set(CMAKE_CXX_FLAGS "${SAVED_CXX_FLAGS}") add_subdirectory(src) -add_subdirectory(test) -add_subdirectory(examples) -if (TESTS_ENABLED) - add_dependencies(TsFile_Test tsfile) +if (BUILD_TEST) + add_subdirectory(test) + if (TESTS_ENABLED) + add_dependencies(TsFile_Test tsfile) + endif () +else() + message("BUILD_TEST is OFF, skipping test directory") endif () +add_subdirectory(examples) + diff --git a/cpp/build.sh b/cpp/build.sh index e7d4d01f..809e6733 100644 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -25,9 +25,20 @@ use_cpp11=1 enable_cov=0 debug_se=0 run_cov_only=0 +enable_antlr4=ON + +enable_snappy=ON +enable_lz4=ON +enable_lzokay=ON +enable_zlib=ON shell_dir=$(cd "$(dirname "$0")";pwd) +# 添加get_key_value函数 +get_key_value() { + echo "${1#*=}" +} + function print_config() { echo "build_type=$build_type" @@ -35,12 +46,17 @@ function print_config() echo "use_cpp11=$use_cpp11" echo "enable_cov=$enable_cov" echo "enable_asan=$enable_asan" + echo "enable_antlr4=$enable_antlr4" + echo "enable_snappy=$enable_snappy" + echo "enable_lz4=$enable_lz4" + echo "enable_lzokay=$enable_lzokay" + echo "enable_zlib=$enable_zlib" } function run_test_for_cov() { # sh ${shell_dir}/scripts/regression_unittest.sh - sh ${shell_dir}/test/libtsfile_test/run_sdk_tests.sh + sh ${shell_dir}/test/libtsfile_test/run_sdk_tests.sh } parse_options() @@ -53,20 +69,40 @@ parse_options() run_cov) run_cov_only=1;; -t=*) - build_type=`get_key_value "$1"`;; + build_type=$(get_key_value "$1");; -t) shift - build_type=`get_key_value "$1"`;; + build_type=$(get_key_value "$1");; -a=*) - enable_asan=`get_key_value "$1"`;; + enable_asan=$(get_key_value "$1");; -a) shift - enable_asan=`get_key_value "$1"`;; + enable_asan=$(get_key_value "$1");; -c=*) - enable_cov=`get_key_value "$1"`;; + enable_cov=$(get_key_value "$1");; -c) shift - enable_cov=`get_key_value "$1"`;; + enable_cov=$(get_key_value "$1");; + --enable-antlr4=*) + enable_antlr4=$(get_key_value "$1");; + --enable-snappy=*) + enable_snappy=$(get_key_value "$1");; + --enable-lz4=*) + enable_lz4=$(get_key_value "$1");; + --enable-lzokay=*) + enable_lzokay=$(get_key_value "$1");; + --enable-zlib=*) + enable_zlib=$(get_key_value "$1");; + --disable-antlr4) + enable_antlr4=OFF;; + --disable-snappy) + enable_snappy=OFF;; + --disable-lz4) + enable_lz4=OFF;; + --disable-lzokay) + enable_lzokay=OFF;; + --disable-zlib) + enable_zlib=OFF;; #-h | --help) # usage # exit 0;; @@ -127,14 +163,12 @@ cmake ../../ \ -DUSE_CPP11=$use_cpp11 \ -DENABLE_COV=$enable_cov \ -DDEBUG_SE=$debug_se \ - -DBUILD_TSFILE_ONLY=$build_tsfile_only + -DENABLE_ANTLR4=$enable_antlr4 \ + -DBUILD_TSFILE_ONLY=$build_tsfile_only \ + -DENABLE_SNAPPY=$enable_snappy \ + -DENABLE_LZ4=$enable_lz4 \ + -DENABLE_LZOKAY=$enable_lzokay \ + -DENABLE_ZLIB=$enable_zlib VERBOSE=1 make -VERBOSE=1 make install - - - - - - - +VERBOSE=1 make install \ No newline at end of file diff --git a/cpp/pom.xml b/cpp/pom.xml index fa6b0070..8efae178 100644 --- a/cpp/pom.xml +++ b/cpp/pom.xml @@ -34,6 +34,12 @@ <coverage.enabled>OFF</coverage.enabled> <build.type>Release</build.type> <enable.asan>OFF</enable.asan> + <build.test>ON</build.test> + <enable.snappy>ON</enable.snappy> + <enable.lz4>ON</enable.lz4> + <enable.lzokay>ON</enable.lzokay> + <enable.zlib>ON</enable.zlib> + <enable.antlr4>ON</enable.antlr4> </properties> <build> <sourceDirectory>${project.basedir}</sourceDirectory> @@ -74,6 +80,12 @@ <option>-DCOV_ENABLED=${coverage.enabled}</option> <option>-DCMAKE_BUILD_TYPE=${build.type}</option> <option>-DENABLE_ASAN=${enable.asan}</option> + <option>-DBUILD_TEST=${build.test}</option> + <option>-DENABLE_ANTLR4=${enable.antlr4}</option> + <option>-DENABLE_SNAPPY=${enable.snappy}</option> + <option>-DENABLE_LZ4=${enable.lz4}</option> + <option>-DENABLE_LZOKAY=${enable.lzokay}</option> + <option>-DENABLE_ZLIB=${enable.zlib}</option> </options> <sourcePath/> <targetPath/> diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 86c67f2b..e922836b 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -25,27 +25,74 @@ if(POLICY CMP0079) cmake_policy(SET CMP0079 NEW) endif() +option(ENABLE_SNAPPY "Enable Google Snappy compression" ON) +message("cmake using: ENABLE_SNAPPY=${ENABLE_SNAPPY}") + +option(ENABLE_LZ4 "Enable LZ4 compression" ON) +message("cmake using: ENABLE_LZ4=${ENABLE_LZ4}") + +option(ENABLE_LZOKAY "Enable LZOKAY compression" ON) +message("cmake using: ENABLE_LZOKAY=${ENABLE_LZOKAY}") + +option(ENABLE_ZLIB "Enable Zlib compression" ON) +message("cmake using: ENABLE_ZLIB=${ENABLE_ZLIB}") + message("Running in src directory") if (${COV_ENABLED}) add_compile_options(-fprofile-arcs -ftest-coverage) endif () -add_definitions(-DANTLR4CPP_STATIC) -set(ANTLR4_WITH_STATIC_CRT OFF) +if (ENABLE_ANTLR4) + add_definitions(-DANTLR4CPP_STATIC) + set(ANTLR4_WITH_STATIC_CRT OFF) + message("ANTLR4 is enabled, adding ANTLR4 definitions") +else() + message("ANTLR4 is disabled") +endif() set(PROJECT_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/src - ${THIRD_PARTY_INCLUDE}/google_snappy - ${CMAKE_SOURCE_DIR}/third_party/lz4 - ${CMAKE_SOURCE_DIR}/third_party/lzokay - ${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13 - ${CMAKE_SOURCE_DIR}/third_party/google_snappy - ${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src ) +if (ENABLE_SNAPPY) + list(APPEND PROJECT_INCLUDE_DIR + ${THIRD_PARTY_INCLUDE}/google_snappy + ${CMAKE_SOURCE_DIR}/third_party/google_snappy + ) +endif() + +if (ENABLE_LZ4) + list(APPEND PROJECT_INCLUDE_DIR + ${CMAKE_SOURCE_DIR}/third_party/lz4 + ) +endif() + +if (ENABLE_LZOKAY) + list(APPEND PROJECT_INCLUDE_DIR + ${CMAKE_SOURCE_DIR}/third_party/lzokay + ) +endif() + +if (ENABLE_ZLIB) + list(APPEND PROJECT_INCLUDE_DIR + ${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13 + ) +endif() + +if (ENABLE_ANTLR4) + list(APPEND PROJECT_INCLUDE_DIR + ${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src + ) + message("Adding ANTLR4 include directory") +endif() + include_directories(${PROJECT_INCLUDE_DIR}) -add_subdirectory(parser) +if (ENABLE_ANTLR4) + add_subdirectory(parser) + message("Adding parser subdirectory") +endif() + add_subdirectory(common) add_subdirectory(compress) add_subdirectory(cwrapper) @@ -55,24 +102,54 @@ add_subdirectory(reader) add_subdirectory(utils) add_subdirectory(writer) +set(COMPRESSION_LIBS "") + +if (ENABLE_SNAPPY) + list(APPEND COMPRESSION_LIBS snappy) +endif() + +if (ENABLE_LZ4) + list(APPEND COMPRESSION_LIBS LZ4) +endif() + +if (ENABLE_LZOKAY) + list(APPEND COMPRESSION_LIBS lzokay) +endif() + +if (ENABLE_ZLIB) + list(APPEND COMPRESSION_LIBS zlibstatic) +endif() + +message("Compression libraries: ${COMPRESSION_LIBS}") + +if (ENABLE_ANTLR4) + target_link_libraries(parser_obj antlr4_static) + message("Linking parser_obj with antlr4_static") +endif() -set(COMPRESSION_LIBS snappy LZ4 lzokay zlibstatic) -target_link_libraries(parser_obj antlr4_static) target_link_libraries(compress_obj ${COMPRESSION_LIBS}) target_link_libraries(common_obj ${COMPRESSION_LIBS}) target_link_libraries(read_obj ${COMPRESSION_LIBS}) target_link_libraries(write_obj ${COMPRESSION_LIBS}) add_library(tsfile SHARED) + if (${COV_ENABLED}) message("Enable code cov...") - target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj -lgcov) + if (ENABLE_ANTLR4) + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj -lgcov) + else() + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj -lgcov) + endif() else() message("Disable code cov...") - target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj) + if (ENABLE_ANTLR4) + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj) + else() + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj) + endif() endif() -# to trigger copy headers add_dependencies(tsfile utils_obj encoding_obj) set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION}) @@ -80,7 +157,4 @@ set(LIBTSFILE_SO_VERSION ${TsFile_CPP_VERSION}) set_target_properties(tsfile PROPERTIES VERSION ${LIBTSFILE_PROJECT_VERSION}) set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION}) -install(TARGETS tsfile LIBRARY DESTINATION ${LIBRARY_OUTPUT_PATH}) - - - +install(TARGETS tsfile LIBRARY DESTINATION ${LIBRARY_OUTPUT_PATH}) \ No newline at end of file diff --git a/cpp/src/common/device_id.cc b/cpp/src/common/device_id.cc new file mode 100644 index 00000000..263cb675 --- /dev/null +++ b/cpp/src/common/device_id.cc @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "common/device_id.h" + +#include <algorithm> +#include <cctype> +#include <numeric> +#include <stdexcept> +#include <unordered_set> + +#include "constant/tsfile_constant.h" +#ifdef ENABLE_ANTLR4 +#include "parser/path_nodes_generator.h" +#endif +#include "utils/errno_define.h" + +namespace storage { + +// IDeviceID implementation +IDeviceID::IDeviceID() : empty_segments_() {} + +int IDeviceID::serialize(common::ByteStream& write_stream) { return 0; } +int IDeviceID::deserialize(common::ByteStream& read_stream) { return 0; } +std::string IDeviceID::get_table_name() { return ""; } +int IDeviceID::segment_num() { return 0; } +const std::vector<std::string*>& IDeviceID::get_segments() const { + return empty_segments_; +} +std::string IDeviceID::get_device_name() const { return ""; } +bool IDeviceID::operator<(const IDeviceID& other) { return false; } +bool IDeviceID::operator==(const IDeviceID& other) { return false; } +bool IDeviceID::operator!=(const IDeviceID& other) { return false; } + +// IDeviceIDComparator implementation +bool IDeviceIDComparator::operator()( + const std::shared_ptr<IDeviceID>& lhs, + const std::shared_ptr<IDeviceID>& rhs) const { + return *lhs < *rhs; +} + +// StringArrayDeviceID implementation +StringArrayDeviceID::StringArrayDeviceID( + const std::vector<std::string>& segments) + : segments_(formalize(segments)) {} + +StringArrayDeviceID::StringArrayDeviceID(const std::string& device_id_string) { + auto segments = split_device_id_string(device_id_string); + segments_.reserve(segments.size()); + for (const auto& segment : segments) { + segments_.push_back(new std::string(segment)); + } +} + +StringArrayDeviceID::StringArrayDeviceID( + const std::vector<std::string*>& segments) { + segments_.reserve(segments.size()); + for (const auto& segment : segments) { + segments_.push_back(segment == nullptr ? nullptr + : new std::string(*segment)); + } +} + +StringArrayDeviceID::StringArrayDeviceID() : segments_() {} + +StringArrayDeviceID::~StringArrayDeviceID() { + for (const auto& segment : segments_) { + delete segment; + } + for (const auto& prefix_segments : prefix_segments_) { + delete prefix_segments; + } +} + +std::string StringArrayDeviceID::get_device_name() const { + if (segments_.empty()) { + return ""; + } + // Builds device name by concatenating segments with '.' delimiter, + // handling null segments by replacing them with "null" + return std::accumulate(std::next(segments_.begin()), segments_.end(), + segments_.front() ? *segments_.front() : "null", + [](std::string acc, const std::string* segment) { + return std::move(acc) + "." + + (segment ? *segment : "null"); + }); +} + +void StringArrayDeviceID::init_prefix_segments() { +#ifdef ENABLE_ANTLR4 + auto splits = storage::PathNodesGenerator::invokeParser(*segments_[0]); +#else + auto splits = split_string(*segments_[0], '.'); +#endif + for (int i = 0; i < splits.size(); ++i) { + prefix_segments_.push_back(new std::string(splits[i])); + } +} + +int StringArrayDeviceID::serialize(common::ByteStream& write_stream) { + int ret = common::E_OK; + if (RET_FAIL(common::SerializationUtil::write_var_uint(segment_num(), + write_stream))) { + return ret; + } + for (const auto& segment : segments_) { + if (RET_FAIL(common::SerializationUtil::write_var_char_ptr( + segment, write_stream))) { + return ret; + } + } + return ret; +} + +int StringArrayDeviceID::deserialize(common::ByteStream& read_stream) { + int ret = common::E_OK; + uint32_t num_segments; + if (RET_FAIL(common::SerializationUtil::read_var_uint(num_segments, + read_stream))) { + return ret; + } + + for (auto& segment : segments_) { + if (segment != nullptr) { + delete segment; + } + } + + segments_.clear(); + for (uint32_t i = 0; i < num_segments; ++i) { + std::string* segment; + if (RET_FAIL(common::SerializationUtil::read_var_char_ptr( + segment, read_stream))) { + delete segment; + return ret; + } + segments_.push_back(segment); + } + return ret; +} + +std::string StringArrayDeviceID::get_table_name() { + return segments_.empty() ? "" : *segments_[0]; +} + +int StringArrayDeviceID::segment_num() { + return static_cast<int>(segments_.size()); +} + +const std::vector<std::string*>& StringArrayDeviceID::get_segments() const { + return segments_; +} + +bool StringArrayDeviceID::operator<(const IDeviceID& other) { + auto other_segments = other.get_segments(); + return std::lexicographical_compare( + segments_.begin(), segments_.end(), other_segments.begin(), + other_segments.end(), [](const std::string* a, const std::string* b) { + if (a == nullptr && b == nullptr) return false; // equal + if (a == nullptr) return true; // nullptr < any string + if (b == nullptr) return false; // any string > nullptr + return *a < *b; + }); +} + +bool StringArrayDeviceID::operator==(const IDeviceID& other) { + auto other_segments = other.get_segments(); + return (segments_.size() == other_segments.size()) && + std::equal(segments_.begin(), segments_.end(), + other_segments.begin(), + [](const std::string* a, const std::string* b) { + if (a == nullptr && b == nullptr) return true; + if (a == nullptr || b == nullptr) return false; + return *a == *b; + }); +} + +bool StringArrayDeviceID::operator!=(const IDeviceID& other) { + return !(*this == other); +} + +std::vector<std::string*> StringArrayDeviceID::formalize( + const std::vector<std::string>& segments) { + std::vector<std::string*> result; + result.reserve(segments.size()); + for (const auto& segment : segments) { + result.emplace_back(new std::string(segment)); + } + return result; +} + +std::vector<std::string> StringArrayDeviceID::split_device_id_string( + const std::string& device_id_string) { +#ifdef ENABLE_ANTLR4 + auto splits = storage::PathNodesGenerator::invokeParser(device_id_string); + return split_device_id_string(splits); +#else + return split_string(device_id_string, '.'); +#endif +} + +std::vector<std::string> StringArrayDeviceID::split_device_id_string( + const std::vector<std::string>& splits) { + size_t segment_cnt = splits.size(); + std::vector<std::string> final_segments; + + if (segment_cnt == 0) { + return final_segments; + } + + if (segment_cnt == 1) { + // "root" -> {"root"} + final_segments.push_back(splits[0]); + } else if (segment_cnt < + static_cast<size_t>(storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + + 1)) { + // "root.a" -> {"root", "a"} + // "root.a.b" -> {"root.a", "b"} + std::string table_name = std::accumulate( + splits.begin(), splits.end() - 1, std::string(), + [](const std::string& a, const std::string& b) { + return a.empty() ? b : a + storage::PATH_SEPARATOR + b; + }); + final_segments.push_back(table_name); + final_segments.push_back(splits.back()); + } else { + // "root.a.b.c" -> {"root.a.b", "c"} + // "root.a.b.c.d" -> {"root.a.b", "c", "d"} + std::string table_name = std::accumulate( + splits.begin(), + splits.begin() + storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, + std::string(), [](const std::string& a, const std::string& b) { + return a.empty() ? b : a + storage::PATH_SEPARATOR + b; + }); + + final_segments.emplace_back(std::move(table_name)); + final_segments.insert( + final_segments.end(), + splits.begin() + storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, + splits.end()); + } + + return final_segments; +} + +std::vector<std::string> IDeviceID::split_string(const std::string& str, + char delimiter) { + std::vector<std::string> tokens; + + // Reject newlines in path explicitly (illegal path name). + if (str.find('\n') != std::string::npos || + str.find('\r') != std::string::npos) { + throw std::runtime_error("Path contains newline"); + } + + std::string token; + bool in_back_quotes = false; // Inside `quoted` section + bool in_double_quotes = false; // Inside "quoted" section + bool in_single_quotes = false; // Inside 'quoted' section + + for (size_t i = 0; i < str.length(); ++i) { + char c = str[i]; + + // Toggle quote state when encountering a quote character outside other + // quote types. + if (c == '`' && !in_double_quotes && !in_single_quotes) { + in_back_quotes = !in_back_quotes; + token += c; // preserve the backtick character + } else if (c == '"' && !in_back_quotes && !in_single_quotes) { + in_double_quotes = !in_double_quotes; + token += c; // preserve + } else if (c == '\'' && !in_back_quotes && !in_double_quotes) { + in_single_quotes = !in_single_quotes; + token += c; // preserve + } else if (c == delimiter && !in_back_quotes && !in_double_quotes && + !in_single_quotes) { + // delimiter outside quotes -> split + if (!token.empty()) { + validate_identifier(token); + tokens.push_back(unquote_identifier(token)); + token.clear(); + } else { + tokens.push_back(token); + token.clear(); + } + } else { + // preserve all characters verbatim (including backslashes and + // doubled backticks) + token += c; + } + } + + // Unmatched quotes are errors + if (in_back_quotes || in_double_quotes || in_single_quotes) { + throw std::runtime_error("Unmatched quotes in path"); + } + + // Add the last token if non-empty (mirror original behaviour) + if (!token.empty()) { + validate_identifier(token); + tokens.push_back(unquote_identifier(token)); + } + + return tokens; +} + +std::string IDeviceID::unquote_identifier(const std::string& identifier) { + if (identifier.length() >= 2) { + char first = identifier.front(); + char last = identifier.back(); + + if ((first == '`' && last == '`') || (first == '"' && last == '"') || + (first == '\'' && last == '\'')) { + std::string inner = identifier.substr(1, identifier.length() - 2); + + // Lowercase copy for case-insensitive comparison + std::string lower_inner = inner; + std::transform(lower_inner.begin(), lower_inner.end(), + lower_inner.begin(), + [](unsigned char ch) { return std::tolower(ch); }); + + static const std::unordered_set<std::string> keywords = { + "select", "device", "drop_trigger", "and", + "or", "not", "null", "contains"}; + + // If the identifier is enclosed in backticks AND inner is a + // keyword, unquote it (testcase: `select` -> select) + if (first == '`' && keywords.find(lower_inner) != keywords.end()) { + return inner; + } + + // Otherwise: keep original quoting + } + } + + return identifier; +} + +void IDeviceID::validate_identifier(const std::string& identifier) { + if (identifier.empty()) return; + + bool quoted = (identifier.size() >= 2 && + ((identifier.front() == '`' && identifier.back() == '`') || + (identifier.front() == '"' && identifier.back() == '"') || + (identifier.front() == '\'' && identifier.back() == '\''))); + + if (quoted) return; // quoted identifiers are always accepted + + // 1. Pure digits - unquoted numeric literals not allowed as identifiers + bool all_digits = true; + for (char c : identifier) { + if (!std::isdigit((unsigned char)c)) { + all_digits = false; + break; + } + } + if (all_digits) { + throw std::runtime_error("Unquoted pure digits are illegal"); + } + + // 2. Numeric-like illegal patterns (e.g., 0e38, 00.12 not allowed unquoted) + if (!identifier.empty() && std::isdigit((unsigned char)identifier[0])) { + throw std::runtime_error("Identifier cannot start with a digit"); + } + + // 3. Illegal wildcards in unquoted identifiers + if (identifier.find('%') != std::string::npos) { + throw std::runtime_error("Illegal wildcard in unquoted identifier"); + } + + // 4. Asterisk (*) validation: if present, must consist only of asterisks + if (identifier.find('*') != std::string::npos) { + // Check if all characters are asterisks + bool all_asterisks = true; + for (char c : identifier) { + if (c != '*') { + all_asterisks = false; + break; + } + } + if (!all_asterisks) { + throw std::runtime_error( + "Asterisk wildcard must be the only character type"); + } + } +} + +} // namespace storage \ No newline at end of file diff --git a/cpp/src/common/device_id.h b/cpp/src/common/device_id.h index 613ebc56..50d0d010 100644 --- a/cpp/src/common/device_id.h +++ b/cpp/src/common/device_id.h @@ -20,40 +20,100 @@ #ifndef COMMON_DEVICE_ID_H #define COMMON_DEVICE_ID_H -#include <algorithm> -#include <cstdint> -#include <cstring> #include <memory> -#include <numeric> #include <string> #include <vector> #include "common/allocator/byte_stream.h" -#include "constant/tsfile_constant.h" -#include "parser/path_nodes_generator.h" -#include "utils/errno_define.h" namespace storage { + class IDeviceID { public: virtual ~IDeviceID() = default; - virtual int serialize(common::ByteStream& write_stream) { return 0; } - virtual int deserialize(common::ByteStream& read_stream) { return 0; } - virtual std::string get_table_name() { return ""; } - virtual int segment_num() { return 0; } - virtual const std::vector<std::string*>& get_segments() const { - return empty_segments_; - } - virtual std::string get_device_name() const { return ""; }; - virtual bool operator<(const IDeviceID& other) { return false; } - virtual bool operator==(const IDeviceID& other) { return false; } - virtual bool operator!=(const IDeviceID& other) { return false; } + virtual int serialize(common::ByteStream& write_stream); + virtual int deserialize(common::ByteStream& read_stream); + virtual std::string get_table_name(); + virtual int segment_num(); + virtual const std::vector<std::string*>& get_segments() const; + virtual std::string get_device_name() const; + virtual bool operator<(const IDeviceID& other); + virtual bool operator==(const IDeviceID& other); + virtual bool operator!=(const IDeviceID& other); virtual std::string* get_split_segname_at(int pos) { return nullptr; } virtual int get_split_seg_num() { return 0; } virtual void split_table_name() {} + /** + * Splits a string by delimiter while respecting quoted sections. + * Handles three quote types: backticks (`), double quotes ("), and single + * quotes ('). Delimiters inside quoted sections are treated as part of the + * token. + * + * Examples: + * - "a.b.c" -> ["a", "b", "c"] (with delimiter '.') + * - "a.`b.c`.d" -> ["a", "`b.c`", "d"] (preserves quotes, treats "b.c" as + * one token) + * - "table.`select`" -> ["table", "select"] (unquotes keywords in + * backticks) + * + * @param str The input string to split + * @param delimiter The character to split on (typically '.' for paths) + * @return Vector of tokens, with quotes preserved except for keyword + * unquoting + * @throws std::runtime_error if newlines found, quotes unmatched, or + * invalid identifier + */ + static std::vector<std::string> split_string(const std::string& str, + char delimiter); + /** + * Removes surrounding quotes from identifiers under specific conditions. + * Only unquotes backtick-enclosed identifiers if the inner content is a + * reserved keyword. Preserves quotes for all other cases (non-keywords, + * other quote types). + * + * Examples: + * - "`select`" -> "select" (keyword in backticks gets unquoted) + * - "`custom`" -> "`custom`" (non-keyword preserves backticks) + * - "'select'" -> "'select'" (single quotes always preserved) + * - "\"table\"" -> "\"table\"" (double quotes always preserved) + * + * @param identifier The potentially quoted identifier + * @return Unquoted string if conditions met, otherwise original identifier + */ + static std::string unquote_identifier(const std::string& identifier); + /** + * Validates identifier syntax according to specific rules. + * Quoted identifiers are always accepted. Unquoted identifiers must: + * - Not be pure digits (e.g., "123" is illegal) + * - Not start with a digit (e.g., "1table" is illegal) + * - Not contain wildcard '%' characters + * - If containing '*', must consist only of asterisks (e.g., "*", "**", + * "***" are allowed) + * + * Examples of illegal unquoted identifiers: + * - "123" (pure digits) + * - "1column" (starts with digit) + * - "col%name" (contains wildcard) + * - "abc*" (contains asterisk but also other characters) + * - "a*b" (contains asterisk mixed with other characters) + * + * Examples of legal identifiers: + * - "table1" (normal identifier) + * - "*" (single asterisk wildcard) + * - "**" (multiple asterisks) + * - "***" (any number of consecutive asterisks) + * - "`123`" (quoted digits are allowed) + * - "`col%name`" (quoted wildcards allowed) + * - "`abc*`" (quoted asterisk allowed in any position) + * + * @param identifier The identifier to validate + * @throws std::runtime_error if identifier violates validation rules + */ + static void validate_identifier(const std::string& identifier); + protected: - IDeviceID() : empty_segments_() {} + IDeviceID(); private: const std::vector<std::string*> empty_segments_; @@ -61,142 +121,26 @@ class IDeviceID { struct IDeviceIDComparator { bool operator()(const std::shared_ptr<IDeviceID>& lhs, - const std::shared_ptr<IDeviceID>& rhs) const { - return *lhs < *rhs; - } + const std::shared_ptr<IDeviceID>& rhs) const; }; class StringArrayDeviceID : public IDeviceID { public: - explicit StringArrayDeviceID(const std::vector<std::string>& segments) - : segments_(formalize(segments)) {} - - explicit StringArrayDeviceID(const std::string& device_id_string) { - auto segments = split_device_id_string(device_id_string); - segments_.reserve(segments.size()); - for (const auto& segment : segments) { - segments_.push_back(new std::string(segment)); - } - } - - explicit StringArrayDeviceID(const std::vector<std::string*>& segments) { - segments_.reserve(segments.size()); - for (const auto& segment : segments) { - segments_.push_back(segment == nullptr ? nullptr - : new std::string(*segment)); - } - } - - explicit StringArrayDeviceID() : segments_() {} - - ~StringArrayDeviceID() override { - for (const auto& segment : segments_) { - delete segment; - } - for (const auto& prefix_segments : prefix_segments_) { - delete prefix_segments; - } - } - - std::string get_device_name() const override { - if (segments_.empty()) { - return ""; - } - - std::string result(*segments_.front()); - for (auto it = std::next(segments_.begin()); it != segments_.end(); - ++it) { - result += '.'; - if (*it != nullptr) { - result += **it; - } else { - result += "null"; - } - } - - return result; - }; - - int serialize(common::ByteStream& write_stream) override { - int ret = common::E_OK; - if (RET_FAIL(common::SerializationUtil::write_var_uint(segment_num(), - write_stream))) { - return ret; - } - for (const auto& segment : segments_) { - if (RET_FAIL(common::SerializationUtil::write_var_char_ptr( - segment, write_stream))) { - return ret; - } - } - return ret; - } - - int deserialize(common::ByteStream& read_stream) override { - int ret = common::E_OK; - uint32_t num_segments; - if (RET_FAIL(common::SerializationUtil::read_var_uint(num_segments, - read_stream))) { - return ret; - } - - for (auto& segment : segments_) { - if (segment != nullptr) { - delete segment; - } - } - - segments_.clear(); - for (uint32_t i = 0; i < num_segments; ++i) { - std::string* segment; - if (RET_FAIL(common::SerializationUtil::read_var_char_ptr( - segment, read_stream))) { - delete segment; - return ret; - } - segments_.push_back(segment); - } - return ret; - } - - std::string get_table_name() override { - return segments_.empty() ? "" : *segments_[0]; - } - - int segment_num() override { return static_cast<int>(segments_.size()); } - - const std::vector<std::string*>& get_segments() const override { - return segments_; - } - - bool operator<(const IDeviceID& other) override { - auto other_segments = other.get_segments(); - return std::lexicographical_compare( - segments_.begin(), segments_.end(), other_segments.begin(), - other_segments.end(), - [](const std::string* a, const std::string* b) { - if (a == nullptr && b == nullptr) return false; // equal - if (a == nullptr) return true; // nullptr < any string - if (b == nullptr) return false; // any string > nullptr - return *a < *b; - }); - } - - bool operator==(const IDeviceID& other) override { - auto other_segments = other.get_segments(); - return (segments_.size() == other_segments.size()) && - std::equal(segments_.begin(), segments_.end(), - other_segments.begin(), - [](const std::string* a, const std::string* b) { - if (a == nullptr && b == nullptr) return true; - if (a == nullptr || b == nullptr) return false; - return *a == *b; - }); - } - - bool operator!=(const IDeviceID& other) override { - return !(*this == other); - } + explicit StringArrayDeviceID(const std::vector<std::string>& segments); + explicit StringArrayDeviceID(const std::string& device_id_string); + explicit StringArrayDeviceID(const std::vector<std::string*>& segments); + explicit StringArrayDeviceID(); + ~StringArrayDeviceID() override; + + std::string get_device_name() const override; + int serialize(common::ByteStream& write_stream) override; + int deserialize(common::ByteStream& read_stream) override; + std::string get_table_name() override; + int segment_num() override; + const std::vector<std::string*>& get_segments() const override; + bool operator<(const IDeviceID& other) override; + bool operator==(const IDeviceID& other) override; + bool operator!=(const IDeviceID& other) override; void split_table_name() override { init_prefix_segments(); } @@ -220,79 +164,19 @@ class StringArrayDeviceID : public IDeviceID { private: std::vector<std::string*> segments_; + std::vector<std::string*> prefix_segments_; - void init_prefix_segments() { - auto splits = storage::PathNodesGenerator::invokeParser(*segments_[0]); - for (int i = 0; i < splits.size(); ++i) { - prefix_segments_.push_back(new std::string(splits[i])); - } - } - static std::vector<std::string*> formalize( - const std::vector<std::string>& segments) { - auto it = - std::find_if(segments.rbegin(), segments.rend(), - [](const std::string& seg) { return !seg.empty(); }); - std::vector<std::string> validate_segments(segments.begin(), it.base()); - std::vector<std::string*> result; - result.reserve(validate_segments.size()); - for (const auto& segment : validate_segments) { - result.emplace_back(new std::string(segment)); - } - return result; - } + void init_prefix_segments(); + static std::vector<std::string*> formalize( + const std::vector<std::string>& segments); static std::vector<std::string> split_device_id_string( - const std::string& device_id_string) { - auto splits = - storage::PathNodesGenerator::invokeParser(device_id_string); - return split_device_id_string(splits); - } - + const std::string& device_id_string); static std::vector<std::string> split_device_id_string( - const std::vector<std::string>& splits) { - size_t segment_cnt = splits.size(); - std::vector<std::string> final_segments; - - if (segment_cnt == 0) { - return final_segments; - } - - if (segment_cnt == 1) { - // "root" -> {"root"} - final_segments.push_back(splits[0]); - } else if (segment_cnt < - static_cast<size_t>( - storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1)) { - // "root.a" -> {"root", "a"} - // "root.a.b" -> {"root.a", "b"} - std::string table_name = std::accumulate( - splits.begin(), splits.end() - 1, std::string(), - [](const std::string& a, const std::string& b) { - return a.empty() ? b : a + storage::PATH_SEPARATOR + b; - }); - final_segments.push_back(table_name); - final_segments.push_back(splits.back()); - } else { - // "root.a.b.c" -> {"root.a.b", "c"} - // "root.a.b.c.d" -> {"root.a.b", "c", "d"} - std::string table_name = std::accumulate( - splits.begin(), - splits.begin() + storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, - std::string(), [](const std::string& a, const std::string& b) { - return a.empty() ? b : a + storage::PATH_SEPARATOR + b; - }); - - final_segments.emplace_back(std::move(table_name)); - final_segments.insert( - final_segments.end(), - splits.begin() + storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, - splits.end()); - } - - return final_segments; - } + const std::vector<std::string>& splits); }; + } // namespace storage #endif \ No newline at end of file diff --git a/cpp/src/common/global.cc b/cpp/src/common/global.cc index 21b3dad7..37b8c1bb 100644 --- a/cpp/src/common/global.cc +++ b/cpp/src/common/global.cc @@ -43,7 +43,11 @@ void init_config_value() { g_config_value_.chunk_group_size_threshold_ = 128 * 1024 * 1024; g_config_value_.time_encoding_type_ = TS_2DIFF; g_config_value_.time_data_type_ = INT64; +#ifdef ENABLE_LZ4 g_config_value_.time_compress_type_ = LZ4; +#else + g_config_value_.time_compress_type_ = UNCOMPRESSED; +#endif // Not support RLE yet. g_config_value_.boolean_encoding_type_ = PLAIN; g_config_value_.int32_encoding_type_ = TS_2DIFF; @@ -51,7 +55,11 @@ void init_config_value() { g_config_value_.float_encoding_type_ = GORILLA; g_config_value_.double_encoding_type_ = GORILLA; // Default compression type is LZ4 +#ifdef ENABLE_LZ4 g_config_value_.default_compression_type_ = LZ4; +#else + g_config_value_.default_compression_type_ = UNCOMPRESSED; +#endif } extern TSEncoding get_value_encoder(TSDataType data_type) { diff --git a/cpp/src/common/path.h b/cpp/src/common/path.h index 8318a473..a780b443 100644 --- a/cpp/src/common/path.h +++ b/cpp/src/common/path.h @@ -22,8 +22,10 @@ #include <string> #include "common/device_id.h" +#ifdef ENABLE_ANTLR4 #include "parser/generated/PathParser.h" #include "parser/path_nodes_generator.h" +#endif #include "utils/errno_define.h" namespace storage { @@ -47,8 +49,13 @@ struct Path { full_path_ = path_sc; device_id_ = std::make_shared<StringArrayDeviceID>(path_sc); } else { +#ifdef ENABLE_ANTLR4 std::vector<std::string> nodes = PathNodesGenerator::invokeParser(path_sc); +#else + std::vector<std::string> nodes = + IDeviceID::split_string(path_sc, '.'); +#endif if (nodes.size() > 1) { device_id_ = std::make_shared<StringArrayDeviceID>( std::vector<std::string>(nodes.begin(), diff --git a/cpp/src/compress/CMakeLists.txt b/cpp/src/compress/CMakeLists.txt index a2cda6b7..6b1d69d8 100644 --- a/cpp/src/compress/CMakeLists.txt +++ b/cpp/src/compress/CMakeLists.txt @@ -19,7 +19,56 @@ under the License. message("Running in src/storage/tsfile/compress directory") set(CMAKE_POSITION_INDEPENDENT_CODE ON) -aux_source_directory(. compress_SRC_LIST) + +# Initialize source file list +set(compress_SRC_LIST "compressor.cc") + +# Initialize header file list +set(HEADERS "") + +# Conditionally add Snappy compression +if(ENABLE_SNAPPY) + list(APPEND compress_SRC_LIST "snappy_compressor.cc") + list(APPEND HEADERS "snappy_compressor.h") +endif() + +# Conditionally add LZ4 compression +if(ENABLE_LZ4) + list(APPEND compress_SRC_LIST "lz4_compressor.cc") + list(APPEND HEADERS "lz4_compressor.h") +endif() + +# Conditionally add LZO-Kay compression +if(ENABLE_LZOKAY) + list(APPEND compress_SRC_LIST "lzo_compressor.cc") + list(APPEND HEADERS "lzo_compressor.h") +endif() + +# Conditionally add Zlib compression +if(ENABLE_ZLIB) + list(APPEND compress_SRC_LIST "gzip_compressor.cc") + list(APPEND HEADERS "gzip_compressor.h") +endif() + +# Add common headers (always copied) +file(GLOB COMMON_HEADERS "*.h") +foreach(header IN LISTS HEADERS) + list(REMOVE_ITEM COMMON_HEADERS ${header}) +endforeach() + +list(APPEND HEADERS ${COMMON_HEADERS}) + add_library(compress_obj OBJECT ${compress_SRC_LIST}) -file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") -copy_to_dir(${HEADERS} "compress_obj") + +# make header paths absolute (relative to this CMakeLists.txt) --- +set(ABS_HEADERS "") +foreach(h IN LISTS HEADERS) + if(IS_ABSOLUTE "${h}") + list(APPEND ABS_HEADERS "${h}") + else() + # prepend current source dir so copy_to_dir sees the real file path + list(APPEND ABS_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/${h}") + endif() +endforeach() +# call copy function with absolute/qualified header paths +copy_to_dir(${ABS_HEADERS} "compress_obj") \ No newline at end of file diff --git a/cpp/src/compress/compressor.c b/cpp/src/compress/compressor.cc similarity index 100% rename from cpp/src/compress/compressor.c rename to cpp/src/compress/compressor.cc diff --git a/cpp/src/compress/compressor_factory.h b/cpp/src/compress/compressor_factory.h index 4507db2f..ea4b464c 100644 --- a/cpp/src/compress/compressor_factory.h +++ b/cpp/src/compress/compressor_factory.h @@ -20,11 +20,23 @@ #ifndef COMPRESS_COMPRESSOR_FACTORY_H #define COMPRESS_COMPRESSOR_FACTORY_H +#include "uncompressed_compressor.h" + +#ifdef ENABLE_SNAPPY +#include "snappy_compressor.h" +#endif + +#ifdef ENABLE_GZIP #include "gzip_compressor.h" -#include "lz4_compressor.h" +#endif + +#ifdef ENABLE_LZOKAY #include "lzo_compressor.h" -#include "snappy_compressor.h" -#include "uncompressed_compressor.h" +#endif + +#ifdef ENABLE_LZ4 +#include "lz4_compressor.h" +#endif namespace storage { @@ -46,19 +58,35 @@ class CompressorFactory { if (type == common::UNCOMPRESSED) { ALLOC_AND_RETURN_COMPRESSPR(UncompressedCompressor); } else if (type == common::SNAPPY) { +#ifdef ENABLE_SNAPPY ALLOC_AND_RETURN_COMPRESSPR(SnappyCompressor); +#else + return nullptr; +#endif } else if (type == common::GZIP) { +#ifdef ENABLE_GZIP ALLOC_AND_RETURN_COMPRESSPR(GZIPCompressor); +#else + return nullptr; +#endif } else if (type == common::LZO) { +#ifdef ENABLE_LZOKAY ALLOC_AND_RETURN_COMPRESSPR(LZOCompressor); +#else + return nullptr; +#endif + } else if (type == common::LZ4) { +#ifdef ENABLE_LZ4 + ALLOC_AND_RETURN_COMPRESSPR(LZ4Compressor); +#else + return nullptr; +#endif } else if (type == common::SDT) { return nullptr; } else if (type == common::PAA) { return nullptr; } else if (type == common::PLA) { return nullptr; - } else if (type == common::LZ4) { - ALLOC_AND_RETURN_COMPRESSPR(LZ4Compressor); } else { ASSERT(false); return nullptr; @@ -70,4 +98,4 @@ class CompressorFactory { } // end namespace storage -#endif // COMPRESS_COMPRESSOR_FACTORY_H +#endif // COMPRESS_COMPRESSOR_FACTORY_H \ No newline at end of file diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index 8a8f462b..cdc30929 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -24,6 +24,8 @@ #include <unistd.h> #include <writer/tsfile_table_writer.h> +#include <set> + #include "common/tablet.h" #include "reader/result_set.h" #include "reader/tsfile_reader.h" diff --git a/cpp/src/reader/meta_data_querier.h b/cpp/src/reader/meta_data_querier.h index 525ecf86..be575323 100644 --- a/cpp/src/reader/meta_data_querier.h +++ b/cpp/src/reader/meta_data_querier.h @@ -20,6 +20,8 @@ #ifndef READER_META_DATA_QUERIER_H #define READER_META_DATA_QUERIER_H +#include <mutex> + #include "common/cache/lru_cache.h" #include "common/device_id.h" #include "device_meta_iterator.h" diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 3ae5cf28..7dece49e 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -30,20 +30,28 @@ using namespace common; namespace storage { + +namespace libtsfile { +bool g_s_is_inited = false; +} + int libtsfile_init() { - static bool g_s_is_inited = false; - if (g_s_is_inited) { + libtsfile::g_s_is_inited = false; + if (libtsfile::g_s_is_inited) { return E_OK; } ModStat::get_instance().init(); init_common(); - g_s_is_inited = true; + libtsfile::g_s_is_inited = true; return E_OK; } -void libtsfile_destroy() { ModStat::get_instance().destroy(); } +void libtsfile_destroy() { + ModStat::get_instance().destroy(); + libtsfile::g_s_is_inited = false; +} void set_page_max_point_count(uint32_t page_max_ponint_count) { config_set_page_max_point_count(page_max_ponint_count); diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 1beb310d..423381e4 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -70,7 +70,6 @@ else () return() endif () - message(STATUS "Adding test configurations...") set(LIB_TSFILE_SDK_DIR ${PROJECT_BINARY_DIR}/lib) @@ -79,14 +78,29 @@ message("LIB_TSFILE_SDK_DIR: ${LIB_TSFILE_SDK_DIR}") include_directories( ${LIBRARY_INCLUDE_DIR} ${THIRD_PARTY_INCLUDE} - ${THIRD_PARTY_INCLUDE}/google_snappy - ${CMAKE_SOURCE_DIR}/third_party/lz4 - ${CMAKE_SOURCE_DIR}/third_party/google_snappy - ${CMAKE_SOURCE_DIR}/third_party/lzokay - ${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13 - ${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src ) +if (ENABLE_SNAPPY) + include_directories(${THIRD_PARTY_INCLUDE}/google_snappy) + include_directories(${CMAKE_SOURCE_DIR}/third_party/google_snappy) +endif() + +if (ENABLE_LZ4) + include_directories(${CMAKE_SOURCE_DIR}/third_party/lz4) +endif() + +if (ENABLE_LZOKAY) + include_directories(${CMAKE_SOURCE_DIR}/third_party/lzokay) +endif() + +if (ENABLE_ZLIB) + include_directories(${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13) +endif() + +if (ENABLE_ANTLR4) + include_directories(${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src) +endif() + enable_testing() file(GLOB_RECURSE TEST_SRCS @@ -95,19 +109,40 @@ file(GLOB_RECURSE TEST_SRCS "utils/*_test.cc" "file/*_test.cc" "parser/*_test.cc" - "compress/*_test.cc" "reader/*_test.cc" "writer/*_test.cc" "cwrapper/*_test.cc" ) +if (ENABLE_SNAPPY) + file(GLOB_RECURSE SNAPPY_TEST_SRCS "compress/*snappy*_test.cc") + list(APPEND TEST_SRCS ${SNAPPY_TEST_SRCS}) +endif() + +if (ENABLE_LZ4) + file(GLOB_RECURSE LZ4_TEST_SRCS "compress/*lz4*_test.cc") + list(APPEND TEST_SRCS ${LZ4_TEST_SRCS}) +endif() + +if (ENABLE_LZOKAY) + file(GLOB_RECURSE LZOKAY_TEST_SRCS "compress/*lzo*_test.cc") + list(APPEND TEST_SRCS ${LZOKAY_TEST_SRCS}) +endif() + +if (ENABLE_ZLIB) + file(GLOB_RECURSE ZLIB_TEST_SRCS "compress/*gzip*_test.cc") + list(APPEND TEST_SRCS ${ZLIB_TEST_SRCS}) +endif() + if (${COV_ENABLED}) message("Enable code cov...") add_compile_options(-fprofile-arcs -ftest-coverage) endif () -add_definitions(-DANTLR4CPP_STATIC) -set(ANTLR4_WITH_STATIC_CRT OFF) +if (ENABLE_ANTLR4) + add_definitions(-DANTLR4CPP_STATIC) + set(ANTLR4_WITH_STATIC_CRT OFF) +endif() add_executable(TsFile_Test ${TEST_SRCS}) target_link_libraries( @@ -130,4 +165,4 @@ if (WIN32) endif () include(GoogleTest) -gtest_discover_tests(TsFile_Test) +gtest_discover_tests(TsFile_Test) \ No newline at end of file diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc index 4386b183..0ca38f53 100644 --- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc +++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc @@ -48,7 +48,10 @@ class TsFileTableReaderTest : public ::testing::Test { mode_t mode = 0666; write_file_.create(file_name_, flags, mode); } - void TearDown() override { remove(file_name_.c_str()); } + void TearDown() override { + remove(file_name_.c_str()); + libtsfile_destroy(); + } std::string file_name_; WriteFile write_file_; diff --git a/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc b/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc index 9dd482e8..a5c46102 100644 --- a/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc +++ b/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc @@ -29,10 +29,6 @@ #include "writer/tsfile_table_writer.h" #include "writer/tsfile_tree_writer.h" -namespace storage { -class QDSWithoutTimeGenerator; -} - using namespace storage; using namespace common; @@ -126,7 +122,7 @@ class TsFileTreeReaderTest : public ::testing::Test { protected: void SetUp() override { libtsfile_init(); - file_name_ = std::string("tsfile_writer_tree_test_") + + file_name_ = std::string("tsfile_writer_tree_reader_test_") + generate_random_string(10) + std::string(".tsfile"); remove(file_name_.c_str()); int flags = O_WRONLY | O_CREAT | O_TRUNC; @@ -137,7 +133,7 @@ class TsFileTreeReaderTest : public ::testing::Test { write_file_.create(file_name_, flags, mode); } - void TearDown() override {} + void TearDown() override { libtsfile_destroy(); } std::string file_name_; WriteFile write_file_; diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index eed05d8b..6d4edd5a 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -53,6 +53,7 @@ class TsFileReaderTest : public ::testing::Test { void TearDown() override { delete tsfile_writer_; remove(file_name_.c_str()); + libtsfile_destroy(); } std::string file_name_; diff --git a/cpp/test/writer/table_view/tsfile_writer_table_test.cc b/cpp/test/writer/table_view/tsfile_writer_table_test.cc index 2fb19d7d..d5861ea1 100644 --- a/cpp/test/writer/table_view/tsfile_writer_table_test.cc +++ b/cpp/test/writer/table_view/tsfile_writer_table_test.cc @@ -45,7 +45,10 @@ class TsFileWriterTableTest : public ::testing::Test { mode_t mode = 0666; write_file_.create(file_name_, flags, mode); } - void TearDown() override { remove(file_name_.c_str()); } + void TearDown() override { + remove(file_name_.c_str()); + libtsfile_destroy(); + } std::string file_name_; WriteFile write_file_; @@ -1013,7 +1016,7 @@ TEST_F(TsFileWriterTableTest, EncodingConfigIntegration) { // Create measurement schemas with configured encodings and compression for (size_t i = 0; i < measurement_names.size(); i++) { measurement_schemas.emplace_back(new MeasurementSchema( - measurement_names[i], data_types[i], encodings[i], SNAPPY)); + measurement_names[i], data_types[i], encodings[i], UNCOMPRESSED)); column_categories.emplace_back(ColumnCategory::FIELD); } diff --git a/cpp/test/writer/tsfile_writer_test.cc b/cpp/test/writer/tsfile_writer_test.cc index c1b3435d..791cf8e7 100644 --- a/cpp/test/writer/tsfile_writer_test.cc +++ b/cpp/test/writer/tsfile_writer_test.cc @@ -53,6 +53,7 @@ class TsFileWriterTest : public ::testing::Test { delete tsfile_writer_; int ret = remove(file_name_.c_str()); ASSERT_EQ(0, ret); + libtsfile_destroy(); } std::string file_name_; @@ -286,6 +287,8 @@ TEST_F(TsFileWriterTest, WriteMultipleRecords) { ASSERT_EQ(tsfile_writer_->close(), E_OK); } +#if defined(ENABLE_ZLIB) && defined(ENABLE_SNAPPY) && defined(ENABLE_LZ4) && \ + defined(ENABLE_LZOKAY) TEST_F(TsFileWriterTest, WriteDiffrentTypeCombination) { std::string device_path = "device1"; std::string measurement_name = "temperature"; @@ -337,6 +340,7 @@ TEST_F(TsFileWriterTest, WriteDiffrentTypeCombination) { ASSERT_EQ(tsfile_writer_->close(), E_OK); delete[] literal; } +#endif TEST_F(TsFileWriterTest, WriteMultipleTabletsMultiFlush) { common::config_set_max_degree_of_index_node(3); diff --git a/cpp/third_party/CMakeLists.txt b/cpp/third_party/CMakeLists.txt index e5d9384a..054591b0 100755 --- a/cpp/third_party/CMakeLists.txt +++ b/cpp/third_party/CMakeLists.txt @@ -16,9 +16,25 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ]] -add_subdirectory(antlr4-cpp-runtime-4) -add_subdirectory(google_snappy) -add_subdirectory(lz4) -add_subdirectory(lzokay) -add_subdirectory(zlib-1.3.1) +if (ENABLE_ANTLR4) + add_subdirectory(antlr4-cpp-runtime-4) + message("ANTLR4 runtime is enabled") +else() + message("ANTLR4 runtime is disabled") +endif() +if (ENABLE_SNAPPY) + add_subdirectory(google_snappy) +endif() + +if (ENABLE_LZ4) + add_subdirectory(lz4) +endif() + +if (ENABLE_LZOKAY) + add_subdirectory(lzokay) +endif() + +if (ENABLE_ZLIB) + add_subdirectory(zlib-1.3.1) +endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) \ No newline at end of file
