This is an automated email from the ASF dual-hosted git repository. colinlee pushed a commit to branch py_refine in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 0db9e5c108c0c56e4248ddd5aa072b91cc0336e1 Author: colin <[email protected]> AuthorDate: Thu Apr 17 20:38:54 2025 +0800 add memory threshold in py writer. --- cpp/src/cwrapper/tsfile_cwrapper.cc | 4 +++- cpp/src/cwrapper/tsfile_cwrapper.h | 5 +++-- python/tsfile/tsfile_cpp.pxd | 5 +++-- python/tsfile/tsfile_py_cpp.pxd | 2 +- python/tsfile/tsfile_py_cpp.pyx | 4 ++-- python/tsfile/tsfile_writer.pyx | 4 ++-- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index 371e8ced..9e5e31a8 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -561,7 +561,8 @@ void free_write_file(WriteFile *write_file) { } // For Python API -TsFileWriter _tsfile_writer_new(const char *pathname, ERRNO *err_code) { +TsFileWriter _tsfile_writer_new(const char *pathname, uint64_t memory_threshold, + ERRNO *err_code) { init_tsfile_config(); auto writer = new storage::TsFileWriter(); int flags = O_WRONLY | O_CREAT | O_TRUNC; @@ -569,6 +570,7 @@ TsFileWriter _tsfile_writer_new(const char *pathname, ERRNO *err_code) { flags |= O_BINARY; #endif int ret = writer->open(pathname, flags, 0644); + common::g_config_value_.chunk_group_size_threshold_ = memory_threshold; if (ret != common::E_OK) { delete writer; *err_code = ret; diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h b/cpp/src/cwrapper/tsfile_cwrapper.h index d43e5dce..cfbfe8ef 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.h +++ b/cpp/src/cwrapper/tsfile_cwrapper.h @@ -156,7 +156,7 @@ TsFileWriter tsfile_writer_new(WriteFile file, TableSchema* schema, * @param schema Table schema definition. * - Ownership: Should be free it by Caller. * @param memory_threshold When the size of written data exceeds - * this value, the data will be automatically flushed to the disk. + * this value, the data will be automatically flushed to the disk. * @param err_code [out] E_OK(0), or check error code in errno_define_c.h. * * @return TsFileWriter Valid handle on success, NULL on failure. @@ -514,7 +514,8 @@ void free_write_file(WriteFile* write_file); * Avoid use: No compatibility/existence guarantees. */ // Create a tsfile writer. -TsFileWriter _tsfile_writer_new(const char* pathname, ERRNO* err_code); +TsFileWriter _tsfile_writer_new(const char* pathname, uint64_t memory_threshold, + ERRNO* err_code); // Create a tablet with name, data_type and max_rows. Tablet _tablet_new_with_target_name(const char* device_id, diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd index b4f9ccf8..3749aa7a 100644 --- a/python/tsfile/tsfile_cpp.pxd +++ b/python/tsfile/tsfile_cpp.pxd @@ -17,7 +17,7 @@ # #cython: language_level=3 -from libc.stdint cimport uint32_t, int32_t, int64_t +from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t ctypedef int32_t ErrorCode @@ -109,7 +109,8 @@ cdef extern from "./tsfile_cwrapper.h": ErrorCode tsfile_reader_close(TsFileReader reader) # writer： new and close - TsFileWriter _tsfile_writer_new(const char * pathname, ErrorCode * err_code); + TsFileWriter _tsfile_writer_new(const char * pathname, uint64_t memory_threshold, + ErrorCode * err_code); ErrorCode _tsfile_writer_close(TsFileWriter writer); # writer : register table, device and timeseries diff --git a/python/tsfile/tsfile_py_cpp.pxd b/python/tsfile/tsfile_py_cpp.pxd index be1d5290..c7310d7c 100644 --- a/python/tsfile/tsfile_py_cpp.pxd +++ b/python/tsfile/tsfile_py_cpp.pxd @@ -40,7 +40,7 @@ cdef public api void free_c_timeseries_schema(TimeseriesSchema* c_schema) cdef public api void free_c_device_schema(DeviceSchema* c_schema) cdef public api void free_c_tablet(Tablet tablet) cdef public api void free_c_row_record(TsRecord record) -cdef public api TsFileWriter tsfile_writer_new_c(object pathname) except + +cdef public api TsFileWriter tsfile_writer_new_c(object pathname, uint64_t memory_threshold) except + cdef public api TsFileReader tsfile_reader_new_c(object pathname) except + cdef public api ErrorCode tsfile_writer_register_device_py_cpp(TsFileWriter writer, DeviceSchema *schema) cdef public api ErrorCode tsfile_writer_register_timeseries_py_cpp(TsFileWriter writer, object device_name, diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx index 649c0755..90b12b4a 100644 --- a/python/tsfile/tsfile_py_cpp.pyx +++ b/python/tsfile/tsfile_py_cpp.pyx @@ -329,12 +329,12 @@ cdef void free_c_row_record(TsRecord record): _free_tsfile_ts_record(&record) # Reader and writer new. -cdef TsFileWriter tsfile_writer_new_c(object pathname) except +: +cdef TsFileWriter tsfile_writer_new_c(object pathname, uint64_t memory_threshold) except +: cdef ErrorCode errno = 0 cdef TsFileWriter writer cdef bytes encoded_path = PyUnicode_AsUTF8String(pathname) cdef const char* c_path = encoded_path - writer = _tsfile_writer_new(c_path, &errno) + writer = _tsfile_writer_new(c_path, memory_threshold, &errno) check_error(errno) return writer diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index a670e5cb..4ceecd3c 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -29,8 +29,8 @@ from tsfile.tablet import Tablet as TabletPy cdef class TsFileWriterPy: cdef TsFileWriter writer - def __init__(self, pathname): - self.writer = tsfile_writer_new_c(pathname) + def __init__(self, pathname, memory_threshold = 128 * 1024 * 1024): + self.writer = tsfile_writer_new_c(pathname, memory_threshold) def register_timeseries(self, device_name : str, timeseries_schema : TimeseriesSchemaPy): """
