This is an automated email from the ASF dual-hosted git repository.
colinlee pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 217cc3c1 Add flush method for tsfile python writer (#487)
217cc3c1 is described below
commit 217cc3c184d85b1adca2c3393d3664a2f692588e
Author: YangCaiyin <[email protected]>
AuthorDate: Fri May 9 16:26:54 2025 +0800
Add flush method for tsfile python writer (#487)
* add flush method for tsfile python writer
* fix
* revert zconf
* fix and add test
---
cpp/src/cwrapper/tsfile_cwrapper.cc | 6 ++++++
cpp/src/cwrapper/tsfile_cwrapper.h | 3 +++
python/tests/test_write.py | 27 +++++++++++++++++++++++++++
python/tsfile/tsfile_cpp.pxd | 3 +++
python/tsfile/tsfile_table_writer.py | 7 +++++++
python/tsfile/tsfile_writer.pyx | 11 +++++++++++
6 files changed, 57 insertions(+)
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc
b/cpp/src/cwrapper/tsfile_cwrapper.cc
index 9e5e31a8..0997dc42 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.cc
+++ b/cpp/src/cwrapper/tsfile_cwrapper.cc
@@ -683,6 +683,12 @@ ERRNO _tsfile_writer_close(TsFileWriter writer) {
return ret;
}
+ERRNO _tsfile_writer_flush(TsFileWriter writer) {
+ auto *w = static_cast<storage::TsFileWriter *>(writer);
+ return w->flush();
+}
+
+
ResultSet _tsfile_reader_query_device(TsFileReader reader,
const char *device_name,
char **sensor_name, uint32_t sensor_num,
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h
b/cpp/src/cwrapper/tsfile_cwrapper.h
index cfbfe8ef..30727539 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.h
+++ b/cpp/src/cwrapper/tsfile_cwrapper.h
@@ -562,6 +562,9 @@ ERRNO _tsfile_writer_write_ts_record(TsFileWriter writer,
TsRecord record);
// Close a TsFile writer, automatically flush data.
ERRNO _tsfile_writer_close(TsFileWriter writer);
+// Flush Chunk into tsfile from current tsFileWriter
+ERRNO _tsfile_writer_flush(TsFileWriter writer);
+
// Queries time-series data for a specific device within a given time range.
ResultSet _tsfile_reader_query_device(TsFileReader reader,
const char* device_name,
diff --git a/python/tests/test_write.py b/python/tests/test_write.py
index d86f5cee..cae992b2 100644
--- a/python/tests/test_write.py
+++ b/python/tests/test_write.py
@@ -106,6 +106,33 @@ def test_table_write():
if os.path.exists("table_write.tsfile"):
os.remove("table_write.tsfile")
+def test_flush():
+ file_name = "table_flush.tsfile"
+ try:
+ with TsFileWriter(file_name) as writer:
+ origin_size = os.path.getsize(file_name)
+ column1 = ColumnSchema("item_id", TSDataType.STRING,
ColumnCategory.TAG)
+ column2 = ColumnSchema("value", TSDataType.DOUBLE)
+ table = TableSchema("test_flush", [column1, column2])
+ writer.register_table(table)
+ row_num = 100
+
+ tablet = Tablet(["item_id", "value"],
+ [TSDataType.STRING, TSDataType.DOUBLE],
+ row_num)
+ tablet.set_table_name("test_flush")
+ for i in range(100):
+ tablet.add_timestamp(i, i)
+ tablet.add_value_by_name("item_id", i, str(i))
+ tablet.add_value_by_name("value", i, i * 10.1)
+ writer.write_table(tablet)
+ assert os.path.getsize(file_name) == origin_size
+ writer.flush()
+ assert os.path.getsize(file_name) > origin_size
+ finally:
+ if os.path.exists(file_name):
+ os.remove(file_name)
+
diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd
index 3749aa7a..e65cd214 100644
--- a/python/tsfile/tsfile_cpp.pxd
+++ b/python/tsfile/tsfile_cpp.pxd
@@ -113,6 +113,9 @@ cdef extern from "./tsfile_cwrapper.h":
ErrorCode * err_code);
ErrorCode _tsfile_writer_close(TsFileWriter writer);
+ # writer : flush
+ ErrorCode _tsfile_writer_flush(TsFileWriter writer);
+
# writer : register table, device and timeseries
ErrorCode _tsfile_writer_register_table(TsFileWriter writer, TableSchema *
schema);
ErrorCode _tsfile_writer_register_timeseries(TsFileWriter writer,
diff --git a/python/tsfile/tsfile_table_writer.py
b/python/tsfile/tsfile_table_writer.py
index c312a0ed..c7ab9492 100644
--- a/python/tsfile/tsfile_table_writer.py
+++ b/python/tsfile/tsfile_table_writer.py
@@ -60,6 +60,13 @@ class TsFileTableWriter:
"""
self.writer.close()
+ def flush(self):
+ """
+ Flush current data to tsfile.
+ :return: no return value.
+ """
+ self.writer.flush()
+
def __dealloc__(self):
self.close()
diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx
index 4ceecd3c..4d1f53ed 100644
--- a/python/tsfile/tsfile_writer.pyx
+++ b/python/tsfile/tsfile_writer.pyx
@@ -124,6 +124,17 @@ cdef class TsFileWriterPy:
check_error(errno)
self.writer = NULL
+ cpdef flush(self):
+ """
+ Flush data in memory to tsfile.
+ Writer will not be closed and can still be used
+ """
+ cdef ErrorCode errno
+ if self.writer == NULL:
+ return
+ errno = _tsfile_writer_flush(self.writer)
+ check_error(errno)
+
def __dealloc__(self):
self.close()
