This is an automated email from the ASF dual-hosted git repository. colinlee pushed a commit to branch colin_fix_config in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 6e3bd65e2cb8042ffd3941629377d006eeb79cd4 Author: colin <[email protected]> AuthorDate: Wed Apr 23 17:01:48 2025 +0800 add config in python. --- cpp/src/common/global.h | 17 ++ cpp/third_party/zlib-1.2.13/zconf.h | 547 ------------------------------------ python/setup.py | 40 ++- python/tests/test_write_and_read.py | 31 ++ python/tsfile/__init__.py | 1 + python/tsfile/tsfile_cpp.pxd | 27 +- python/tsfile/tsfile_py_cpp.pxd | 4 +- python/tsfile/tsfile_py_cpp.pyx | 74 +++++ 8 files changed, 181 insertions(+), 560 deletions(-) diff --git a/cpp/src/common/global.h b/cpp/src/common/global.h index 3a288cd6..9d79946b 100644 --- a/cpp/src/common/global.h +++ b/cpp/src/common/global.h @@ -28,6 +28,23 @@ namespace common { extern ConfigValue g_config_value_; extern ColumnSchema g_time_column_schema; + +FORCE_INLINE void set_global_time_data_type(uint8_t data_type) { + ASSERT(data_type >= BOOLEAN && data_type <= STRING); + g_config_value_.time_data_type_ = static_cast<TSDataType>(data_type); +} + +FORCE_INLINE void set_global_time_encoding(uint8_t encoding) { + ASSERT(encoding >= PLAIN && encoding <= FREQ); + g_config_value_.time_encoding_type_ = static_cast<TSEncoding>(encoding); +} + +FORCE_INLINE void set_global_time_compression(uint8_t compression) { + ASSERT(compression >= UNCOMPRESSED && compression <= LZ4); + g_config_value_.time_compress_type_ = + static_cast<CompressionType>(compression); +} + extern int init_common(); extern bool is_timestamp_column_name(const char *time_col_name); extern void cols_to_json(ByteStream *byte_stream, diff --git a/cpp/third_party/zlib-1.2.13/zconf.h b/cpp/third_party/zlib-1.2.13/zconf.h deleted file mode 100644 index bf977d3e..00000000 --- a/cpp/third_party/zlib-1.2.13/zconf.h +++ /dev/null @@ -1,547 +0,0 @@ -/* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* @(#) $Id$ */ - -#ifndef ZCONF_H -#define ZCONF_H - -/* - * If you *really* need a unique prefix for all types and library functions, - * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. - * Even better than compiling with -DZ_PREFIX would be to use configure to set - * this permanently in zconf.h using "./configure --zprefix". - */ -#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ -# define Z_PREFIX_SET - -/* all linked symbols and init macros */ -# define _dist_code z__dist_code -# define _length_code z__length_code -# define _tr_align z__tr_align -# define _tr_flush_bits z__tr_flush_bits -# define _tr_flush_block z__tr_flush_block -# define _tr_init z__tr_init -# define _tr_stored_block z__tr_stored_block -# define _tr_tally z__tr_tally -# define adler32 z_adler32 -# define adler32_combine z_adler32_combine -# define adler32_combine64 z_adler32_combine64 -# define adler32_z z_adler32_z -# ifndef Z_SOLO -# define compress z_compress -# define compress2 z_compress2 -# define compressBound z_compressBound -# endif -# define crc32 z_crc32 -# define crc32_combine z_crc32_combine -# define crc32_combine64 z_crc32_combine64 -# define crc32_combine_gen z_crc32_combine_gen -# define crc32_combine_gen64 z_crc32_combine_gen64 -# define crc32_combine_op z_crc32_combine_op -# define crc32_z z_crc32_z -# define deflate z_deflate -# define deflateBound z_deflateBound -# define deflateCopy z_deflateCopy -# define deflateEnd z_deflateEnd -# define deflateGetDictionary z_deflateGetDictionary -# define deflateInit z_deflateInit -# define deflateInit2 z_deflateInit2 -# define deflateInit2_ z_deflateInit2_ -# define deflateInit_ z_deflateInit_ -# define deflateParams z_deflateParams -# define deflatePending z_deflatePending -# define deflatePrime z_deflatePrime -# define deflateReset z_deflateReset -# define deflateResetKeep z_deflateResetKeep -# define deflateSetDictionary z_deflateSetDictionary -# define deflateSetHeader z_deflateSetHeader -# define deflateTune z_deflateTune -# define deflate_copyright z_deflate_copyright -# define get_crc_table z_get_crc_table -# ifndef Z_SOLO -# define gz_error z_gz_error -# define gz_intmax z_gz_intmax -# define gz_strwinerror z_gz_strwinerror -# define gzbuffer z_gzbuffer -# define gzclearerr z_gzclearerr -# define gzclose z_gzclose -# define gzclose_r z_gzclose_r -# define gzclose_w z_gzclose_w -# define gzdirect z_gzdirect -# define gzdopen z_gzdopen -# define gzeof z_gzeof -# define gzerror z_gzerror -# define gzflush z_gzflush -# define gzfread z_gzfread -# define gzfwrite z_gzfwrite -# define gzgetc z_gzgetc -# define gzgetc_ z_gzgetc_ -# define gzgets z_gzgets -# define gzoffset z_gzoffset -# define gzoffset64 z_gzoffset64 -# define gzopen z_gzopen -# define gzopen64 z_gzopen64 -# ifdef _WIN32 -# define gzopen_w z_gzopen_w -# endif -# define gzprintf z_gzprintf -# define gzputc z_gzputc -# define gzputs z_gzputs -# define gzread z_gzread -# define gzrewind z_gzrewind -# define gzseek z_gzseek -# define gzseek64 z_gzseek64 -# define gzsetparams z_gzsetparams -# define gztell z_gztell -# define gztell64 z_gztell64 -# define gzungetc z_gzungetc -# define gzvprintf z_gzvprintf -# define gzwrite z_gzwrite -# endif -# define inflate z_inflate -# define inflateBack z_inflateBack -# define inflateBackEnd z_inflateBackEnd -# define inflateBackInit z_inflateBackInit -# define inflateBackInit_ z_inflateBackInit_ -# define inflateCodesUsed z_inflateCodesUsed -# define inflateCopy z_inflateCopy -# define inflateEnd z_inflateEnd -# define inflateGetDictionary z_inflateGetDictionary -# define inflateGetHeader z_inflateGetHeader -# define inflateInit z_inflateInit -# define inflateInit2 z_inflateInit2 -# define inflateInit2_ z_inflateInit2_ -# define inflateInit_ z_inflateInit_ -# define inflateMark z_inflateMark -# define inflatePrime z_inflatePrime -# define inflateReset z_inflateReset -# define inflateReset2 z_inflateReset2 -# define inflateResetKeep z_inflateResetKeep -# define inflateSetDictionary z_inflateSetDictionary -# define inflateSync z_inflateSync -# define inflateSyncPoint z_inflateSyncPoint -# define inflateUndermine z_inflateUndermine -# define inflateValidate z_inflateValidate -# define inflate_copyright z_inflate_copyright -# define inflate_fast z_inflate_fast -# define inflate_table z_inflate_table -# ifndef Z_SOLO -# define uncompress z_uncompress -# define uncompress2 z_uncompress2 -# endif -# define zError z_zError -# ifndef Z_SOLO -# define zcalloc z_zcalloc -# define zcfree z_zcfree -# endif -# define zlibCompileFlags z_zlibCompileFlags -# define zlibVersion z_zlibVersion - -/* all zlib typedefs in zlib.h and zconf.h */ -# define Byte z_Byte -# define Bytef z_Bytef -# define alloc_func z_alloc_func -# define charf z_charf -# define free_func z_free_func -# ifndef Z_SOLO -# define gzFile z_gzFile -# endif -# define gz_header z_gz_header -# define gz_headerp z_gz_headerp -# define in_func z_in_func -# define intf z_intf -# define out_func z_out_func -# define uInt z_uInt -# define uIntf z_uIntf -# define uLong z_uLong -# define uLongf z_uLongf -# define voidp z_voidp -# define voidpc z_voidpc -# define voidpf z_voidpf - -/* all zlib structs in zlib.h and zconf.h */ -# define gz_header_s z_gz_header_s -# define internal_state z_internal_state - -#endif - -#if defined(__MSDOS__) && !defined(MSDOS) -# define MSDOS -#endif -#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) -# define OS2 -#endif -#if defined(_WINDOWS) && !defined(WINDOWS) -# define WINDOWS -#endif -#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__) -# ifndef WIN32 -# define WIN32 -# endif -#endif -#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) -# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) -# ifndef SYS16BIT -# define SYS16BIT -# endif -# endif -#endif - -/* - * Compile with -DMAXSEG_64K if the alloc function cannot allocate more - * than 64k bytes at a time (needed on systems with 16-bit int). - */ -#ifdef SYS16BIT -# define MAXSEG_64K -#endif -#ifdef MSDOS -# define UNALIGNED_OK -#endif - -#ifdef __STDC_VERSION__ -# ifndef STDC -# define STDC -# endif -# if __STDC_VERSION__ >= 199901L -# ifndef STDC99 -# define STDC99 -# endif -# endif -#endif -#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) -# define STDC -#endif -#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) -# define STDC -#endif -#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) -# define STDC -#endif -#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) -# define STDC -#endif - -#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ -# define STDC -#endif - -#ifndef STDC -# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ -# define const /* note: need a more gentle solution here */ -# endif -#endif - -#if defined(ZLIB_CONST) && !defined(z_const) -# define z_const const -#else -# define z_const -#endif - -#ifdef Z_SOLO - typedef unsigned long z_size_t; -#else -# define z_longlong long long -# if defined(NO_SIZE_T) - typedef unsigned NO_SIZE_T z_size_t; -# elif defined(STDC) -# include <stddef.h> - typedef size_t z_size_t; -# else - typedef unsigned long z_size_t; -# endif -# undef z_longlong -#endif - -/* Maximum value for memLevel in deflateInit2 */ -#ifndef MAX_MEM_LEVEL -# ifdef MAXSEG_64K -# define MAX_MEM_LEVEL 8 -# else -# define MAX_MEM_LEVEL 9 -# endif -#endif - -/* Maximum value for windowBits in deflateInit2 and inflateInit2. - * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files - * created by gzip. (Files created by minigzip can still be extracted by - * gzip.) - */ -#ifndef MAX_WBITS -# define MAX_WBITS 15 /* 32K LZ77 window */ -#endif - -/* The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) - plus a few kilobytes for small objects. For example, if you want to reduce - the default memory requirements from 256K to 128K, compile with - make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" - Of course this will generally degrade compression (there's no free lunch). - - The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus about 7 kilobytes - for small objects. -*/ - - /* Type declarations */ - -#ifndef OF /* function prototypes */ -# ifdef STDC -# define OF(args) args -# else -# define OF(args) () -# endif -#endif - -#ifndef Z_ARG /* function prototypes for stdarg */ -# if defined(STDC) || defined(Z_HAVE_STDARG_H) -# define Z_ARG(args) args -# else -# define Z_ARG(args) () -# endif -#endif - -/* The following definitions for FAR are needed only for MSDOS mixed - * model programming (small or medium model with some far allocations). - * This was tested only with MSC; for other MSDOS compilers you may have - * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, - * just define FAR to be empty. - */ -#ifdef SYS16BIT -# if defined(M_I86SM) || defined(M_I86MM) - /* MSC small or medium model */ -# define SMALL_MEDIUM -# ifdef _MSC_VER -# define FAR _far -# else -# define FAR far -# endif -# endif -# if (defined(__SMALL__) || defined(__MEDIUM__)) - /* Turbo C small or medium model */ -# define SMALL_MEDIUM -# ifdef __BORLANDC__ -# define FAR _far -# else -# define FAR far -# endif -# endif -#endif - -#if defined(WINDOWS) || defined(WIN32) - /* If building or using zlib as a DLL, define ZLIB_DLL. - * This is not mandatory, but it offers a little performance increase. - */ -# ifdef ZLIB_DLL -# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) -# ifdef ZLIB_INTERNAL -# define ZEXTERN extern __declspec(dllexport) -# else -# define ZEXTERN extern __declspec(dllimport) -# endif -# endif -# endif /* ZLIB_DLL */ - /* If building or using zlib with the WINAPI/WINAPIV calling convention, - * define ZLIB_WINAPI. - * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. - */ -# ifdef ZLIB_WINAPI -# ifdef FAR -# undef FAR -# endif -# ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -# endif -# include <windows.h> - /* No need for _export, use ZLIB.DEF instead. */ - /* For complete Windows compatibility, use WINAPI, not __stdcall. */ -# define ZEXPORT WINAPI -# ifdef WIN32 -# define ZEXPORTVA WINAPIV -# else -# define ZEXPORTVA FAR CDECL -# endif -# endif -#endif - -#if defined (__BEOS__) -# ifdef ZLIB_DLL -# ifdef ZLIB_INTERNAL -# define ZEXPORT __declspec(dllexport) -# define ZEXPORTVA __declspec(dllexport) -# else -# define ZEXPORT __declspec(dllimport) -# define ZEXPORTVA __declspec(dllimport) -# endif -# endif -#endif - -#ifndef ZEXTERN -# define ZEXTERN extern -#endif -#ifndef ZEXPORT -# define ZEXPORT -#endif -#ifndef ZEXPORTVA -# define ZEXPORTVA -#endif - -#ifndef FAR -# define FAR -#endif - -#if !defined(__MACTYPES__) -typedef unsigned char Byte; /* 8 bits */ -#endif -typedef unsigned int uInt; /* 16 bits or more */ -typedef unsigned long uLong; /* 32 bits or more */ - -#ifdef SMALL_MEDIUM - /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ -# define Bytef Byte FAR -#else - typedef Byte FAR Bytef; -#endif -typedef char FAR charf; -typedef int FAR intf; -typedef uInt FAR uIntf; -typedef uLong FAR uLongf; - -#ifdef STDC - typedef void const *voidpc; - typedef void FAR *voidpf; - typedef void *voidp; -#else - typedef Byte const *voidpc; - typedef Byte FAR *voidpf; - typedef Byte *voidp; -#endif - -#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) -# include <limits.h> -# if (UINT_MAX == 0xffffffffUL) -# define Z_U4 unsigned -# elif (ULONG_MAX == 0xffffffffUL) -# define Z_U4 unsigned long -# elif (USHRT_MAX == 0xffffffffUL) -# define Z_U4 unsigned short -# endif -#endif - -#ifdef Z_U4 - typedef Z_U4 z_crc_t; -#else - typedef unsigned long z_crc_t; -#endif - -#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ -# define Z_HAVE_UNISTD_H -#endif - -#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ -# define Z_HAVE_STDARG_H -#endif - -#ifdef STDC -# ifndef Z_SOLO -# include <sys/types.h> /* for off_t */ -# endif -#endif - -#if defined(STDC) || defined(Z_HAVE_STDARG_H) -# ifndef Z_SOLO -# include <stdarg.h> /* for va_list */ -# endif -#endif - -#ifdef _WIN32 -# ifndef Z_SOLO -# include <stddef.h> /* for wchar_t */ -# endif -#endif - -/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and - * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even - * though the former does not conform to the LFS document), but considering - * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as - * equivalently requesting no 64-bit operations - */ -#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 -# undef _LARGEFILE64_SOURCE -#endif - -#ifndef Z_HAVE_UNISTD_H -# ifdef __WATCOMC__ -# define Z_HAVE_UNISTD_H -# endif -#endif -#ifndef Z_HAVE_UNISTD_H -# if defined(_LARGEFILE64_SOURCE) && !defined(_WIN32) -# define Z_HAVE_UNISTD_H -# endif -#endif -#ifndef Z_SOLO -# if defined(Z_HAVE_UNISTD_H) -# include <unistd.h> /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ -# ifdef VMS -# include <unixio.h> /* for off_t */ -# endif -# ifndef z_off_t -# define z_off_t off_t -# endif -# endif -#endif - -#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 -# define Z_LFS64 -#endif - -#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) -# define Z_LARGE64 -#endif - -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) -# define Z_WANT64 -#endif - -#if !defined(SEEK_SET) && !defined(Z_SOLO) -# define SEEK_SET 0 /* Seek from beginning of file. */ -# define SEEK_CUR 1 /* Seek from current position. */ -# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ -#endif - -#ifndef z_off_t -# define z_off_t long -#endif - -#if !defined(_WIN32) && defined(Z_LARGE64) -# define z_off64_t off64_t -#else -# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) -# define z_off64_t __int64 -# else -# define z_off64_t z_off_t -# endif -#endif - -/* MVS linker does not support external names larger than 8 bytes */ -#if defined(__MVS__) - #pragma map(deflateInit_,"DEIN") - #pragma map(deflateInit2_,"DEIN2") - #pragma map(deflateEnd,"DEEND") - #pragma map(deflateBound,"DEBND") - #pragma map(inflateInit_,"ININ") - #pragma map(inflateInit2_,"ININ2") - #pragma map(inflateEnd,"INEND") - #pragma map(inflateSync,"INSY") - #pragma map(inflateSetDictionary,"INSEDI") - #pragma map(compressBound,"CMBND") - #pragma map(inflate_table,"INTABL") - #pragma map(inflate_fast,"INFA") - #pragma map(inflate_copyright,"INCOPY") -#endif - -#endif /* ZCONF_H */ diff --git a/python/setup.py b/python/setup.py index 6edeea0b..c25121f4 100644 --- a/python/setup.py +++ b/python/setup.py @@ -24,7 +24,7 @@ import platform import shutil import os -version = "2.1.0.dev0" +version = "2.1.0.dev" system = platform.system() def copy_tsfile_lib(source_dir, target_dir, suffix): @@ -33,6 +33,7 @@ def copy_tsfile_lib(source_dir, target_dir, suffix): target = os.path.join(target_dir, lib_file_name) if os.path.exists(source): + print("copy shared library from %s to %s" % (source, target)) shutil.copyfile(source, target) if system == "Linux": @@ -40,6 +41,8 @@ def copy_tsfile_lib(source_dir, target_dir, suffix): if os.path.exists(link_name): os.remove(link_name) os.symlink(lib_file_name, link_name) + print("create link %s" % link_name) + print("so exist:", os.path.exists(target)) elif system == "Darwin": link_name = os.path.join(target_dir, "libtsfile.dylib") if os.path.exists(link_name): @@ -47,19 +50,34 @@ def copy_tsfile_lib(source_dir, target_dir, suffix): os.symlink(lib_file_name, link_name) -def copy_tsfile_header(source, target): - if os.path.exists(source): - shutil.copyfile(source, target) - project_dir = os.path.dirname(os.path.abspath(__file__)) +tsfile_py_include = os.path.join(project_dir, "tsfile", "include") + +if os.path.exists(tsfile_py_include): + shutil.rmtree(tsfile_py_include) + +shutil.copytree( + os.path.join(project_dir, "..", "cpp", "target", "build", "include"), + os.path.join(tsfile_py_include, ""), +) + + + +def copy_tsfile_header(source): + for file in source: + if os.path.exists(file): + target = os.path.join(tsfile_py_include, os.path.basename(file)) + shutil.copyfile(file, target) + ## Copy C wrapper header. # tsfile/cpp/src/cwrapper/tsfile_cwrapper.h -tsfile_c_include_file = os.path.join( - project_dir, "..", "cpp", "src", "cwrapper", "tsfile_cwrapper.h" -) -tsfile_py_include_file = os.path.join(project_dir, "tsfile", "tsfile_cwrapper.h") -copy_tsfile_header(tsfile_c_include_file, tsfile_py_include_file) +source_headers = [ + os.path.join(project_dir, "..", "cpp", "src", "cwrapper", "tsfile_cwrapper.h"), +] + +copy_tsfile_header(source_headers) + ## Copy shared library tsfile_shared_source_dir = os.path.join(project_dir, "..", "cpp", "target", "build", "lib") @@ -72,7 +90,7 @@ elif system == "Linux": else: copy_tsfile_lib(tsfile_shared_source_dir, tsfile_shared_dir, "dll") -tsfile_include_dir=os.path.join(project_dir, "tsfile") +tsfile_include_dir=os.path.join(project_dir, "tsfile", "include") ext_modules_tsfile = [ diff --git a/python/tests/test_write_and_read.py b/python/tests/test_write_and_read.py index 6d0dbe7a..0b666d15 100644 --- a/python/tests/test_write_and_read.py +++ b/python/tests/test_write_and_read.py @@ -213,4 +213,35 @@ def test_lower_case_name(): assert data_frame.shape == (100, 3) assert data_frame["value"].sum() == 5445.0 +def test_tsfile_config(): + from tsfile import get_tsfile_config, set_tsfile_config + + config = get_tsfile_config() + + table = TableSchema("tEst_Table", + [ColumnSchema("Device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("vAlue", TSDataType.DOUBLE, ColumnCategory.FIELD)]) + + with TsFileTableWriter("test1.tsfile", table) as writer: + tablet = Tablet(["device", "VALUE"], [TSDataType.STRING, TSDataType.DOUBLE]) + for i in range(100): + tablet.add_timestamp(i, i) + tablet.add_value_by_name("device", i, "device" + str(i)) + tablet.add_value_by_name("valuE", i, i * 1.1) + + writer.write_table(tablet) + + config_normal = get_tsfile_config() + + with TsFileTableWriter("test2.tsfile", table, 100 * 100) as writer: + tablet = Tablet(["device", "VALUE"], [TSDataType.STRING, TSDataType.DOUBLE]) + for i in range(100): + tablet.add_timestamp(i, i) + tablet.add_value_by_name("device", i, "device" + str(i)) + tablet.add_value_by_name("valuE", i, i * 1.1) + + writer.write_table(tablet) + + + diff --git a/python/tsfile/__init__.py b/python/tsfile/__init__.py index df51bcfa..0c5081fa 100644 --- a/python/tsfile/__init__.py +++ b/python/tsfile/__init__.py @@ -32,4 +32,5 @@ from .date_utils import * from .exceptions import * from .tsfile_reader import TsFileReaderPy as TsFileReader, ResultSetPy as ResultSet from .tsfile_writer import TsFileWriterPy as TsFileWriter +from .tsfile_py_cpp import get_tsfile_config, set_tsfile_config from .tsfile_table_writer import TsFileTableWriter \ No newline at end of file diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd index e65cd214..e4782672 100644 --- a/python/tsfile/tsfile_cpp.pxd +++ b/python/tsfile/tsfile_cpp.pxd @@ -17,7 +17,7 @@ # #cython: language_level=3 -from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t +from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t, uint8_t ctypedef int32_t ErrorCode @@ -193,3 +193,28 @@ cdef extern from "./tsfile_cwrapper.h": ResultSetMetaData tsfile_result_set_get_metadata(ResultSet result_set); void free_result_set_meta_data(ResultSetMetaData result_set_meta_data); + + + +cdef extern from "./config.h" namespace "common": + cdef cppclass ConfigValue: + uint32_t tsblock_mem_inc_step_size_ + uint32_t tsblock_max_memory_ + uint32_t page_writer_max_point_num_ + uint32_t page_writer_max_memory_bytes_ + uint32_t max_degree_of_index_node_ + double tsfile_index_bloom_filter_error_percent_ + uint8_t time_encoding_type_ + uint8_t time_data_type_ + uint8_t time_compress_type_ + int32_t chunk_group_size_threshold_ + int32_t record_count_for_next_mem_check_ + bint encrypt_flag_ + +cdef extern from "./common/global.h" namespace "common": + ConfigValue g_config_value_ + +cdef extern from "./common/db_common.h" namespace "common": + void set_global_time_data_type(uint8_t data_type); + void set_global_time_encoding(uint8_t encoding); + void set_global_time_compression(uint8_t compression); \ No newline at end of file diff --git a/python/tsfile/tsfile_py_cpp.pxd b/python/tsfile/tsfile_py_cpp.pxd index c7310d7c..ce907a79 100644 --- a/python/tsfile/tsfile_py_cpp.pxd +++ b/python/tsfile/tsfile_py_cpp.pxd @@ -52,4 +52,6 @@ cdef public api ResultSet tsfile_reader_query_table_c(TsFileReader reader, objec cdef public api ResultSet tsfile_reader_query_paths_c(TsFileReader reader, object device_name, object sensor_list, int64_t start_time, int64_t end_time) cdef public api object get_table_schema(TsFileReader reader, object table_name) -cdef public api object get_all_table_schema(TsFileReader reader) \ No newline at end of file +cdef public api object get_all_table_schema(TsFileReader reader) +cpdef public api object get_tsfile_config() +cpdef public api void set_tsfile_config(dict new_config) \ No newline at end of file diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx index 90b12b4a..a554bab9 100644 --- a/python/tsfile/tsfile_py_cpp.pyx +++ b/python/tsfile/tsfile_py_cpp.pyx @@ -347,6 +347,80 @@ cdef TsFileReader tsfile_reader_new_c(object pathname) except +: check_error(errno) return reader +cpdef object get_tsfile_config(): + return { + "tsblock_mem_inc_step_size_": g_config_value_.tsblock_mem_inc_step_size_, + "tsblock_max_memory_": g_config_value_.tsblock_max_memory_, + "page_writer_max_point_num_": g_config_value_.page_writer_max_point_num_, + "page_writer_max_memory_bytes_": g_config_value_.page_writer_max_memory_bytes_, + "max_degree_of_index_node_": g_config_value_.max_degree_of_index_node_, + "tsfile_index_bloom_filter_error_percent_": g_config_value_.tsfile_index_bloom_filter_error_percent_, + "time_encoding_type_":TSEncodingPy(int(g_config_value_.time_encoding_type_)), + "time_data_type_": TSDataTypePy(int(g_config_value_.time_data_type_)), + "time_compress_type_": CompressorPy(int(g_config_value_.time_compress_type_)), + "chunk_group_size_threshold_": g_config_value_.chunk_group_size_threshold_, + "record_count_for_next_mem_check_":g_config_value_.record_count_for_next_mem_check_, + "encrypt_flag_":g_config_value_.encrypt_flag_ + } + + +cpdef void set_tsfile_config(dict new_config): + if "tsblock_mem_inc_step_size_" in new_config: + _check_uint32(new_config["tsblock_mem_inc_step_size_"]) + g_config_value_.tsblock_max_memory_ = new_config["tsblock_mem_inc_step_size_"] + if "tsblock_max_memory_" in new_config: + _check_uint32(new_config["tsblock_max_memory_"]) + g_config_value_.tsblock_max_memory_ = new_config["tsblock_max_memory_"] + if "page_writer_max_point_num_" in new_config: + _check_uint32(new_config["page_writer_max_point_num_"]) + g_config_value_.page_writer_max_point_num_ = new_config["page_writer_max_point_num_"] + if "page_writer_max_memory_bytes_" in new_config: + _check_uint32(new_config["page_writer_max_memory_bytes_"]) + g_config_value_.page_writer_max_memory_bytes_ = new_config["page_writer_max_memory_bytes_"] + if "max_degree_of_index_node_" in new_config: + _check_uint32(new_config["max_degree_of_index_node_"]) + g_config_value_.max_degree_of_index_node_ = new_config["max_degree_of_index_node_"] + if "tsfile_index_bloom_filter_error_percent_" in new_config: + _check_double(new_config["tsfile_index_bloom_filter_error_percent_"]) + g_config_value_.tsfile_index_bloom_filter_error_percent_ = new_config["tsfile_index_bloom_filter_error_percent_"] + if "time_encoding_type_" in new_config: + if new_config["time_encoding_type_"] not in TSEncodingPy: + raise ValueError(f"Unsupported TSEncoding: {new_config['time_encoding_type_']}") + set_global_time_encoding(<uint8_t>(new_config["time_encoding_type_"].value)) + if "time_data_type_" in new_config: + if new_config["time_data_type_"] not in TSDataTypePy: + raise ValueError(f"Unsupported TSDataType: {new_config['time_data_type_']}") + set_global_time_data_type(<uint8_t>(new_config["time_data_type_"].value)) + if "time_compress_type_" in new_config: + if new_config["time_compress_type_"] not in CompressorPy: + raise ValueError(f"Unsupported Compressor: {new_config['time_compress_type_']}") + set_global_time_compression(<uint8_t>(new_config["time_compress_type_"].value)) + if "chunk_group_size_threshold_" in new_config: + _check_int32(new_config["chunk_group_size_threshold_"]) + g_config_value_.chunk_group_size_threshold_ = new_config["chunk_group_size_threshold_"] + if "record_count_for_next_mem_check_" in new_config: + _check_int32(new_config["record_count_for_next_mem_check_"]) + g_config_value_.record_count_for_next_mem_check_ = new_config["record_count_for_next_mem_check_"] + if "encrypt_flag_" in new_config: + _check_bool(new_config["encrypt_flag_"]) + g_config_value_.encrypt_flag_ = <bint>new_config["encrypt_flag_"] + +cdef _check_uint32(value): + if not isinstance(value, int) or value < 0 or value > 0xFFFFFFFF: + raise TypeError(f"Expected uint32, got {type(value)}") + +cdef _check_int32(value): + if not isinstance(value, int) or value < -0x80000000 or value > 0x7FFFFFFF: + raise TypeError(f"Expected int32, got {type(value)}") + +cdef _check_double(value): + if not isinstance(value, (int, float)): + raise TypeError(f"Expected float, got {type(value)}") + +cdef _check_bool(value): + if not isinstance(value, bool): + raise TypeError(f"Expected bool, got {type(value)}") + # Register table and device cdef ErrorCode tsfile_writer_register_device_py_cpp(TsFileWriter writer, DeviceSchema *schema): cdef ErrorCode errno
