This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch optimize_records
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from cbaa7d2e325 remove old code
add 1b07cc58e9c [IOTDB-6280] Parallel RegionGroup activation (#11789)
add 92e297a9112 Pipe IT: enable pipe-it-2cluster.yml IT for branch
pipe-meta-sync
add 3f36705994c avoid lz4 factory performance problem (#11709)
add 7b06f1182b0 Add more write metrics (#11734)
add 7cf82e32b33 Add missing synchronized on abort of SourceHandle (#11793)
add ffdcbf3be79 [IOTDB-6278] Optimize Greedy Copyset algorithm (#11784)
add 3dd989a90f5 Pipe IT: enable Github Actions for branch pipe-meta-sync
(#11799)
add 8a0bc3d2e88 [IOTDB-6275] Disable limit/offset push down in ORDER BY
(#11772)
add 539ec0f83f9 [IOTDB-6282] Remove DataNode RPC address conflict check
(#11803)
add cea15bbe37d [IOTDB-6283] Adjust the leader election waiting time when
create RegionGroup (#11804)
add 3d81881fd8b [IOTDB-6261] Pipe: support leader cache in
iotdb-thrift-async-connector (#11807)
add d67f0556d0a Optimize log according to real scenarios (#11808)
add afde755d4f7 [IOTDB-6273] Load: Memory Management Framework (#11756)
add 48f2215747f PBTree: Fix unlock logic (#11809)
add 9d0322cfd5b PBTree: Fix deadlock of PBTree release and flush (#11810)
add a7d5b511346 Support_NewTemplateFilteringSyntax_IsNull_IsNotNull
add a42ec0e7c88 Implement PBTree Metrics
add 96a6df6e33a Optimize the situation when using align by device with
template and value filter (#11783)
add dd7bada581d Pipe: conceal SSL trust store password & improve exception
messages for sink connection establishment (#11806)
add 0c7fef076d8 [IOTDB-6285] Alter Procedure timeout (#11817)
add 177cc9a0e5b Add log for NotEnoughDataNodeException (#11815)
add 5a19759e11d Pipe: accurate recording of heartbeat event duration to
improve observability (#11820)
add e5f05105ada Clear status in InsertNodes to fix consensus endless retry
(#11819)
add 88feb511cb5 Move path related antlr defined files to tsfile module
(#11813)
add 0654b726313 try fix SessionPoolTest UT
add 209bb6999a1 Bump org.apache.maven.plugins:maven-clean-plugin from
3.3.1 to 3.3.2 (#11812)
add 5e26face893 Add the iotdb-cluster.properties configuration file and
start stop cleanup script (#11788)
add 3866f7dad47 Use systemctl to control confignode and datanode (#11629)
add 9fe128c7c21 Pipe: add memory control for leader cache (#11818)
add b60005e470e Pipe: improve exception messages for sink connection
establishment and node urls validation (#11826)
add 914ed16e14a Fix timeout doesn't take effect in schema fetch bug
(#11823)
add f920cdc8e86 Print Exception only when encountering unexpected TSStatus
(#11827)
add 9aedab88e12 PBTree: Fix deadlock when segment deleted (#11828)
add 9dece729fa9 Imporve some log printing of Storage engine (#11829)
add 34000269fa1 Pipe: reduce log when reporting pipe meta via heartbeat
(#11830)
add c29c46a4c60 feat(logback): change config (#11825)
add 8e9ef0aa4a9 PBTree: Prevent Concurrent Read With Stale Page (#11831)
add bcbe84ec51e Pipe: fix PipeDataRegionPluginAgent initialization order &
avoid instantiating meta source and sink (#11821)
add f42debd7e92 [IOTDB-6285] Alter Procedure timeout (#11836)
add 121c60b5936 Optimize log according to real scenarios after checking
(#11835)
add 05c7ed990c1 Add disk space log in cluster heartbeat (#11833)
add d0eed0ce738 Add error logs for NotEnoughDataNodeException and
DatabaseNotExistsException (#11832)
add 491246db1fc fix insert&delete cases dead data
add 7595a1a52bd Pipe: append logs generated by org.apache.iotdb.pipe.api.*
and org.apache.iotdb.commons.pipe.* into PIPE appender (#11845)
add 9fdeb95b6f6 Optimized wal file deletion algorithm (#11682)
add 42ea2cd3733 Fix aligned timeseries query error after delete some data
in memtable
add 712f4c4671a Return when leader election waiting is interrupted (#11842)
add 2f19b1a75e0 Reduce stack message of known compaction exceptions in log
(#11716)
add f077a18567f [IOTDB-6281] Enhance Procedure metrics (#11798)
add 4953d6fd3a0 fix the error of writing metrics (#11846)
add ca446d93748 Fix conflict
No new revisions were added by this update.
Summary of changes:
.github/workflows/cluster-it-1c1d.yml | 2 +
.github/workflows/cluster-it-1c3d.yml | 2 +
.github/workflows/pipe-it-2cluster.yml | 2 +
.github/workflows/unit-test.yml | 2 +
.gitignore | 2 +
.../it/env/cluster/config/MppDataNodeConfig.java | 9 +
.../it/env/remote/config/RemoteDataNodeConfig.java | 6 +
.../apache/iotdb/itbase/env/DataNodeConfig.java | 3 +
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 4 +-
.../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 4 +-
.../IoTDBAlignByDeviceWithTemplateIT.java | 308 ++++++++++-
.../IoTDBAlignedSeriesQueryWithDeletionIT.java | 28 +
.../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 54 +-
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 4 +-
.../api/customizer/parameter/PipeParameters.java | 33 +-
.../api/exception/PipeConnectionException.java | 3 +
.../java/org/apache/iotdb/session/Session.java | 2 +-
.../iotdb/session/SessionCacheLeaderTest.java | 1 +
.../iotdb/session/SessionConnectionTest.java | 26 +-
.../java/org/apache/iotdb/session/SessionTest.java | 9 +-
.../session/pool/SessionPoolExceptionTest.java | 9 +-
.../apache/iotdb/session/pool/SessionPoolTest.java | 13 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 3 +-
.../assembly/resources/conf/logback-confignode.xml | 52 +-
.../assembly/resources/sbin/register-confignode.sh | 80 +++
.../exception/NotEnoughDataNodeException.java | 13 +-
.../iotdb/confignode/manager/ConfigManager.java | 2 +
.../iotdb/confignode/manager/ProcedureManager.java | 26 +-
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../region/GreedyCopySetRegionGroupAllocator.java | 181 ++++---
.../confignode/manager/load/cache/LoadCache.java | 10 +-
.../manager/node/ClusterNodeStartUtils.java | 5 -
.../manager/partition/PartitionManager.java | 10 +-
.../manager/pipe/agent/PipeConfigNodeAgent.java | 2 +-
.../agent/plugin/PipePluginConfigNodeAgent.java | 14 +-
.../persistence/schema/ConfigMTreeStore.java | 5 +
.../iotdb/confignode/procedure/Procedure.java | 46 +-
.../confignode/procedure/ProcedureExecutor.java | 19 +-
.../confignode/procedure/ProcedureMetrics.java | 184 +++++++
.../statemachine/CreateRegionGroupsProcedure.java | 62 ++-
.../impl/statemachine/StateMachineProcedure.java | 2 +-
.../region/AllocatorScatterWidthManualTest.java | 136 +++++
.../GreedyCopySetRegionGroupAllocatorTest.java | 15 +-
.../procedure/TestProcedureExecutor.java | 2 +-
.../procedure/entity/StuckProcedure.java | 4 +-
.../src/test/resources/confignode1conf/logback.xml | 52 +-
.../src/test/resources/confignode2conf/logback.xml | 52 +-
.../src/test/resources/confignode3conf/logback.xml | 52 +-
.../consensus/iot/IoTConsensusServerImpl.java | 2 +-
.../assembly/resources/conf/logback-datanode.xml | 122 ++---
.../assembly/resources/sbin/register-datanode.sh | 80 +++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 57 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 11 +-
.../dataregion/DataExecutionVisitor.java | 3 +
...n.java => LoadRuntimeOutOfMemoryException.java} | 5 +-
.../pipe/agent/plugin/PipePluginDataNodeAgent.java | 8 +-
.../dataregion/PipeDataRegionPluginAgent.java | 23 +-
.../schemaregion/PipeSchemaRegionPluginAgent.java | 14 +-
.../db/pipe/agent/task/PipeTaskDataNodeAgent.java | 4 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 27 +-
...eManager.java => IoTDBThriftClientManager.java} | 19 +-
.../protocol/thrift/LeaderCacheManager.java | 80 ++-
.../async/IoTDBThriftAsyncClientManager.java | 200 +++++++
.../thrift/async/IoTDBThriftAsyncConnector.java | 220 ++------
.../PipeTransferTabletInsertNodeEventHandler.java | 12 +
.../PipeTransferTabletInsertionEventHandler.java | 17 +-
.../handler/PipeTransferTabletRawEventHandler.java | 7 +
.../thrift/sync/IoTDBThriftSyncClientManager.java | 68 +--
.../thrift/sync/IoTDBThriftSyncConnector.java | 2 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 44 +-
.../pipe/resource/wal/PipeWALResourceManager.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 10 +-
.../db/queryengine/common/MPPQueryContext.java | 10 +
.../iotdb/db/queryengine/common/NodeRef.java | 4 +
.../execution/exchange/source/SourceHandle.java | 2 +-
.../execution/executor/RegionWriteExecutor.java | 9 +-
.../db/queryengine/execution/load/ChunkData.java | 3 +
.../queryengine/execution/load/DeletionData.java | 17 +-
.../execution/load/LoadTsFileManager.java | 51 +-
.../db/queryengine/execution/load/TsFileData.java | 3 -
.../operator/schema/source/DeviceSchemaSource.java | 35 +-
.../load/LoadTsFileAbstractMemoryBlock.java | 66 +++
.../load/LoadTsFileAnalyzeSchemaMemoryBlock.java | 98 ++++
.../load/LoadTsFileDataCacheMemoryBlock.java | 147 ++++++
.../queryengine/load/LoadTsFileMemoryManager.java | 149 ++++++
.../queryengine/metric/LoadTsFileMemMetricSet.java | 101 ++++
.../iotdb/db/queryengine/plan/Coordinator.java | 8 +-
.../db/queryengine/plan/analyze/Analysis.java | 2 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 29 +-
.../plan/analyze/ExpressionTypeAnalyzer.java | 118 +++--
.../plan/analyze/LoadTsfileAnalyzer.java | 284 ++++++++--
.../queryengine/plan/analyze/TemplatedAnalyze.java | 61 +--
.../db/queryengine/plan/analyze/TemplatedInfo.java | 104 +++-
.../analyze/cache/schema/DataNodeSchemaCache.java | 22 +
.../plan/analyze/schema/ClusterSchemaFetcher.java | 9 +
.../visitor/ColumnTransformerVisitor.java | 14 +-
...dConcatRemoveUnExistentMeasurementVisitor.java} | 31 +-
...ncatDeviceAndBindSchemaForPredicateVisitor.java | 1 +
.../plan/optimization/LimitOffsetPushDown.java | 8 +
.../db/queryengine/plan/parser/ASTVisitor.java | 23 +-
.../plan/planner/LocalExecutionPlanner.java | 33 +-
.../plan/planner/OperatorTreeGenerator.java | 169 +++++-
.../plan/planner/SubPlanTypeExtractor.java | 13 +
.../plan/planner/TemplatedLogicalPlan.java | 181 ++++---
.../plan/planner/TemplatedLogicalPlanBuilder.java | 11 +-
.../plan/planner/plan/node/PlanNodeType.java | 2 +
.../plan/planner/plan/node/process/FilterNode.java | 30 ++
.../planner/plan/node/process/TransformNode.java | 2 +-
.../plan/node/write/InsertMultiTabletsNode.java | 4 +
.../planner/plan/node/write/InsertRowsNode.java | 4 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 4 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 34 +-
.../apache/iotdb/db/schemaengine/SchemaEngine.java | 10 +-
.../metric/SchemaEngineCachedMetric.java | 23 +-
.../schemaengine/metric/SchemaMetricManager.java | 13 +-
.../metric/SchemaRegionCachedMetric.java | 328 +++++++++---
.../schemaengine/metric/SchemaRegionMemMetric.java | 31 ++
.../rescon/CachedSchemaRegionStatistics.java | 9 +-
.../schemaengine/schemaregion/ISchemaRegion.java | 2 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 12 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 12 +-
.../schemaregion/mtree/IMTreeStore.java | 2 +
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 9 +-
.../schemaregion/mtree/impl/mem/MemMTreeStore.java | 24 +-
.../mtree/impl/pbtree/CachedMTreeStore.java | 42 +-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 14 +-
.../mtree/impl/pbtree/PBTreeFactory.java | 19 +-
.../pbtree/ReentrantReadOnlyCachedMTreeStore.java | 5 +
.../impl/pbtree/flush/PBTreeFlushExecutor.java | 19 +-
.../mtree/impl/pbtree/flush/Scheduler.java | 21 +-
.../mtree/impl/pbtree/lock/LockManager.java | 4 +
.../impl/pbtree/memcontrol/MemoryStatistics.java | 11 +-
.../mtree/impl/pbtree/memory/IMemoryManager.java | 3 +-
.../mtree/impl/pbtree/memory/MemoryManager.java | 11 +-
.../impl/pbtree/memory/ReleaseFlushMonitor.java | 5 +-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 36 +-
.../schemafile/pagemgr/BTreePageManager.java | 80 ++-
.../pbtree/schemafile/pagemgr/PageManager.java | 36 +-
.../schemaregion/mtree/traverser/Traverser.java | 6 +-
.../read/resp/info/impl/ShowDevicesResult.java | 13 +-
.../utils/filter/DeviceFilterVisitor.java | 29 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 4 +
.../iotdb/db/service/metrics/WritingMetrics.java | 169 ++++++
.../iotdb/db/storageengine/StorageEngine.java | 4 +
.../db/storageengine/dataregion/DataRegion.java | 48 +-
.../CompactionFileCountExceededException.java | 5 +
.../CompactionLastTimeCheckFailedException.java | 5 +
.../CompactionMemoryNotEnoughException.java | 5 +
.../CompactionValidationFailedException.java | 5 +
.../performer/impl/FastCompactionPerformer.java | 4 +
.../execute/task/AbstractCompactionTask.java | 11 +-
.../compaction/schedule/CompactionWorker.java | 2 +-
.../dataregion/flush/CompressionRatio.java | 3 -
.../dataregion/flush/MemTableFlushTask.java | 18 -
.../memtable/AlignedWritableMemChunk.java | 2 +-
.../dataregion/memtable/TsFileProcessor.java | 50 +-
.../dataregion/modification/Deletion.java | 4 +
.../dataregion/modification/ModificationFile.java | 19 +
.../io/LocalTextModificationAccessor.java | 9 +-
.../modification/io/ModificationWriter.java | 9 +
.../dataregion/wal/buffer/AbstractWALBuffer.java | 1 +
.../dataregion/wal/buffer/WALBuffer.java | 24 +
.../wal/checkpoint/CheckpointManager.java | 28 +-
.../storageengine/dataregion/wal/node/WALNode.java | 187 ++++---
.../db/storageengine/rescon/memory/SystemInfo.java | 43 +-
.../db/utils/datastructure/AlignedTVList.java | 6 +-
.../schema/SchemaQueryScanOperatorTest.java | 3 +-
.../dataregion/wal/node/WALEntryHandlerTest.java | 13 +-
.../wal/node/WalDeleteOutdatedNewTest.java | 585 +++++++++++++++++++++
.../src/test/resources/datanode1conf/logback.xml | 122 ++---
.../src/test/resources/datanode2conf/logback.xml | 122 ++---
.../src/test/resources/datanode3conf/logback.xml | 122 ++---
.../resources/conf/iotdb-cluster.properties | 32 +-
.../sbin/{stop-standalone.bat => clean-all.bat} | 27 +-
.../src/assembly/resources/sbin/clean-all.sh | 95 ++++
.../assembly/resources/sbin/clean-confignode.bat | 82 +++
.../assembly/resources/sbin/clean-confignode.sh | 55 ++
.../src/assembly/resources/sbin/clean-datanode.bat | 203 +++++++
.../src/assembly/resources/sbin/clean-datanode.sh | 70 +++
.../src/assembly/resources/sbin/start-all.sh | 81 +++
.../src/assembly/resources/sbin/stop-all.sh | 79 +++
.../async/AsyncPipeDataTransferServiceClient.java | 8 +
.../apache/iotdb/commons/conf/CommonConfig.java | 9 +
.../iotdb/commons/conf/CommonDescriptor.java | 5 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 -
.../commons/pipe/agent/plugin/PipePluginAgent.java | 18 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 7 +
.../builtin/connector/iotdb/IoTDBConnector.java | 81 ++-
.../connector/iotdb/thrift/IoTDBMetaConnector.java | 32 +-
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 29 +
.../commons/pipe/task/meta/PipeStaticMeta.java | 6 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 8 +
.../iotdb/commons/service/metric/enums/Metric.java | 18 +-
iotdb-core/tsfile/pom.xml | 56 +-
.../antlr4/org/apache/tsfile/parser/PathLexer.g4 | 212 ++++++++
.../antlr4/org/apache/tsfile/parser}/PathParser.g4 | 9 +-
.../tsfile/common/constant/TsFileConstant.java | 4 -
.../apache/iotdb/tsfile/compress/ICompressor.java | 14 +-
.../iotdb/tsfile/compress/IUnCompressor.java | 9 +-
...leSequenceReaderTimeseriesMetadataIterator.java | 42 +-
.../read/common/parser/PathNodesGenerator.java | 8 +-
.../tsfile/read/common/parser/PathVisitor.java | 6 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 -
...quenceReaderTimeseriesMetadataIteratorTest.java | 8 +-
pom.xml | 2 +-
205 files changed, 6471 insertions(+), 1801 deletions(-)
create mode 100644
iotdb-core/confignode/src/assembly/resources/sbin/register-confignode.sh
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/region/AllocatorScatterWidthManualTest.java
create mode 100644
iotdb-core/datanode/src/assembly/resources/sbin/register-datanode.sh
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{DirectoryNotLegalException.java
=> LoadRuntimeOutOfMemoryException.java} (86%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/{LeaderCacheManager.java
=> IoTDBThriftClientManager.java} (66%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBThriftAsyncClientManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileAbstractMemoryBlock.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileAnalyzeSchemaMemoryBlock.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileDataCacheMemoryBlock.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileMemoryManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/LoadTsFileMemMetricSet.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/{RemoveRootPrefixVisitor.java
=> TemplatedConcatRemoveUnExistentMeasurementVisitor.java} (58%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WalDeleteOutdatedNewTest.java
copy .asf.yaml =>
iotdb-core/node-commons/src/assembly/resources/conf/iotdb-cluster.properties
(66%)
copy iotdb-core/node-commons/src/assembly/resources/sbin/{stop-standalone.bat
=> clean-all.bat} (67%)
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/clean-all.sh
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/clean-confignode.bat
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/clean-confignode.sh
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/clean-datanode.bat
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/clean-datanode.sh
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/start-all.sh
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/stop-all.sh
create mode 100644
iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathLexer.g4
rename iotdb-core/{antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql =>
tsfile/src/main/antlr4/org/apache/tsfile/parser}/PathParser.g4 (92%)
