This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch ssl_between_nodes
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 5b1a27a16c9 recover commom
add 08b4f8f5c04 Pipe: Fixed the ping-pong of certain realtime tsFiles
add 772bab4abe0 Fix display of Blob type in last query
add 393aba20740 [AINode] Support concurrent inference for Timer-Sundial
(#15897)
add 736cdf7a56a Pipe: Add a tool for validating and repairing
isGeneratedByPipe mark in tsfile resources (#15934) (#15945)
add 061d2de0285 Pipe:Fix DataNodeShutdownHook waiting report logic and add
capture history file log (#15952)
add 905eb93633c Pipe: Added progressIndex reset function to the resource
resetting script (#15955)
add d44c81cdbe0 Bump io.projectreactor.netty:reactor-netty-http from
1.1.20 to 1.2.8 (#15958)
add b1b0c0192ab Root FragmentInstance placement error may occur when
querying table model aggregation with null data partition
add 2f07b0599f0 Add beforeDestroy in TableFunctionLeafProcessor and
TableFunctionDataProcessor
add 800f95ddc3c [AINode] Modify heartbeat field (#15953)
add b5a7f078c45 [AINode] AINode Dockerfile (#15946)
add 0f91e57a697 Pipe: Report progress index for filter-outed tsfile events
in PipeHistoricalDataRegionTsFileAndDeletionExtractor (#15965)
add 589e07bd080 Fix the handling of empty value chunk when load splitting
time partitions (#15966)
add 33eec34e837 Pipe: Fixed multiple bugs (#15964)
add 942e9118eb7 Fix some Accumulator when process TIMESTAMP Type
add 672e72dece7 Fix the issue that querying exact device with some fuzzy
filter may failed
new 5bd569ab2c9 Merge branch 'master' of github.com:apache/iotdb into
ssl_between_nodes
new e9a0d58348a add more client supports
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docker/ReadMe.md | 28 +-
docker/src/main/DockerCompose/do-docker-build.sh | 12 +-
.../main/DockerCompose/docker-compose-ainode.yml | 14 +-
...alone.yml => docker-compose-cluster-1c1d1a.yml} | 30 +-
docker/src/main/DockerCompose/entrypoint.sh | 41 +-
.../main/DockerCompose/replace-conf-from-env.sh | 19 +-
docker/src/main/Dockerfile-1.0.0-ainode | 69 +++
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 3 +
.../db/it/last/IoTDBLastQueryLastCacheIT.java | 2 +-
.../manual/AbstractPipeTableModelDualManualIT.java | 6 +-
.../treemodel/auto/basic/IoTDBPipeExtractorIT.java | 6 +-
.../treemodel/auto/basic/IoTDBPipeProcessorIT.java | 6 +-
.../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 6 +-
.../auto/enhanced/IoTDBPipeAutoConflictIT.java | 6 +-
.../auto/enhanced/IoTDBPipeClusterIT.java | 6 +-
.../enhanced/IoTDBPipeConnectorCompressionIT.java | 7 +-
.../auto/enhanced/IoTDBPipeIdempotentIT.java | 6 +-
.../auto/enhanced/IoTDBPipeWithLoadIT.java | 6 +-
.../manual/AbstractPipeDualTreeModelManualIT.java | 6 +-
.../manual/IoTDBPipeTypeConversionISessionIT.java | 12 +-
.../manual/IoTDBPipeTypeConversionIT.java | 2 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 1 +
.../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 20 +-
...alueIT.java => IoTDBQueryAttributeTableIT.java} | 36 +-
...ableAggregationQueryWithNetworkPartitionIT.java | 186 +++++++
.../it/cluster/IoTDBSubscriptionRestartIT.java | 3 +-
.../it/dual/AbstractSubscriptionDualIT.java | 3 +
.../IoTDBSubscriptionConsumerGroupIT.java | 3 +
.../IoTDBSubscriptionTimePrecisionIT.java | 2 +
.../dual/treemodel/IoTDBSubscriptionTopicIT.java | 3 +
.../it/local/AbstractSubscriptionLocalIT.java | 7 +-
.../it/triple/AbstractSubscriptionTripleIT.java | 5 +-
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 6 +-
.../event/dml/insertion/TsFileInsertionEvent.java | 10 +
.../processor/TableFunctionDataProcessor.java | 5 +
.../processor/TableFunctionLeafProcessor.java | 5 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 +
iotdb-core/ainode/ainode/core/config.py | 17 +
iotdb-core/ainode/ainode/core/constant.py | 7 +-
.../ainode/ainode/core/inference}/__init__.py | 0
.../ainode/core/inference/inference_request.py | 121 +++++
.../core/inference/inference_request_pool.py | 140 +++++
.../ainode/core/inference/strategy}/__init__.py | 0
.../strategy/abstract_inference_pipeline.py | 60 +++
.../strategy/timer_sundial_inference_pipeline.py | 51 ++
iotdb-core/ainode/ainode/core/inference/utils.py | 80 +++
.../ainode/core/manager/inference_manager.py | 132 ++++-
.../ainode/core/model/built_in_model_factory.py | 4 +-
.../core/model/sundial/configuration_sundial.py | 2 -
.../ainode/core/model/timerxl/modeling_timer.py | 3 -
iotdb-core/ainode/ainode/core/rpc/client.py | 30 +-
iotdb-core/ainode/poetry.lock | 279 +++++++---
iotdb-core/ainode/pyproject.toml | 4 +-
.../ainode/resources/conf/iotdb-ainode.properties | 2 +-
.../manager/load/service/HeartbeatService.java | 4 +-
.../agent/task/PipeConfigNodeSubtaskExecutor.java | 4 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 8 +
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../connector/PipeConfigNodeThriftRequestTest.java | 4 -
.../iot/client/SyncIoTConsensusServiceClient.java | 20 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 20 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 -
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 547 +++++++++----------
.../task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../execution/PipeConnectorSubtaskExecutor.java | 8 +-
.../execution/PipeProcessorSubtaskExecutor.java | 2 +-
.../task/execution/PipeSubtaskExecutorManager.java | 18 +-
.../agent/task/stage/PipeTaskConnectorStage.java | 6 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 5 +
.../connector/PipeConnectorSubtaskManager.java | 19 +-
.../PipeRealtimePriorityBlockingQueue.java | 151 +++++-
.../client/IoTDBDataNodeAsyncClientManager.java | 36 ++
.../evolvable/batch/PipeTabletEventBatch.java | 39 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 10 +-
.../batch/PipeTabletEventTsFileBatch.java | 7 +
.../evolvable/request/PipeTransferPlanNodeReq.java | 1 -
.../request/PipeTransferTabletBatchReq.java | 1 -
.../request/PipeTransferTabletBinaryReq.java | 1 -
.../request/PipeTransferTabletInsertNodeReq.java | 1 -
.../request/PipeTransferTabletRawReq.java | 1 -
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +-
.../PipeConsensusTransferBatchReqBuilder.java | 3 +-
.../async/IoTDBDataRegionAsyncConnector.java | 91 ++--
.../async/handler/PipeTransferTsFileHandler.java | 45 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 3 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../consensus/PipeConsensusSubtaskExecutor.java | 6 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 101 ++--
.../event/common/terminate/PipeTerminateEvent.java | 44 --
.../tsfile/PipeCompactedTsFileInsertionEvent.java | 264 ++++++++++
.../common/tsfile/PipeTsFileInsertionEvent.java | 98 ++--
.../parser/TsFileInsertionEventParserProvider.java | 4 +-
.../event/realtime/PipeRealtimeEventFactory.java | 12 +-
.../dataregion/DataRegionWatermarkInjector.java | 4 -
.../dataregion/IoTDBDataRegionExtractor.java | 107 ++--
...oricalDataRegionTsFileAndDeletionExtractor.java | 123 +++--
.../realtime/PipeRealtimeDataRegionExtractor.java | 35 ++
.../PipeRealtimeDataRegionHybridExtractor.java | 180 ++-----
.../PipeRealtimeDataRegionLogExtractor.java | 5 +-
.../PipeRealtimeDataRegionTsFileExtractor.java | 7 +
.../realtime/assigner/PipeDataRegionAssigner.java | 50 +-
.../PipeTsFileEpochProgressIndexKeeper.java | 30 +-
.../dataregion/realtime/epoch/TsFileEpoch.java | 17 +-
.../realtime/epoch/TsFileEpochManager.java | 4 +-
.../listener/PipeInsertionDataNodeListener.java | 8 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 3 -
.../PipeDataNodeRemainingEventAndTimeOperator.java | 22 +-
.../overview/PipeDataNodeSinglePipeMetrics.java | 84 ++-
.../pipe/metric/overview/PipeResourceMetrics.java | 38 +-
.../overview/PipeWALInsertNodeCacheMetrics.java | 86 ---
.../sink/PipeDataRegionConnectorMetrics.java | 34 ++
.../twostage/exchange/payload/CombineRequest.java | 1 -
.../payload/FetchCombineResultRequest.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 16 +
.../pipe/resource/PipeDataNodeResourceManager.java | 25 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 33 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 8 +-
.../resource/tsfile/PipeCompactionManager.java | 68 +++
...Resource.java => PipeTsFilePublicResource.java} | 158 +-----
.../pipe/resource/tsfile/PipeTsFileResource.java | 253 +--------
.../resource/tsfile/PipeTsFileResourceManager.java | 329 ++++++------
.../db/pipe/resource/wal/PipeWALResource.java | 169 ------
.../pipe/resource/wal/PipeWALResourceManager.java | 142 -----
.../wal/hardlink/PipeWALHardlinkResource.java | 47 --
.../hardlink/PipeWALHardlinkResourceManager.java | 152 ------
.../wal/selfhost/PipeWALSelfHostResource.java | 41 --
.../selfhost/PipeWALSelfHostResourceManager.java | 38 --
.../impl/DataNodeInternalRPCServiceImpl.java | 93 +++-
.../fragment/FragmentInstanceManager.java | 10 +
.../function/TableFunctionLeafOperator.java | 18 +-
.../process/function/TableFunctionOperator.java | 4 +
.../relational/aggregation/FirstAccumulator.java | 4 +-
.../relational/aggregation/FirstByAccumulator.java | 6 -
.../relational/aggregation/LastAccumulator.java | 4 +-
.../relational/aggregation/LastByAccumulator.java | 6 -
.../relational/aggregation/MaxAccumulator.java | 4 +-
.../relational/aggregation/MinAccumulator.java | 4 +-
.../iotdb/db/queryengine/plan/ClusterTopology.java | 16 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 4 +-
.../plan/planner/LocalExecutionPlanContext.java | 13 +-
.../plan/planner/LogicalPlanBuilder.java | 6 +-
.../plan/planner/OperatorTreeGenerator.java | 19 +-
.../plan/node/process/last/LastQueryNode.java | 12 +-
.../plan/node/source/LastQueryScanNode.java | 22 +-
.../ConvertSchemaPredicateToFilterVisitor.java | 8 +-
.../distribute/TableDistributedPlanGenerator.java | 4 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 1 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 2 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 7 -
.../db/storageengine/dataregion/DataRegion.java | 4 +-
.../task/InsertionCrossSpaceCompactionTask.java | 2 +-
.../compaction/execute/utils/CompactionUtils.java | 7 +-
.../dataregion/flush/NotifyFlushMemTable.java | 7 -
.../dataregion/memtable/AbstractMemTable.java | 13 -
.../dataregion/memtable/IMemTable.java | 4 -
.../dataregion/memtable/TsFileProcessor.java | 26 +-
.../dataregion/tsfile/TsFileManager.java | 13 +
.../dataregion/tsfile/TsFileResource.java | 31 +-
.../dataregion/wal/buffer/WALBuffer.java | 6 -
.../dataregion/wal/buffer/WALEntry.java | 4 +-
.../wal/checkpoint/CheckpointManager.java | 70 +--
.../dataregion/wal/checkpoint/MemTableInfo.java | 19 +-
.../dataregion/wal/io/WALByteBufReader.java | 13 -
.../dataregion/wal/node/WALFakeNode.java | 4 +-
.../storageengine/dataregion/wal/node/WALNode.java | 96 +---
.../dataregion/wal/utils/WALEntryHandler.java | 233 --------
.../dataregion/wal/utils/WALEntryPosition.java | 242 ---------
.../dataregion/wal/utils/WALInsertNodeCache.java | 338 ------------
.../wal/utils/listener/WALFlushListener.java | 12 +-
.../db/storageengine/load/LoadTsFileManager.java | 1 +
...leStatementDataTypeConvertExecutionVisitor.java | 90 ++--
...eeStatementDataTypeConvertExecutionVisitor.java | 172 ++----
.../converter/LoadTsFileDataTypeConverter.java | 49 +-
.../load/splitter/TsFileSplitter.java | 3 +
.../execution/SubscriptionSubtaskExecutor.java | 9 +-
.../task/stage/SubscriptionTaskConnectorStage.java | 4 +-
.../TsFileResourcePipeStatisticsSetTool.java | 302 +++++++++++
.../connector/PipeDataNodeThriftRequestTest.java | 12 -
.../pipe/event/TsFileInsertionEventParserTest.java | 1 +
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 101 ++--
.../resource/PipeTsFileResourceManagerTest.java | 96 ++--
.../PipeWALHardlinkResourceManagerTest.java | 114 ----
.../plan/planner/distribution/LastQueryTest.java | 1 +
.../logical/DataQueryLogicalPlannerTest.java | 5 +-
.../node/source/LastQueryScanNodeSerdeTest.java | 2 +
.../TsFileResourceProgressIndexTest.java | 23 +-
.../BatchedCompactionWithTsFileSplitterTest.java | 2 +-
.../wal/checkpoint/CheckpointManagerTest.java | 172 ------
.../dataregion/wal/node/WALEntryHandlerTest.java | 586 ---------------------
.../dataregion/wal/node/WALNodeTest.java | 17 -
.../wal/node/WalDeleteOutdatedNewTest.java | 65 ---
.../wal/utils/WALInsertNodeCacheTest.java | 190 -------
.../iotdb/commons/client/ainode/AINodeClient.java | 41 +-
.../client/ainode/AsyncAINodeServiceClient.java | 17 +-
.../async/AsyncPipeConsensusServiceClient.java | 16 +-
.../commons/concurrent/IoTDBThreadPoolFactory.java | 15 +
.../iotdb/commons/concurrent/ThreadName.java | 7 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 381 +++++---------
.../commons/consensus/index/ProgressIndex.java | 2 +-
.../consensus/index/impl/IoTProgressIndex.java | 27 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 58 +-
.../agent/task/execution/PipeSubtaskExecutor.java | 45 +-
.../task/progress/PipeEventCommitManager.java | 25 +-
.../agent/task/progress/PipeEventCommitter.java | 12 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 5 -
.../agent/task/subtask/PipeReportableSubtask.java | 21 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 137 ++---
.../iotdb/commons/pipe/config/PipeDescriptor.java | 124 ++---
.../config/constant/PipeConnectorConstant.java | 2 +-
.../thrift/request/PipeTransferFilePieceReq.java | 1 -
.../thrift/request/PipeTransferFileSealReqV1.java | 1 -
.../thrift/request/PipeTransferFileSealReqV2.java | 1 -
.../thrift/request/PipeTransferHandshakeV1Req.java | 1 -
.../thrift/request/PipeTransferHandshakeV2Req.java | 1 -
.../thrift/request/PipeTransferSliceReq.java | 1 -
.../pipe/connector/protocol/IoTDBConnector.java | 12 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 13 +
.../commons/pipe/resource/log/PipeLogStatus.java | 2 +-
.../iotdb/commons/service/metric/enums/Metric.java | 9 +-
.../thrift-ainode/src/main/thrift/ainode.thrift | 10 +-
pom.xml | 2 +-
...tsfile.sh => reset-resource-pipe-statistics.sh} | 8 +-
.../reset-resource-pipe-statistics.bat} | 11 +-
232 files changed, 4343 insertions(+), 5613 deletions(-)
copy
example/mqtt-customize/src/main/resources/META-INF/services/org.apache.iotdb.db.protocol.mqtt.PayloadFormatter
=> docker/src/main/DockerCompose/docker-compose-ainode.yml (68%)
copy docker/src/main/DockerCompose/{docker-compose-standalone.yml =>
docker-compose-cluster-1c1d1a.yml} (69%)
create mode 100644 docker/src/main/Dockerfile-1.0.0-ainode
copy
integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/{IoTDBNullValueIT.java
=> IoTDBQueryAttributeTableIT.java} (67%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java
copy {iotdb-client/client-py/iotdb =>
iotdb-core/ainode/ainode/core/inference}/__init__.py (100%)
create mode 100644 iotdb-core/ainode/ainode/core/inference/inference_request.py
create mode 100644
iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
copy {iotdb-client/client-py/iotdb =>
iotdb-core/ainode/ainode/core/inference/strategy}/__init__.py (100%)
create mode 100644
iotdb-core/ainode/ainode/core/inference/strategy/abstract_inference_pipeline.py
create mode 100644
iotdb-core/ainode/ainode/core/inference/strategy/timer_sundial_inference_pipeline.py
create mode 100644 iotdb-core/ainode/ainode/core/inference/utils.py
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/{PipeTsFileResource.java
=> PipeTsFilePublicResource.java} (63%)
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourcePipeStatisticsSetTool.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeWALHardlinkResourceManagerTest.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManagerTest.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALEntryHandlerTest.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCacheTest.java
copy scripts/tools/tsfile/{validate-tsfile.sh =>
reset-resource-pipe-statistics.sh} (80%)
copy scripts/tools/windows/{schema/print-schema-log.bat =>
tsfile/reset-resource-pipe-statistics.bat} (85%)
