This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch strong_password
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 03d88332e92 fix writing redundant password history
add 0fce0ccfdb0 Update tsfile version & add table view query it
add 309f205859d Update Release info after 2.0.4 released (#15898)
add 1ad56c78138 Pipe: Enabled waiting for pipes to finish & progress index
persist to config node in shutdown hook (#15896)
add 14b4382ff44 [AINode]Fix the parameter "predict_length" (#15900)
add 4be53debee7 Fix potential memory leak of Memory Pool
add 9042a0d2c43 Pipe: Reduced the progress index report interval & Added
some logs (#15905)
add e6bc39f4da5 Add more checkpoints in series scan
add bf8329b1517 Pipe: Fix connection leak caused by clients not closed
after task dropped (2 situations) (#15910)
add f8e03f987f3 Fix the issue that the DeviceIteratorScanOperator.next()
method throws NoSuchElementException
add eb1e786fa15 Revert Pipe/IoTV2: Try to persist progressIndex in local
for resend event more efficently (#15924)
add ba6eed15d29 [AINode] Support model management for Table (#15923)
add 03385938f74 Bump org.apache.commons:commons-lang3 from 3.13.0 to
3.18.0 (#15922)
add ccfd563960b Correct Memory Allocation for SinkHandle in Table model
add ac44d8b8594 Make div and mod return long type if types of two inputs
are all int64 or int32
add 1f24444a868 Fix timestamp type data conversion issue in exported data
of table model (#15918)
add b2cdbe03f77 Fix display of Blob type in last query
add 59eb9bbb0a9 Fix explicit time query process in TreeModel
add d7844931a87 Fixed the issue that non aligned tree view query used too
much memory
add 77a91e0f60a Remove CI on windows-2022 (#15938)
add 8472c71bcdc Fixed the issue where devices in the cache may be
repeatedly fetched
add e139623037e [AINode] Remove AINode through SQL (#15937)
add f69347580e8 Avoid NPE while stop DN (#15939)
add 08b4f8f5c04 Pipe: Fixed the ping-pong of certain realtime tsFiles
add 772bab4abe0 Fix display of Blob type in last query
add 393aba20740 [AINode] Support concurrent inference for Timer-Sundial
(#15897)
add 736cdf7a56a Pipe: Add a tool for validating and repairing
isGeneratedByPipe mark in tsfile resources (#15934) (#15945)
add ace78115358 Merge branch 'master' into strong_password
No new revisions were added by this update.
Summary of changes:
.github/workflows/multi-language-client.yml | 10 +-
RELEASE_NOTES.md | 22 ++
.../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 268 -------------------
.../iotdb/ainode/it/AINodeClusterConfigIT.java | 113 ++++++++
.../iotdb/ainode/it/AINodeInferenceSQLIT.java | 292 +++++++++++++++++++++
.../iotdb/ainode/it/AINodeModelManageIT.java | 215 +++++++++++++++
.../apache/iotdb/ainode/utils/AINodeTestUtils.java | 126 +++++++++
.../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 4 +-
.../maxby/IoTDBMaxByAlignedSeriesIT.java | 20 ++
.../db/it/aggregation/maxby/IoTDBMaxByIT.java | 21 +-
.../db/it/aggregation/minby/IoTDBMinByIT.java | 5 +-
.../db/it/last/IoTDBLastQueryLastCacheIT.java | 15 +-
.../IoTDBArithmeticDivAndModWithoutDoubleIT.java} | 34 +--
.../withoutNull/IoTDBWithoutNullAllFilterIT.java | 48 ++--
.../withoutNull/IoTDBWithoutNullAnyFilterIT.java | 48 ++--
.../view/recent/IoTDBNullValueTableViewIT.java | 14 +-
.../IoTDBTableViewQueryWithCachedDeviceIT.java | 127 +++++++++
.../relational/it/schema/IoTDBDatabaseIT.java | 4 +-
.../src/test/resources/ainode-example/config.yaml | 8 +-
.../apache/iotdb/tool/data/ExportDataTable.java | 5 +-
iotdb-core/ainode/ainode/core/ainode.py | 167 ++++++++++++
iotdb-core/ainode/ainode/core/config.py | 17 ++
iotdb-core/ainode/ainode/core/constant.py | 5 +
.../ainode/ainode/core/inference}/__init__.py | 0
.../ainode/core/inference/inference_request.py | 121 +++++++++
.../core/inference/inference_request_pool.py | 140 ++++++++++
.../ainode/core/inference/strategy}/__init__.py | 0
.../strategy/abstract_inference_pipeline.py | 60 +++++
.../strategy/timer_sundial_inference_pipeline.py | 51 ++++
iotdb-core/ainode/ainode/core/inference/utils.py | 80 ++++++
.../ainode/core/manager/inference_manager.py | 138 ++++++++--
.../ainode/ainode/core/manager/model_manager.py | 2 +-
.../core/model/sundial/configuration_sundial.py | 2 -
.../core/model/sundial/ts_generation_mixin.py | 5 +
.../ainode/core/model/timerxl/modeling_timer.py | 3 -
.../ainode/ainode/core/rpc}/__init__.py | 0
iotdb-core/ainode/ainode/core/{ => rpc}/client.py | 2 +-
iotdb-core/ainode/ainode/core/{ => rpc}/handler.py | 10 +-
iotdb-core/ainode/ainode/core/rpc/service.py | 101 +++++++
.../ainode/ainode/core/{util => rpc}/status.py | 0
iotdb-core/ainode/ainode/core/script.py | 112 +-------
iotdb-core/ainode/ainode/core/service.py | 57 ----
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 9 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../rpc/DataNodeAsyncRequestRPCHandler.java | 2 +-
.../handlers/rpc/PipeHeartbeatRPCHandler.java | 2 +-
.../request/write/ainode/RemoveAINodePlan.java | 5 +
.../iotdb/confignode/manager/ConfigManager.java | 23 +-
.../apache/iotdb/confignode/manager/IManager.java | 7 +-
.../iotdb/confignode/manager/ModelManager.java | 2 +-
.../iotdb/confignode/manager/node/NodeManager.java | 20 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 22 +-
.../protocol/IoTDBConfigRegionConnector.java | 8 +-
.../runtime/heartbeat/PipeHeartbeatParser.java | 2 -
.../runtime/heartbeat/PipeHeartbeatScheduler.java | 2 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 7 +-
.../procedure/impl/node/RemoveAINodeProcedure.java | 24 ++
.../impl/pipe/runtime/PipeMetaSyncProcedure.java | 4 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 16 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 18 +-
.../procedure/state/RemoveAINodeState.java | 1 +
.../confignode/service/ConfigNodeShutdownHook.java | 3 -
.../thrift/ConfigNodeRPCServiceProcessor.java | 15 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +-
.../consensus/response/pipe/PipeTableRespTest.java | 6 +-
.../agent/PipeConfigNodeSubtaskExecutorTest.java | 3 +-
.../iotdb/confignode/persistence/PipeInfoTest.java | 4 +-
.../tools/tsfile/mark-is-generated-by-pipe.bat | 13 +-
.../tools/tsfile/mark-is-generated-by-pipe.sh | 10 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 90 +++----
.../subtask/connector/PipeConnectorSubtask.java | 5 +
.../subtask/processor/PipeProcessorSubtask.java | 4 +-
.../client/IoTDBDataNodeAsyncClientManager.java | 20 +-
.../async/IoTDBDataRegionAsyncConnector.java | 4 +-
.../handler/PipeTransferTrackableHandler.java | 2 +
.../async/handler/PipeTransferTsFileHandler.java | 22 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 6 +-
.../statement/PipeStatementInsertionEvent.java | 6 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 6 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 6 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 6 +-
.../dataregion/IoTDBDataRegionExtractor.java | 4 +-
...oricalDataRegionTsFileAndDeletionExtractor.java | 9 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 5 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 4 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 6 +-
.../PipeDataNodeRemainingEventAndTimeOperator.java | 18 +-
...ics.java => PipeDataNodeSinglePipeMetrics.java} | 15 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 11 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 8 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 2 +-
.../execution/exchange/SharedTsBlockQueue.java | 2 +-
.../queryengine/execution/exchange/sink/ISink.java | 4 +-
.../execution/exchange/sink/LocalSinkChannel.java | 14 +-
.../execution/exchange/sink/ShuffleSinkHandle.java | 40 ++-
.../execution/exchange/sink/SinkChannel.java | 10 +-
.../fragment/FragmentInstanceExecution.java | 42 ++-
.../operator/process/ProjectOperator.java | 3 +-
.../last/AlignedUpdateLastCacheOperator.java | 4 +-
.../AlignedUpdateViewPathLastCacheOperator.java | 4 +-
.../process/last/LastQueryTransformOperator.java | 4 +-
.../operator/process/last/LastQueryUtil.java | 34 +++
.../process/last/UpdateLastCacheOperator.java | 4 +-
.../last/UpdateViewPathLastCacheOperator.java | 4 +-
.../AbstractSeriesAggregationScanOperator.java | 70 +++--
.../source/AbstractSeriesScanOperator.java | 46 ++--
.../execution/operator/source/SeriesScanUtil.java | 73 ++++--
.../relational/AbstractAggTableScanOperator.java | 75 ++++--
.../AbstractDefaultAggTableScanOperator.java | 4 +-
.../relational/AbstractTableScanOperator.java | 16 +-
.../relational/DeviceIteratorScanOperator.java | 3 +-
.../InformationSchemaContentSupplierFactory.java | 4 +-
.../relational/LastQueryAggTableScanOperator.java | 4 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 6 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 8 +-
.../plan/analyze/ExpressionTypeAnalyzer.java | 29 ++
.../execution/config/TableConfigTaskVisitor.java | 36 +++
.../execution/config/TreeConfigTaskVisitor.java | 22 +-
.../config/executor/ClusterConfigTaskExecutor.java | 48 +++-
.../config/executor/IConfigTaskExecutor.java | 11 +-
...moveDataNodeTask.java => RemoveAINodeTask.java} | 12 +-
.../config/metadata/ai/CreateModelTask.java | 14 +-
.../execution/config/sys/pipe/ShowPipeTask.java | 4 +-
.../visitor/ColumnTransformerVisitor.java | 11 +
.../db/queryengine/plan/parser/ASTVisitor.java | 32 ++-
.../plan/planner/LocalExecutionPlanContext.java | 13 +-
.../plan/planner/LogicalPlanBuilder.java | 6 +-
.../plan/planner/OperatorTreeGenerator.java | 33 ++-
.../plan/planner/TableOperatorGenerator.java | 8 +-
.../plan/node/process/last/LastQueryNode.java | 12 +-
.../plan/node/source/LastQueryScanNode.java | 22 +-
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 35 ++-
.../plan/relational/sql/ast/AstVisitor.java | 12 +
.../sql/ast/{ShowModels.java => CreateModel.java} | 48 ++--
.../plan/relational/sql/ast/CreateTraining.java | 27 +-
.../sql/ast/{ShowModels.java => DropModel.java} | 44 ++--
.../ast/{ShowFunctions.java => RemoveAINode.java} | 33 ++-
.../plan/relational/sql/parser/AstBuilder.java | 72 +++--
.../scheduler/load/LoadTsFileDispatcherImpl.java | 1 +
.../plan/statement/StatementVisitor.java | 5 +
.../plan/statement/crud/QueryStatement.java | 10 +-
...deStatement.java => RemoveAINodeStatement.java} | 14 +-
.../metadata/model/CreateModelStatement.java | 10 +-
.../metadata/model/DropModelStatement.java | 10 +-
.../metadata/model/ShowModelsStatement.java | 12 +-
.../dag/input/QueryDataSetInputLayer.java | 3 +
.../apache/iotdb/db/schemaengine/SchemaEngine.java | 10 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 39 ++-
.../utils/reader/SeriesDataBlockReader.java | 20 +-
...GeneratedByPipeMarkValidationAndRepairTool.java | 263 +++++++++++++++++++
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 3 +-
.../queryengine/execution/exchange/StubSink.java | 6 +-
...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 69 +++--
.../db/queryengine/plan/analyze/AnalyzeTest.java | 4 +-
.../plan/planner/distribution/LastQueryTest.java | 1 +
.../logical/DataQueryLogicalPlannerTest.java | 5 +-
.../node/source/LastQueryScanNodeSerdeTest.java | 2 +
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 97 +++----
.../AlignedSeriesScanPredicatePushDownTest.java | 47 ++--
...gleColumnSeriesScanLimitOffsetPushDownTest.java | 48 ++--
.../read/reader/series/SeriesReaderTestUtil.java | 17 ++
.../series/SeriesScanLimitOffsetPushDownTest.java | 60 ++---
.../series/SeriesScanPredicatePushDownTest.java | 40 +--
.../apache/iotdb/commons/client/ClientManager.java | 22 +-
.../iotdb/commons/client/ainode/AINodeClient.java | 16 ++
.../async/AsyncPipeDataTransferServiceClient.java | 2 +-
.../iotdb/commons/concurrent/ThreadName.java | 5 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 79 ++----
.../agent/runtime/PipePeriodicalJobExecutor.java | 33 ---
.../commons/pipe/agent/task/PipeTaskAgent.java | 5 +-
.../commons/pipe/agent/task/meta/PipeMeta.java | 6 +-
.../pipe/agent/task/meta/PipeRuntimeMeta.java | 51 +---
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 142 +---------
.../iotdb/commons/pipe/config/PipeConfig.java | 28 +-
.../iotdb/commons/pipe/config/PipeDescriptor.java | 25 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 14 +-
.../commons/pipe/resource/log/PipeLogManager.java | 2 +-
.../commons/pipe/resource/log/PipeLogStatus.java | 4 +-
.../apache/iotdb/commons/utils/StatusUtils.java | 1 -
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 22 +-
.../db/relational/grammar/sql/RelationalSql.g4 | 14 +-
iotdb-doap.rdf | 8 +
.../thrift-ainode/src/main/thrift/ainode.thrift | 2 +
.../thrift-commons/src/main/thrift/common.thrift | 7 +
.../src/main/thrift/confignode.thrift | 5 +-
.../src/main/thrift/datanode.thrift | 9 +-
pom.xml | 4 +-
187 files changed, 3675 insertions(+), 1664 deletions(-)
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeClusterConfigIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeModelManageIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/utils/AINodeTestUtils.java
copy
integration-test/src/test/java/org/apache/iotdb/db/it/{groupby/IoTDBLeftORightCIT.java
=> query/IoTDBArithmeticDivAndModWithoutDoubleIT.java} (67%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java
create mode 100644 iotdb-core/ainode/ainode/core/ainode.py
copy {iotdb-client/client-py/iotdb =>
iotdb-core/ainode/ainode/core/inference}/__init__.py (100%)
create mode 100644 iotdb-core/ainode/ainode/core/inference/inference_request.py
create mode 100644
iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
copy {iotdb-client/client-py/iotdb =>
iotdb-core/ainode/ainode/core/inference/strategy}/__init__.py (100%)
create mode 100644
iotdb-core/ainode/ainode/core/inference/strategy/abstract_inference_pipeline.py
create mode 100644
iotdb-core/ainode/ainode/core/inference/strategy/timer_sundial_inference_pipeline.py
create mode 100644 iotdb-core/ainode/ainode/core/inference/utils.py
copy {iotdb-client/client-py/iotdb =>
iotdb-core/ainode/ainode/core/rpc}/__init__.py (100%)
rename iotdb-core/ainode/ainode/core/{ => rpc}/client.py (99%)
rename iotdb-core/ainode/ainode/core/{ => rpc}/handler.py (88%)
create mode 100644 iotdb-core/ainode/ainode/core/rpc/service.py
rename iotdb-core/ainode/ainode/core/{util => rpc}/status.py (100%)
delete mode 100644 iotdb-core/ainode/ainode/core/service.py
copy scripts/tools/windows/schema/print-schema-log.bat =>
iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.bat
(83%)
copy scripts/tools/tsfile/validate-tsfile.sh =>
iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.sh
(76%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/{PipeDataNodeRemainingEventAndTimeMetrics.java
=> PipeDataNodeSinglePipeMetrics.java} (96%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/{RemoveDataNodeTask.java
=> RemoveAINodeTask.java} (82%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{ShowModels.java
=> CreateModel.java} (66%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{ShowModels.java
=> DropModel.java} (65%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{ShowFunctions.java
=> RemoveAINode.java} (69%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/{RemoveConfigNodeStatement.java
=> RemoveAINodeStatement.java} (86%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool.java
