This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a change to branch ycy/AINodeTraining
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from a66e25d4de8 fix import error
add 93f0795233d IoTConsensusV2: persistent isGeneratedByPipeConsensus
(#14938)
add 1669f503907 [Fix](query)(table model) GroupedAccumulator are not
discovered before because of stream optimize
add b8947ff54fd Delete Maintain Auth
add 809ecd4cad1 Fix (#15027)
add 23068b42bbe Add memory metrics and refact memory allocation (#14710)
add f58ba839f2e Fixed the memory leakage of TableDeviceSchemaCache & The
invalidation logic when filter by SK & The bug that last time may not be
invalidated when the measurement with largest last time is invalidated (#15018)
add fddf0a6eaa2 Fixed the metadata query & Fragment Instance & retry's
timeout
add 93abf64655f Fix overflow when using format in ns timestamp precision
add 25af23d1b19 Added comments to tables and columns (#15033)
add 8608710e89a Fix Memory NPE when use IDEA to debug #15034
add 727ad63f214 Load: Detect 2PC Second Phase and RM Running in Parallel
(#15020)
add c557e3e42fc perf: more adjustment for memtable/tvlist (#15035)
add ee8b308d922 Fix resetPeerListTest for unstable port binding #15038
add ce47273ba7f Subscription: let subscription module fully manage the
parsing process of the insert node event (#15012)
add ea4322722f3 Fix some issue in auth model.
add 5bb6803e999 Pipe: Implemented runtime permission check (#14885)
add bdabc3474fb Merge branch 'master' into ycy/AINodeTraining
No new revisions were added by this update.
Summary of changes:
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 17 +-
.../it/auth/IoTDBClusterAuthorityRelationalIT.java | 1 -
.../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 62 +-
.../pipe/it/dual/tablemodel/TableModelUtils.java | 6 +
.../manual/basic/IoTDBPipeDataSinkIT.java | 6 +
.../manual/basic/IoTDBPipeExtractorIT.java | 7 +
.../manual/basic/IoTDBPipeLifeCycleIT.java | 64 +-
.../manual/basic/IoTDBPipePermissionIT.java | 226 +++++++
.../manual/basic/IoTDBPipeProtocolIT.java | 4 +
.../manual/basic/IoTDBPipeSwitchStatusIT.java | 4 +
.../tablemodel/manual/basic/IoTDBPipeSyntaxIT.java | 1 +
.../manual/basic/IoTDBPipeWithLoadIT.java | 4 +
.../manual/basic/IoTDBTablePatternFormatIT.java | 7 +
.../manual/enhanced/IoTDBPipeAutoConflictIT.java | 4 +
.../manual/enhanced/IoTDBPipeAutoDropIT.java | 2 +
.../manual/enhanced/IoTDBPipeClusterIT.java | 13 +
.../enhanced/IoTDBPipeConnectorCompressionIT.java | 12 +-
.../enhanced/IoTDBPipeConnectorParallelIT.java | 2 +
.../manual/enhanced/IoTDBPipeIdempotentIT.java | 1 +
.../manual/enhanced/IoTDBPipeMetaIT.java | 4 +
.../manual/enhanced/IoTDBPipeNullValueIT.java | 1 +
.../pipe/it/single/IoTDBPipePermissionIT.java | 157 +++++
.../scalar/IoTDBFormatFunctionTableIT.java | 14 +-
.../it/query/recent/IoTDBMaintainAuthIT.java | 89 +--
.../it/query/recent/IoTDBTableAggregationIT.java | 4 +-
.../IoTDBTableAggregationNonStreamIT.java} | 27 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 6 +-
.../it/local/IoTDBSubscriptionBasicIT.java | 38 +-
.../IoTDBRelationalDatabaseMetadata.java | 25 +-
.../base/AbstractSubscriptionConsumer.java | 7 +-
.../confignode/manager/PermissionManager.java | 4 +
.../manager/load/balancer/PartitionBalancer.java | 8 +-
.../iotdb/confignode/manager/node/NodeManager.java | 45 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 14 +-
.../pipe/event/PipeConfigRegionWritePlanEvent.java | 26 +-
.../pipe/extractor/IoTDBConfigRegionExtractor.java | 108 ++++
...nfigPhysicalPlanTablePrivilegeParseVisitor.java | 364 +++++++++++
.../iotdb/confignode/persistence/AuthorInfo.java | 4 +
.../impl/pipe/task/AlterPipeProcedureV2.java | 66 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 97 +++
.../iotdb/consensus/config/IoTConsensusConfig.java | 30 +-
.../apache/iotdb/consensus/iot/IoTConsensus.java | 4 +-
.../logdispatcher/IoTConsensusMemoryManager.java | 50 +-
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 80 +--
.../consensus/pipe/PipeConsensusServerImpl.java | 4 +-
.../pipe/consensuspipe/ConsensusPipeManager.java | 8 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 6 +-
.../apache/iotdb/consensus/iot/StabilityTest.java | 49 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 2 +
.../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 684 +++++++++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 345 +----------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 307 +--------
.../db/consensus/DataRegionConsensusImpl.java | 11 +-
.../agent/task/connection/PipeEventCollector.java | 24 +-
.../agent/task/stage/PipeTaskProcessorStage.java | 4 +-
.../subtask/processor/PipeProcessorSubtask.java | 3 +
.../protocol/writeback/WriteBackConnector.java | 83 ++-
.../db/pipe/event/UserDefinedEnrichedEvent.java | 14 +-
.../db/pipe/event/common/PipeInsertionEvent.java | 30 +-
.../common/deletion/PipeDeleteDataNodeEvent.java | 31 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 15 +-
.../schema/PipeSchemaRegionSnapshotEvent.java | 16 +-
.../schema/PipeSchemaRegionWritePlanEvent.java | 18 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 60 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 18 +-
.../event/common/terminate/PipeTerminateEvent.java | 13 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 97 ++-
.../parser/TsFileInsertionEventParserProvider.java | 5 +-
.../table/TsFileInsertionEventTableParser.java | 22 +-
.../db/pipe/event/realtime/PipeRealtimeEvent.java | 28 +-
.../event/realtime/PipeRealtimeEventFactory.java | 27 +-
.../dataregion/IoTDBDataRegionExtractor.java | 2 +-
...oricalDataRegionTsFileAndDeletionExtractor.java | 25 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 20 +
.../realtime/assigner/PipeDataRegionAssigner.java | 8 +
.../realtime/epoch/TsFileEpochManager.java | 8 +-
.../listener/PipeInsertionDataNodeListener.java | 14 +-
.../matcher/CachedSchemaPatternMatcher.java | 40 +-
.../realtime/matcher/PipeDataRegionMatcher.java | 3 +
.../schemaregion/IoTDBSchemaRegionExtractor.java | 78 +++
...ava => PipePlanTablePrivilegeParseVisitor.java} | 46 +-
.../schemaregion/PipeStatementToPlanVisitor.java | 48 ++
.../pipeconsensus/PipeConsensusReceiver.java | 2 +-
...leStatementDataTypeConvertExecutionVisitor.java | 1 +
.../db/pipe/resource/memory/PipeMemoryBlock.java | 2 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 100 +--
.../impl/DataNodeInternalRPCServiceImpl.java | 91 ++-
.../execution/exchange/SharedTsBlockQueue.java | 2 +-
.../execution/exchange/sink/ShuffleSinkHandle.java | 2 +-
.../execution/exchange/sink/SinkChannel.java | 2 +-
.../execution/exchange/source/SourceHandle.java | 2 +-
.../execution/memory/LocalMemoryManager.java | 5 +-
.../queryengine/execution/memory/MemoryPool.java | 38 +-
.../aggregation/grouped/GroupedAggregator.java | 4 +-
.../grouped/GroupedFirstAccumulator.java | 11 +-
.../grouped/GroupedFirstByAccumulator.java | 2 +-
.../grouped/GroupedLastAccumulator.java | 1 +
.../grouped/GroupedLastByAccumulator.java | 2 +-
.../grouped/GroupedVarianceAccumulator.java | 2 +-
.../execution/schedule/DriverScheduler.java | 6 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 4 +-
.../analyze/cache/partition/PartitionCache.java | 16 +-
.../cache/schema/DataNodeDevicePathCache.java | 23 +-
.../schema/dualkeycache/impl/DualKeyCacheImpl.java | 9 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 4 +
.../analyze/schema/AutoCreateSchemaExecutor.java | 6 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 7 +-
.../queryengine/plan/execution/QueryExecution.java | 4 -
.../execution/config/TableConfigTaskVisitor.java | 146 +++--
.../plan/planner/LocalExecutionPlanContext.java | 3 +-
.../plan/planner/LocalExecutionPlanner.java | 124 ++--
.../SimpleFragmentParallelPlanner.java | 2 +-
.../distribution/WriteFragmentParallelPlanner.java | 3 +-
.../analyzer/StatementAnalyzerFactory.java | 4 +
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 6 +-
.../fetcher/TableDeviceSchemaValidator.java | 6 +-
.../fetcher/cache/TableAttributeSchema.java | 6 +-
.../fetcher/cache/TableDeviceLastCache.java | 5 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 19 +-
.../distribute/TableModelQueryFragmentPlanner.java | 2 +-
.../plan/relational/security/AccessControl.java | 18 +-
.../relational/security/AccessControlImpl.java | 29 +-
.../relational/security/AllowAllAccessControl.java | 11 +-
.../relational/security/ITableAuthChecker.java | 3 +
.../relational/security/ITableAuthCheckerImpl.java | 18 +
.../relational/security/TableModelPrivilege.java | 5 -
.../plan/relational/sql/ast/AstVisitor.java | 4 +
...onfiguration.java => ShowQueriesStatement.java} | 18 +-
.../plan/relational/sql/parser/AstBuilder.java | 49 +-
.../plan/relational/sql/rewrite/ShowRewrite.java | 26 +-
.../sql/rewrite/StatementRewriteFactory.java | 6 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 22 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 93 +--
.../unary/scalar/FormatColumnTransformer.java | 4 +-
.../metric/SchemaEngineCachedMetric.java | 5 +-
.../rescon/MemSchemaEngineStatistics.java | 41 +-
.../ReleaseFlushStrategySizeBasedImpl.java | 6 +-
.../db/service/metrics/CompactionMetrics.java | 2 +-
.../iotdb/db/service/metrics/WritingMetrics.java | 2 +-
.../metrics/memory/ConsensusMemoryMetrics.java | 50 +-
.../metrics/memory/GlobalMemoryMetrics.java | 82 +--
.../metrics/memory/OffHeapMemoryMetrics.java | 102 +++
.../metrics/memory/QueryEngineMemoryMetrics.java | 281 ++++++---
.../metrics/memory/SchemaEngineMemoryMetrics.java | 159 +++--
.../metrics/memory/StorageEngineMemoryMetrics.java | 361 +++++++----
.../metrics/memory/StreamEngineMemoryMetrics.java | 50 +-
.../db/storageengine/buffer/BloomFilterCache.java | 27 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 27 +-
.../buffer/TimeSeriesMetadataCache.java | 27 +-
.../storageengine/dataregion/DataRegionInfo.java | 6 +-
.../execute/task/CrossSpaceCompactionTask.java | 2 +-
.../execute/task/InnerSpaceCompactionTask.java | 2 +-
.../task/RepairUnsortedFileCompactionTask.java | 2 +-
.../compaction/execute/utils/CompactionUtils.java | 12 +-
.../dataregion/memtable/WritableMemChunk.java | 120 ++--
.../dataregion/tsfile/TsFileResource.java | 19 +
.../wal/utils/MemoryControlledWALEntryQueue.java | 9 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 5 +-
...leStatementDataTypeConvertExecutionVisitor.java | 1 +
.../memory/LoadTsFileAnalyzeSchemaMemoryBlock.java | 2 +-
.../memory/LoadTsFileDataCacheMemoryBlock.java | 4 +-
.../load/memory/LoadTsFileMemoryManager.java | 2 +-
.../rescon/memory/PrimitiveArrayManager.java | 16 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 187 ++----
.../rescon/memory/TimePartitionManager.java | 31 +-
.../rescon/memory/TsFileResourceManager.java | 40 +-
.../db/tools/schema/SRStatementGenerator.java | 23 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 2 +-
.../db/utils/datastructure/BooleanTVList.java | 2 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 2 +-
.../datastructure/FixedPriorityBlockingQueue.java | 2 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 2 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 2 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 2 +-
.../MergeSortAlignedTVListIterator.java | 33 +-
.../datastructure/MergeSortTVListIterator.java | 31 +-
.../db/pipe/consensus/DeletionResourceTest.java | 7 +-
.../pipe/event/TsFileInsertionEventParserTest.java | 5 +-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 5 +
.../pattern/CachedSchemaPatternMatcherTest.java | 53 +-
.../execution/exchange/LocalSinkChannelTest.java | 7 +-
.../exchange/MPPDataExchangeManagerTest.java | 7 +-
.../execution/exchange/SharedTsBlockQueueTest.java | 4 +-
.../execution/exchange/ShuffleSinkHandleTest.java | 4 +-
.../execution/exchange/SinkChannelTest.java | 7 +-
.../execution/exchange/SourceHandleTest.java | 15 +-
.../execution/memory/MemoryPoolTest.java | 6 +-
.../plan/relational/analyzer/AnalyzerTest.java | 4 +-
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 21 +-
.../plan/relational/planner/PlanTester.java | 9 +-
.../relational/sql/parser/AuthorStatementTest.java | 5 -
.../compaction/CompactionWorkerTest.java | 16 +-
.../FastCrossCompactionPerformerTest.java | 5 +-
.../cross/CrossSpaceCompactionSelectorTest.java | 5 +-
.../compaction/utils/CompactionClearUtils.java | 2 +-
.../compaction/utils/CompactionTaskQueueTest.java | 13 +-
.../rescon/memory/ResourceManagerTest.java | 6 +-
.../rescon/memory/TimePartitionManagerTest.java | 8 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +-
.../datastructure/PrimitiveArrayManagerTest.java | 8 +-
.../commons/auth/authorizer/BasicAuthorizer.java | 35 +-
.../iotdb/commons/auth/authorizer/IAuthorizer.java | 10 +-
.../iotdb/commons/auth/entity/PrivilegeType.java | 2 +-
.../org/apache/iotdb/commons/auth/entity/Role.java | 4 +
.../iotdb/commons/auth/user/BasicUserManager.java | 12 +-
.../iotdb/commons/concurrent/ThreadName.java | 1 +
.../iotdb/commons/conf/CommonDescriptor.java | 4 +
.../commons/memory/AtomicLongMemoryBlock.java | 152 +++++
.../apache/iotdb/commons/memory/IMemoryBlock.java | 138 +++++
.../MemoryBlockType.java} | 11 +-
.../apache/iotdb/commons/memory/MemoryConfig.java} | 26 +-
.../iotdb/commons/memory/MemoryException.java} | 52 +-
.../apache/iotdb/commons/memory/MemoryManager.java | 523 ++++++++++++++++
.../config/constant/PipeConnectorConstant.java | 4 +
.../config/constant/PipeExtractorConstant.java | 12 +
.../queue/ConcurrentIterableLinkedQueue.java | 30 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 28 +
.../commons/pipe/event/PipeSnapshotEvent.java | 4 +
.../commons/pipe/event/PipeWritePlanEvent.java | 4 +
.../commons/pipe/event/ProgressReportEvent.java | 25 +-
.../commons/pipe/extractor/IoTDBExtractor.java | 48 +-
.../extractor/IoTDBNonDataRegionExtractor.java | 74 ++-
.../apache/iotdb/commons/service/ServiceType.java | 1 +
.../commons/subscription/meta/topic/TopicMeta.java | 3 +
.../iotdb/commons/memory/MemoryBlockTest.java | 67 ++
.../iotdb/commons/memory/MemoryManagerTest.java | 171 ++++++
.../ConcurrentIterableLinkedQueueTest.java | 1 +
.../db/relational/grammar/sql/RelationalSql.g4 | 6 +-
.../src/main/thrift/datanode.thrift | 1 +
229 files changed, 6084 insertions(+), 2286 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java
copy
integration-test/src/test/java/org/apache/iotdb/relational/it/query/{old/aligned/IoTDBAlignedSeriesQueryTable5IT.java
=> recent/IoTDBTableAggregationNonStreamIT.java} (61%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePrivilegeParseVisitor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/{PipePlanTablePatternParseVisitor.java
=> PipePlanTablePrivilegeParseVisitor.java} (59%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/PipeStatementToPlanVisitor.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{LoadConfiguration.java
=> ShowQueriesStatement.java} (69%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/OffHeapMemoryMetrics.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/AtomicLongMemoryBlock.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IMemoryBlock.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/{pipe/datastructure/visibility/Visibility.java
=> memory/MemoryBlockType.java} (86%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/UnSetTTLTask.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryConfig.java}
(58%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveConfigNode.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryException.java}
(50%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/memory/MemoryBlockTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/memory/MemoryManagerTest.java
