This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch optimize_infer_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cadafb6d4c41b3f8d4ff0d8e11646d6d7212b847 Merge: 757bc0a5cc4 8c044fcfb7d Author: HTHou <[email protected]> AuthorDate: Fri Mar 22 18:31:23 2024 +0800 Merge branch 'master' of github.com:apache/iotdb into optimize_infer_type .../apache/iotdb/tsfile/TsFileSequenceRead.java | 3 +- .../java/org/apache/iotdb/udf/UDAFExample.java | 16 +- .../apache/iotdb/db/query/udf/example/UDAFAvg.java | 16 +- .../iotdb/db/query/udf/example/UDAFCount.java | 2 +- .../apache/iotdb/db/query/udf/example/UDAFSum.java | 16 +- .../confignode/it/procedure/IoTDBProcedureIT.java | 4 +- .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 3 +- .../org/apache/iotdb/tsfile/utils/Accountable.java | 18 +- .../iotdb/tsfile/utils/RamUsageEstimator.java | 4 +- .../main/java/org/apache/iotdb/udf/api/UDAF.java | 2 +- .../client-py/iotdb/template/TemplateNode.py | 9 +- .../iotdb/tsfile/common/constant/TsFileConstant.py | 3 +- iotdb-client/client-py/requirements_dev.txt | 2 +- .../tsfile/qp/optimizer/PhysicalOptimizer.java | 14 +- .../apache/iotdb/spark/tsfile/DefaultSource.scala | 3 +- .../consensus/request/ConfigPhysicalPlanType.java | 3 + .../write/procedure/UpdateProcedurePlan.java | 9 +- .../statemachine/ConfigRegionStateMachine.java | 6 +- .../iotdb/confignode/manager/ConfigManager.java | 4 +- .../iotdb/confignode/manager/ProcedureManager.java | 36 +- .../manager/consensus/ConsensusManager.java | 6 + .../manager/load/balancer/RouteBalancer.java | 25 +- .../confignode/persistence/ProcedureInfo.java | 235 +++- .../persistence/executor/ConfigPlanExecutor.java | 3 + .../procedure/CompletedProcedureRecycler.java | 6 +- .../iotdb/confignode/procedure/Procedure.java | 14 +- .../confignode/procedure/ProcedureExecutor.java | 108 +- .../procedure/TimeoutExecutorThread.java | 2 +- .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 26 +- .../AddNeverFinishSubProcedureProcedure.java | 6 + .../impl/testonly/NeverFinishProcedure.java | 17 + .../procedure/store/ConfigProcedureStore.java | 44 +- .../procedure/store/IProcedureStore.java | 17 +- .../confignode/procedure/store/ProcedureType.java | 4 +- .../confignode/procedure/store/ProcedureWAL.java | 46 +- .../consensus/request/TestOnlyPlan.java} | 23 +- .../confignode/persistence/ProcedureInfoTest.java | 75 ++ .../confignode/procedure/NoopProcedureStore.java | 31 +- .../UpgradeFromWALToConsensusLayerTest.java | 137 ++ .../procedure/util/ProcedureTestUtil.java | 11 - .../iotdb/consensus/ratis/RatisConsensus.java | 3 +- .../src/assembly/resources/conf/datanode-env.bat | 37 +- .../src/assembly/resources/conf/datanode-env.sh | 39 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 37 +- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 17 +- .../tsfile/TsFileInsertionDataContainer.java | 40 +- .../realtime/epoch/TsFileEpochManager.java | 6 +- .../pipe/resource/memory/PipeMemoryWeighUtil.java | 17 +- .../pipe/resource/tsfile/PipeTsFileResource.java | 16 +- .../resource/tsfile/PipeTsFileResourceManager.java | 5 +- .../execution/aggregation/UDAFAccumulator.java | 8 + .../execution/load/LoadTsFileManager.java | 3 +- .../queryengine/execution/load/TsFileSplitter.java | 24 +- .../execution/operator/source/FileLoaderUtils.java | 6 +- .../execution/operator/source/SeriesScanUtil.java | 16 +- .../metric/QueryRelatedResourceMetricSet.java | 88 ++ .../iotdb/db/queryengine/plan/Coordinator.java | 3 +- .../db/queryengine/plan/analyze/Analysis.java | 8 + .../plan/analyze/LoadTsfileAnalyzer.java | 51 +- .../config/metadata/ShowFunctionsTask.java | 3 +- .../plan/optimization/ColumnInjectionPushDown.java | 2 +- .../plan/optimization/LimitOffsetPushDown.java | 2 +- .../OrderByExpressionWithLimitChangeToTopK.java | 2 +- .../plan/optimization/PredicatePushDown.java | 2 +- .../db/queryengine/plan/parser/ASTVisitor.java | 22 - .../queryengine/plan/planner/LogicalPlanner.java | 2 +- .../plan/planner/plan/node/PlanVisitor.java | 8 +- .../plan/node/load/LoadSingleTsFileNode.java | 5 +- .../plan/node/pipe/PipeEnrichedInsertNode.java | 2 +- .../plan/planner/plan/node/write/InsertNode.java | 38 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 23 +- .../statement/sys/ExplainAnalyzeStatement.java | 7 + .../apache/iotdb/db/schemaengine/SchemaEngine.java | 5 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 12 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 7 +- .../schemaregion/utils/ResourceByPathUtils.java | 8 +- .../java/org/apache/iotdb/db/service/DataNode.java | 5 +- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 9 +- .../iotdb/db/storageengine/StorageEngine.java | 8 +- .../buffer/TimeSeriesMetadataCache.java | 25 +- .../db/storageengine/dataregion/DataRegion.java | 55 +- .../dataregion/DeviceLastFlushTime.java | 8 +- .../dataregion/HashLastFlushTimeMap.java | 45 +- .../storageengine/dataregion/ILastFlushTime.java | 6 +- .../dataregion/ILastFlushTimeMap.java | 16 +- .../dataregion/PartitionLastFlushTime.java | 6 +- .../performer/impl/FastCompactionPerformer.java | 9 +- .../impl/ReadChunkCompactionPerformer.java | 9 +- .../impl/ReadPointCompactionPerformer.java | 20 +- .../subtask/FastCompactionPerformerSubTask.java | 7 +- .../task/subtask/ReadPointPerformerSubTask.java | 5 +- .../execute/utils/CompactionPathUtils.java | 12 +- .../compaction/execute/utils/CompactionUtils.java | 3 +- .../execute/utils/MultiTsFileDeviceIterator.java | 13 +- .../fast/AlignedSeriesCompactionExecutor.java | 3 +- .../fast/NonAlignedSeriesCompactionExecutor.java | 3 +- .../executor/fast/SeriesCompactionExecutor.java | 5 +- .../readchunk/AlignedSeriesCompactionExecutor.java | 8 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 8 +- .../readchunk/SingleSeriesCompactionExecutor.java | 8 +- .../utils/writer/AbstractCompactionWriter.java | 10 +- .../writer/AbstractCrossCompactionWriter.java | 3 +- .../writer/AbstractInnerCompactionWriter.java | 3 +- .../compaction/io/CompactionTsFileReader.java | 8 +- .../compaction/repair/RepairDataFileScanUtil.java | 20 +- .../estimator/AbstractCompactionEstimator.java | 5 +- .../estimator/CompactionEstimateUtils.java | 5 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 3 +- .../utils/CrossSpaceCompactionCandidate.java | 3 +- .../compaction/selector/utils/DeviceInfo.java | 6 +- .../selector/utils/TsFileResourceCandidate.java | 11 +- .../compaction/tool/SingleSequenceFileTask.java | 3 +- .../compaction/tool/TsFileStatisticReader.java | 11 +- .../compaction/tool/UnseqSpaceStatistics.java | 18 +- .../dataregion/flush/MemTableFlushTask.java | 14 +- .../dataregion/memtable/AbstractMemTable.java | 41 +- .../dataregion/memtable/DeviceIDFactory.java | 7 +- .../dataregion/memtable/IMemTable.java | 3 +- .../dataregion/memtable/PrimitiveMemTable.java | 2 + .../dataregion/memtable/TsFileProcessor.java | 56 +- .../dataregion/read/QueryDataSource.java | 5 +- .../dataregion/tsfile/TsFileResource.java | 31 +- .../tsfile/timeindex/DeviceTimeIndex.java | 56 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 23 +- .../dataregion/tsfile/timeindex/ITimeIndex.java | 29 +- .../dataregion/utils/TsFileResourceUtils.java | 25 +- .../storageengine/dataregion/wal/WALManager.java | 5 +- .../wal/recover/file/TsFilePlanRedoer.java | 6 +- .../file/UnsealedTsFileRecoverPerformer.java | 18 +- .../dataregion/wal/utils/WALWriteUtils.java | 18 + .../rescon/memory/TsFileResourceManager.java | 35 +- .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 5 +- .../iotdb/db/tools/TsFileResourcePrinter.java | 6 +- .../apache/iotdb/db/tools/TsFileSelfCheckTool.java | 24 +- .../apache/iotdb/db/tools/TsFileSketchTool.java | 52 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 27 +- .../org/apache/iotdb/db/tools/TsFileSplitTool.java | 11 +- .../TsFileOverlapValidationAndRepairTool.java | 14 +- .../db/tools/validate/TsFileValidationTool.java | 15 +- .../db/metadata/idtable/entry/DeviceIDTest.java | 2 +- .../schemaRegion/AbstractSchemaRegionTest.java | 22 +- .../event/TsFileInsertionDataContainerTest.java | 7 +- .../db/pipe/extractor/PipeRealtimeExtractTest.java | 4 +- .../execution/operator/AlignedSeriesTestUtil.java | 13 +- .../db/storageengine/buffer/ChunkCacheTest.java | 5 +- .../storageengine/dataregion/DataRegionTest.java | 4 +- .../dataregion/LastFlushTimeMapTest.java | 31 +- .../TsFileResourceProgressIndexTest.java | 11 +- .../compaction/AbstractCompactionTest.java | 26 +- .../compaction/CompactionValidationTest.java | 33 +- .../CrossSpaceCompactionWithUnusualCasesTest.java | 15 +- .../compaction/FastAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../FastCompactionPerformerWithEmptyPageTest.java | 4 +- ...InconsistentCompressionTypeAndEncodingTest.java | 5 +- .../FastCrossCompactionPerformerTest.java | 1152 +++++++++++------ .../FastInnerCompactionPerformerTest.java | 63 +- .../FastNonAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../compaction/ReadChunkInnerCompactionTest.java | 55 +- .../ReadPointAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../ReadPointCompactionPerformerTest.java | 1357 +++++++++++++------- .../ReadPointNonAlignedCrossCompactionTest.java | 1267 ++++++++++++------ .../compaction/TestUtilsForAlignedSeries.java | 9 +- .../TsFileValidationCorrectnessTests.java | 25 +- ...eCompactionWithFastPerformerValidationTest.java | 16 +- ...actionWithReadPointPerformerValidationTest.java | 11 +- .../InsertionCrossSpaceCompactionRecoverTest.java | 22 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 156 +-- .../dataregion/compaction/cross/MergeTest.java | 12 +- .../compaction/cross/MergeUpgradeTest.java | 5 +- .../cross/RewriteCompactionFileSelectorTest.java | 17 +- .../inner/AbstractInnerSpaceCompactionTest.java | 5 +- .../inner/InnerCompactionMoreDataTest.java | 5 +- .../compaction/inner/InnerCompactionTest.java | 5 +- .../ReadChunkCompactionPerformerAlignedTest.java | 3 +- .../SizeTieredCompactionSelectorTest.java | 5 +- .../inner/sizetiered/SizeTieredCompactionTest.java | 5 +- .../repair/RepairUnsortedFileCompactionTest.java | 3 +- .../compaction/tools/UnseqSpaceStatisticsTest.java | 46 +- .../compaction/utils/CompactionCheckerUtils.java | 14 +- .../utils/CompactionFileGeneratorUtils.java | 29 +- .../compaction/utils/CompactionTestFileWriter.java | 8 +- .../utils/MultiTsFileDeviceIteratorTest.java | 144 ++- .../dataregion/memtable/MemTableTestUtils.java | 6 +- .../dataregion/memtable/TsFileProcessorTest.java | 11 +- .../series/AbstractAlignedSeriesScanTest.java | 10 +- .../read/reader/series/AbstractSeriesScanTest.java | 6 +- .../AlignedSeriesScanLimitOffsetPushDownTest.java | 3 +- .../AlignedSeriesScanPredicatePushDownTest.java | 3 +- ...gleColumnSeriesScanLimitOffsetPushDownTest.java | 3 +- .../read/reader/series/SeriesReaderTestUtil.java | 5 +- .../dataregion/snapshot/IoTDBSnapshotTest.java | 6 +- .../dataregion/tsfile/TsFileResourceTest.java | 16 +- .../wal/allocation/ElasticStrategyTest.java | 12 + .../wal/allocation/FirstCreateStrategyTest.java | 12 + .../wal/allocation/RoundRobinStrategyTest.java | 12 + .../dataregion/wal/buffer/WALBufferCommonTest.java | 3 - .../dataregion/wal/io/WALFileTest.java | 5 - .../wal/node/ConsensusReqReaderTest.java | 2 - .../dataregion/wal/node/WALEntryHandlerTest.java | 4 - .../dataregion/wal/node/WALNodeTest.java | 23 +- .../wal/node/WalDeleteOutdatedNewTest.java | 16 +- .../wal/recover/WALRecoverManagerTest.java | 18 +- .../file/SealedTsFileRecoverPerformerTest.java | 6 +- .../wal/recover/file/TsFilePlanRedoerTest.java | 18 +- .../file/UnsealedTsFileRecoverPerformerTest.java | 11 +- .../wal/utils/WALInsertNodeCacheTest.java | 4 - .../rescon/memory/ResourceManagerTest.java | 16 +- .../org/apache/iotdb/db/tools/MLogParserTest.java | 8 + .../TsFileOverlapValidationAndRepairToolTest.java | 97 +- .../iotdb/db/tools/TsFileSelfCheckToolTest.java | 2 +- .../iotdb/db/tools/TsFileSketchToolTest.java | 13 +- .../db/utils/SchemaRegionSnapshotParserTest.java | 18 +- .../assembly/resources/sbin/start-standalone.bat | 3 + .../assembly/resources/sbin/start-standalone.sh | 2 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 1 + .../apache/iotdb/commons/path/MeasurementPath.java | 7 + .../org/apache/iotdb/commons/path/PartialPath.java | 10 + .../iotdb/tsfile/file/IMetadataIndexEntry.java} | 23 +- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 24 +- .../tsfile/file/metadata/ChunkGroupMetadata.java | 6 +- ...dexEntry.java => DeviceMetadataIndexEntry.java} | 55 +- .../iotdb/tsfile/file/metadata}/IDeviceID.java | 33 +- ...try.java => MeasurementMetadataIndexEntry.java} | 36 +- .../file/metadata/MetadataIndexConstructor.java | 37 +- .../tsfile/file/metadata/MetadataIndexNode.java | 51 +- .../iotdb/tsfile/file/metadata}/PlainDeviceID.java | 36 +- .../iotdb/tsfile/file/metadata/TsFileMetadata.java | 2 +- .../iotdb/tsfile/read/TsFileDeviceIterator.java | 17 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 320 ++--- ...leSequenceReaderTimeseriesMetadataIterator.java | 41 +- .../org/apache/iotdb/tsfile/read/common/Path.java | 16 + .../read/controller/MetadataQuerierByFileImpl.java | 25 +- .../apache/iotdb/tsfile/write/TsFileWriter.java | 46 +- .../write/chunk/AlignedChunkGroupWriterImpl.java | 8 +- .../chunk/NonAlignedChunkGroupWriterImpl.java | 8 +- .../apache/iotdb/tsfile/write/record/TSRecord.java | 7 + .../write/writer/ForceAppendTsFileWriter.java | 5 +- .../write/writer/RestorableTsFileIOWriter.java | 10 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 31 +- .../write/writer/TsFileIOWriterEndFileTest.java | 3 +- .../write/writer/tsmiterator/TSMIterator.java | 19 +- .../file/metadata/MetadataIndexNodeTest.java | 13 +- .../tsfile/file/metadata/utils/TestHelper.java | 4 +- .../iotdb/tsfile/read/GetAllDevicesTest.java | 8 +- ...easurementChunkMetadataListMapIteratorTest.java | 18 +- .../tsfile/read/TimeSeriesMetadataReadTest.java | 4 +- .../apache/iotdb/tsfile/read/TsFileReaderTest.java | 15 +- .../tsfile/read/TsFileSequenceReaderTest.java | 9 +- .../tsfile/write/MetadataIndexConstructorTest.java | 75 +- .../iotdb/tsfile/write/TsFileIOWriterTest.java | 15 +- .../tsfile/write/TsFileIntegrityCheckingTool.java | 10 +- .../iotdb/tsfile/write/TsFileWriteApiTest.java | 7 +- .../write/writer/RestorableTsFileIOWriterTest.java | 3 +- .../writer/TsFileIOWriterMemoryControlTest.java | 96 +- pom.xml | 4 +- 255 files changed, 8022 insertions(+), 4448 deletions(-)
