This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch KeepAllNullRows in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ce6ab037f63c5afee0ebe081d17c02e48aca5c3e Merge: eb6d437c891 13097fe60d6 Author: JackieTien97 <[email protected]> AuthorDate: Thu Sep 12 12:01:53 2024 +0800 resolve conflicts .asf.yaml | 2 + .github/workflows/multi-language-client.yml | 17 +- .../org/apache/iotdb/it/env/cluster/env/AIEnv.java | 2 +- .../java/org/apache/iotdb/ainode/it/AIEnvIT.java | 80 -- .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 228 ++++ .../org/apache/iotdb/db/it/IoTDBEncodingIT.java | 100 +- .../iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java | 13 +- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 22 +- .../src/test/resources/ainode-example/config.yaml | 5 + .../src/test/resources/ainode-example/model.pt | Bin 0 -> 1906 bytes iotdb-client/client-py/TableModelSessionExample.py | 154 +++ .../client-py/TableModelSessionPoolExample.py | 160 +++ iotdb-client/client-py/iotdb/Session.py | 113 +- iotdb-client/client-py/iotdb/SessionPool.py | 11 + iotdb-client/client-py/iotdb/utils/Field.py | 15 +- .../client-py/iotdb/utils/IoTDBConstants.py | 6 + .../client-py/iotdb/utils/IoTDBRpcDataSet.py | 16 +- iotdb-client/client-py/iotdb/utils/NumpyTablet.py | 58 +- .../client-py/iotdb/utils/SessionDataSet.py | 4 +- iotdb-client/client-py/iotdb/utils/Tablet.py | 67 +- iotdb-client/client-py/pom.xml | 2 +- iotdb-client/client-py/release.sh | 1 + .../tests/integration/test_relational_session.py | 143 +++ iotdb-core/ainode/iotdb/ainode/client.py | 225 ++-- iotdb-core/ainode/iotdb/ainode/config.py | 240 ++-- iotdb-core/ainode/iotdb/ainode/constant.py | 1 - iotdb-core/ainode/iotdb/ainode/exception.py | 23 +- iotdb-core/ainode/iotdb/ainode/factory.py | 272 ---- iotdb-core/ainode/iotdb/ainode/handler.py | 105 +- iotdb-core/ainode/iotdb/ainode/log.py | 45 +- .../ainode/{encryption.py => manager/__init__.py} | 0 .../ainode/iotdb/ainode/manager/cluster_manager.py | 41 + .../{inference.py => manager/inference_manager.py} | 176 ++- .../ainode/iotdb/ainode/manager/model_manager.py | 84 ++ .../ainode/{encryption.py => model/__init__.py} | 0 .../built_in_model_factory.py} | 315 ++++- .../ainode/iotdb/ainode/model/model_factory.py | 237 ++++ .../ainode/iotdb/ainode/model/model_storage.py | 113 ++ iotdb-core/ainode/iotdb/ainode/parser.py | 191 --- iotdb-core/ainode/iotdb/ainode/script.py | 177 ++- iotdb-core/ainode/iotdb/ainode/service.py | 100 +- iotdb-core/ainode/iotdb/ainode/storage.py | 318 ----- iotdb-core/ainode/iotdb/ainode/util.py | 79 -- .../ainode/{encryption.py => util/__init__.py} | 0 .../ainode/{encryption.py => util/decorator.py} | 9 + iotdb-core/ainode/iotdb/ainode/util/lock.py | 84 ++ iotdb-core/ainode/iotdb/ainode/{ => util}/serde.py | 0 .../ainode/iotdb/ainode/util/status.py | 31 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 5 + .../client/IoTDBDataNodeAsyncClientManager.java | 49 +- .../connector/protocol/opcua/OpcUaConnector.java | 57 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 18 +- .../protocol/opcua/OpcUaServerBuilder.java | 43 +- .../async/IoTDBDataRegionAsyncConnector.java | 8 + .../plan/analyze/LoadTsFileAnalyzer.java | 44 +- .../TableModelStatementMemorySourceVisitor.java | 5 +- .../plan/planner/TableOperatorGenerator.java | 13 + .../plan/planner/plan/node/PlanGraphPrinter.java | 58 + .../plan/planner/plan/node/PlanNodeType.java | 8 + .../plan/planner/plan/node/PlanVisitor.java | 12 + .../ContinuousSameSearchIndexSeparatorNode.java | 83 +- .../plan/relational/analyzer/Analysis.java | 38 + .../relational/analyzer/ExpressionAnalyzer.java | 27 + .../relational/metadata/FunctionNullability.java | 10 + .../plan/relational/metadata/Metadata.java | 5 + .../metadata/TableBuiltinAggregationFunction.java | 116 ++ .../relational/metadata/TableMetadataImpl.java | 8 +- .../plan/relational/planner/NodeAndMappings.java | 38 + .../plan/relational/planner/QueryPlanner.java | 357 ++++++ .../plan/relational/planner/SymbolAllocator.java | 10 + .../plan/relational/planner/SymbolsExtractor.java | 27 +- .../relational/planner/TableLogicalPlanner.java | 24 +- .../plan/relational/planner/TableModelPlanner.java | 6 +- .../distribute/TableDistributedPlanGenerator.java | 194 ++- .../distribute/TableDistributedPlanner.java | 18 +- .../TableModelTypeProviderExtractor.java | 12 + .../iterative/rule/PruneAggregationColumns.java | 47 + .../rule/PruneAggregationSourceColumns.java | 54 + .../iterative/rule/PruneTableScanColumns.java | 5 +- .../relational/planner/node/AggregationNode.java | 542 ++++++++ .../planner/node/AggregationTableScanNode.java | 261 ++++ .../plan/relational/planner/node/Patterns.java | 9 +- .../relational/planner/node/TableScanNode.java | 45 +- .../optimizations/LogicalOptimizeFactory.java | 17 + .../PushAggregationIntoTableScan.java | 226 ++++ .../PushLimitOffsetIntoTableScan.java | 7 + .../optimizations/PushPredicateIntoTableScan.java | 76 ++ .../planner/optimizations/SymbolMapper.java | 248 ++++ .../TransformAggregationToStreamable.java | 160 +++ .../optimizations/TransformSortToStreamSort.java | 26 +- .../optimizations/UnaliasSymbolReferences.java | 470 +++++++ .../relational/planner/optimizations/Util.java | 189 +++ .../plan/relational/sql/parser/AstBuilder.java | 2 +- .../iotdb/db/schemaengine/SchemaEngineMode.java | 3 +- .../java/org/apache/iotdb/db/service/DataNode.java | 28 +- .../iotdb/db/service/DataNodeShutdownHook.java | 88 +- .../execute/task/SettleCompactionTask.java | 8 + ...BatchedFastAlignedSeriesCompactionExecutor.java | 41 +- ...edReadChunkAlignedSeriesCompactionExecutor.java | 42 +- .../utils/AlignedSeriesBatchCompactionUtils.java | 161 ++- .../ReadChunkAlignedSeriesCompactionExecutor.java | 50 +- .../selector/utils/TsFileResourceCandidate.java | 3 +- .../dataregion/memtable/DeviceIDFactory.java | 16 +- .../dataregion/wal/buffer/WALEntry.java | 3 + .../file/UnsealedTsFileRecoverPerformer.java | 3 + .../load/memory/LoadTsFileAbstractMemoryBlock.java | 4 + .../memory/LoadTsFileAnalyzeSchemaMemoryBlock.java | 28 +- .../memory/LoadTsFileDataCacheMemoryBlock.java | 7 + .../load/memory/LoadTsFileMemoryManager.java | 28 +- .../load/splitter/TsFileSplitter.java | 13 +- .../apache/iotdb/db/utils/ModificationUtils.java | 14 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 51 +- ...nuousSameSearchIndexSeparatorNodeSerDeTest.java | 56 + .../plan/relational/analyzer/AggregationTest.java | 584 +++++++++ .../plan/relational/analyzer/JoinTest.java | 68 +- .../relational/analyzer/MockTSBSDataPartition.java | 173 +++ .../plan/relational/analyzer/SubQueryTest.java | 54 +- .../{TestMatadata.java => TSBSMetadata.java} | 303 +++-- .../plan/relational/analyzer/TSBSTest.java | 1342 ++++++++++++++++++++ .../plan/relational/analyzer/TestMatadata.java | 8 +- .../plan/relational/planner/ExampleTest.java | 13 +- .../plan/relational/planner/PlanTester.java | 34 +- .../planner/assertions/AggregationFunction.java | 62 + .../assertions/AggregationFunctionMatcher.java | 72 ++ .../assertions/AggregationFunctionProvider.java | 87 ++ .../planner/assertions/AggregationMatcher.java | 145 +++ .../planner/assertions/AggregationStepMatcher.java | 57 + .../assertions/AggregationTableScanMatcher.java | 186 +++ .../planner/assertions/ColumnReference.java | 10 +- .../planner/assertions/ExpectedValueProvider.java | 30 + .../planner/assertions/PlanMatchPattern.java | 179 ++- .../planner/assertions/PredicateMatcher.java | 46 + .../planner/assertions/TableScanMatcher.java | 10 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 76 ++ ...nkCompactionPerformerWithAlignedSeriesTest.java | 32 + .../settle/SettleCompactionTaskTest.java | 31 + .../compaction/utils/BatchCompactionUtilsTest.java | 12 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 4 - .../iotdb/commons/conf/CommonDescriptor.java | 3 - pom.xml | 1 + 141 files changed, 9704 insertions(+), 2339 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 4bb66a620ba,6d10c731e7e..5ecdc354ddc --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@@ -254,12 -191,13 +254,13 @@@ public class ModificationUtils * Check whether the device with start time and end time is completely deleted by mods or not. * There are some slight differences from that in {@link SettleSelectorImpl}. */ - public static boolean isDeviceDeletedByMods( + public static boolean isAllDeletedByMods( Collection<Modification> modifications, IDeviceID device, long startTime, long endTime) throws IllegalPathException { + final MeasurementPath deviceWithWildcard = new MeasurementPath(device, ONE_LEVEL_PATH_WILDCARD); for (Modification modification : modifications) { - PartialPath path = modification.getPath(); - if (path.include(new MeasurementPath(device, IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)) + MeasurementPath path = modification.getPath(); + if (path.matchFullPath(deviceWithWildcard) && ((Deletion) modification).getTimeRange().contains(startTime, endTime)) { return true; }
