This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch fix_0427 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6c3d538791e3da39c6dce655f0f7435f5c01ca34 Author: shuwenwei <[email protected]> AuthorDate: Mon Apr 27 17:00:30 2026 +0800 Use MemoizedCheckedSupplier for COPY TO folder manager & keep compatibility with another branch --- .../relational/ColumnTransformerBuilder.java | 11 ++++++++++- .../planner/ITableOperatorGeneratorContext.java | 3 +++ .../calc/plan/planner/TableOperatorGenerator.java | 6 ++++-- iotdb-core/datanode/pom.xml | 3 +++ .../planner/DataNodeTableOperatorGenerator.java | 3 ++- .../plan/planner/LocalExecutionPlanContext.java | 6 ++++++ .../plan/relational/sql/ast/DeleteDevice.java | 3 ++- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 6 ++++-- .../db/service/DataNodeInternalRPCService.java | 2 +- .../db/storageengine/rescon/disk/TierManager.java | 23 +++++++++------------- 10 files changed, 44 insertions(+), 22 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java index 42f8f8bed59..6b74741e4fd 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java @@ -19,6 +19,7 @@ package org.apache.iotdb.calc.execution.relational; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.calc.plan.relational.metadata.ITypeMetadata; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.FailFunctionColumnTransformer; @@ -215,6 +216,8 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.utils.Binary; +import javax.annotation.Nullable; + import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -1944,6 +1947,10 @@ public class ColumnTransformerBuilder private final ITypeMetadata metadata; + // used in other branch + @SuppressWarnings("unused") + private final Optional<MemoryReservationManager> memoryReservationManager; + public Context( SessionInfo sessionInfo, List<LeafColumnTransformer> leafList, @@ -1954,7 +1961,8 @@ public class ColumnTransformerBuilder List<TSDataType> inputDataTypes, int originSize, ITableTypeProvider typeProvider, - ITypeMetadata metadata) { + ITypeMetadata metadata, + @Nullable MemoryReservationManager memoryReservationManager) { this.sessionInfo = sessionInfo; this.leafList = leafList; this.inputLocations = inputLocations; @@ -1965,6 +1973,7 @@ public class ColumnTransformerBuilder this.originSize = originSize; this.typeProvider = typeProvider; this.metadata = metadata; + this.memoryReservationManager = Optional.ofNullable(memoryReservationManager); } public Type getType(SymbolReference symbolReference) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java index ffee1d2e8da..ccd843ccb62 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java @@ -19,6 +19,7 @@ package org.apache.iotdb.calc.plan.planner; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.commons.queryengine.plan.analyze.ITableTypeProvider; import java.time.ZoneId; @@ -26,5 +27,7 @@ import java.time.ZoneId; public interface ITableOperatorGeneratorContext { ITableTypeProvider getTableTypeProvider(); + MemoryReservationManager getMemoryReservationManager(); + ZoneId getZoneId(); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java index aebce95bdb9..86edc71a8e0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java @@ -331,7 +331,8 @@ public abstract class TableOperatorGenerator< ImmutableList.of(), 0, context.getTableTypeProvider(), - metadata); + metadata, + context.getMemoryReservationManager()); return visitor.process(p, filterColumnTransformerContext); }) @@ -358,7 +359,8 @@ public abstract class TableOperatorGenerator< filterOutputDataTypes, inputLocations.size(), context.getTableTypeProvider(), - metadata); + metadata, + context.getMemoryReservationManager()); for (Expression expression : projectExpressions) { projectOutputTransformerList.add( diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index d3f5f4a669f..bce270a5933 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -407,6 +407,9 @@ <resource> <directory>${maven.multiModuleProjectDirectory}/iotdb-core/calc-commons/src/main/codegen</directory> <filtering>false</filtering> + <excludes> + <exclude>templates/**</exclude> + </excludes> </resource> <resource> <directory>src/main/codegen</directory> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index d63b1a0962d..e2640bfb5ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1240,7 +1240,8 @@ public class DataNodeTableOperatorGenerator ImmutableList.of(), 0, context.getTypeProvider(), - metadata)), + metadata, + context.getMemoryReservationManager())), columnSchemaList, database, table) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index 54e475c00d8..65908708394 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.planner; import org.apache.iotdb.calc.execution.operator.Operator; import org.apache.iotdb.calc.plan.planner.ITableOperatorGeneratorContext; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.commons.queryengine.plan.analyze.ITableTypeProvider; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -314,6 +315,11 @@ public class LocalExecutionPlanContext implements ITableOperatorGeneratorContext return typeProvider; } + @Override + public MemoryReservationManager getMemoryReservationManager() { + return driverContext.getFragmentInstanceContext().getMemoryReservationContext(); + } + public FragmentInstanceContext getInstanceContext() { return driverContext.getFragmentInstanceContext(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java index 5ec7102c515..d2fa9bbabde 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java @@ -233,7 +233,8 @@ public class DeleteDevice extends AbstractTraverseDevice { ImmutableList.of(), 0, mockTypeProvider, - metadata)) + metadata, + null)) : null; return new DeviceBlackListConstructor( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 270623b23ee..754cca02092 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1574,7 +1574,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { ImmutableList.of(), 0, mockTypeProvider, - metadata)) + metadata, + null)) : null; final List<TSDataType> filterOutputDataTypes = @@ -1601,7 +1602,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { filterOutputDataTypes, inputLocations.size(), mockTypeProvider, - metadata); + metadata, + null); final List<String> attributeNames = updateNode.getAssignments().stream() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java index f3bf8e507c2..3af1e668dfa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java @@ -44,7 +44,7 @@ public class DataNodeInternalRPCService extends ThriftService private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); - private final AtomicReference<DataNodeInternalRPCServiceImpl> impl = new AtomicReference<>(); + protected final AtomicReference<DataNodeInternalRPCServiceImpl> impl = new AtomicReference<>(); private DataNodeContext dataNodeContext; private DataNodeInternalRPCService() {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index 0dec9ce465d..02a95ff5511 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.storageengine.rescon.disk.strategy.RandomOnDiskUsable import org.apache.iotdb.metrics.utils.FileStoreUtils; import com.google.common.io.BaseEncoding; +import org.apache.ratis.util.MemoizedCheckedSupplier; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.utils.FSUtils; @@ -82,7 +83,10 @@ public class TierManager { private List<String> copyToTargetDirs; - private FolderManager copyToFolderManager; + // private FolderManager copyToFolderManager; + + private MemoizedCheckedSupplier<FolderManager, DiskSpaceInsufficientException> + copyToFolderManager; /** total space of each tier, Long.MAX_VALUE when one tier contains remote storage */ private long[] tierDiskTotalSpace; @@ -175,11 +179,9 @@ public class TierManager { .getFile(v, IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME) .getPath()) .collect(Collectors.toList()); - try { - copyToFolderManager = new FolderManager(copyToTargetDirs, directoryStrategyType); - } catch (DiskSpaceInsufficientException e) { - logger.error("All disks of tier {} are full.", tierLevel, e); - } + copyToFolderManager = + MemoizedCheckedSupplier.valueOf( + () -> new FolderManager(copyToTargetDirs, directoryStrategyType)); } objectDirs = @@ -248,14 +250,7 @@ public class TierManager { } public String getNextFolderForCopyToTargetFile() throws DiskSpaceInsufficientException { - if (copyToFolderManager == null) { - throw new DiskSpaceInsufficientException( - "copyToFolderManager is not initialized. This usually indicates that folder " - + "initialization in TierManager.initFolders() failed due to insufficient disk " - + "space. Please check disk space and related configuration before retrying the " - + "copy-to-target operation."); - } - return copyToFolderManager.getNextFolder(); + return copyToFolderManager.get().getNextFolder(); } public String getNextFolderForObjectFile() throws DiskSpaceInsufficientException {
