This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new cf718f6e444 Use MemoizedCheckedSupplier for COPY TO folder manager &
keep compatibility with another branch (#17559)
cf718f6e444 is described below
commit cf718f6e44454968797e8e4cbe18eb179f9eb2b5
Author: shuwenwei <[email protected]>
AuthorDate: Mon Apr 27 18:35:44 2026 +0800
Use MemoizedCheckedSupplier for COPY TO folder manager & keep compatibility
with another branch (#17559)
---
.../relational/ColumnTransformerBuilder.java | 11 ++++++++++-
.../planner/ITableOperatorGeneratorContext.java | 3 +++
.../calc/plan/planner/TableOperatorGenerator.java | 6 ++++--
iotdb-core/datanode/pom.xml | 3 +++
.../planner/DataNodeTableOperatorGenerator.java | 3 ++-
.../plan/planner/LocalExecutionPlanContext.java | 6 ++++++
.../plan/relational/sql/ast/DeleteDevice.java | 3 ++-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 6 ++++--
.../db/service/DataNodeInternalRPCService.java | 4 ++--
.../db/storageengine/rescon/disk/TierManager.java | 23 +++++++++-------------
10 files changed, 45 insertions(+), 23 deletions(-)
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java
index 42f8f8bed59..6b74741e4fd 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/relational/ColumnTransformerBuilder.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.calc.execution.relational;
+import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager;
import org.apache.iotdb.calc.plan.relational.metadata.ITypeMetadata;
import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer;
import
org.apache.iotdb.calc.transformation.dag.column.FailFunctionColumnTransformer;
@@ -215,6 +216,8 @@ import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.read.common.type.TypeEnum;
import org.apache.tsfile.utils.Binary;
+import javax.annotation.Nullable;
+
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
@@ -1944,6 +1947,10 @@ public class ColumnTransformerBuilder
private final ITypeMetadata metadata;
+ // used in other branch
+ @SuppressWarnings("unused")
+ private final Optional<MemoryReservationManager> memoryReservationManager;
+
public Context(
SessionInfo sessionInfo,
List<LeafColumnTransformer> leafList,
@@ -1954,7 +1961,8 @@ public class ColumnTransformerBuilder
List<TSDataType> inputDataTypes,
int originSize,
ITableTypeProvider typeProvider,
- ITypeMetadata metadata) {
+ ITypeMetadata metadata,
+ @Nullable MemoryReservationManager memoryReservationManager) {
this.sessionInfo = sessionInfo;
this.leafList = leafList;
this.inputLocations = inputLocations;
@@ -1965,6 +1973,7 @@ public class ColumnTransformerBuilder
this.originSize = originSize;
this.typeProvider = typeProvider;
this.metadata = metadata;
+ this.memoryReservationManager =
Optional.ofNullable(memoryReservationManager);
}
public Type getType(SymbolReference symbolReference) {
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java
index ffee1d2e8da..ccd843ccb62 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/ITableOperatorGeneratorContext.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.calc.plan.planner;
+import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager;
import org.apache.iotdb.commons.queryengine.plan.analyze.ITableTypeProvider;
import java.time.ZoneId;
@@ -26,5 +27,7 @@ import java.time.ZoneId;
public interface ITableOperatorGeneratorContext {
ITableTypeProvider getTableTypeProvider();
+ MemoryReservationManager getMemoryReservationManager();
+
ZoneId getZoneId();
}
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java
index aebce95bdb9..86edc71a8e0 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java
@@ -331,7 +331,8 @@ public abstract class TableOperatorGenerator<
ImmutableList.of(),
0,
context.getTableTypeProvider(),
- metadata);
+ metadata,
+ context.getMemoryReservationManager());
return visitor.process(p, filterColumnTransformerContext);
})
@@ -358,7 +359,8 @@ public abstract class TableOperatorGenerator<
filterOutputDataTypes,
inputLocations.size(),
context.getTableTypeProvider(),
- metadata);
+ metadata,
+ context.getMemoryReservationManager());
for (Expression expression : projectExpressions) {
projectOutputTransformerList.add(
diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml
index d3f5f4a669f..bce270a5933 100644
--- a/iotdb-core/datanode/pom.xml
+++ b/iotdb-core/datanode/pom.xml
@@ -407,6 +407,9 @@
<resource>
<directory>${maven.multiModuleProjectDirectory}/iotdb-core/calc-commons/src/main/codegen</directory>
<filtering>false</filtering>
+ <excludes>
+ <exclude>templates/**</exclude>
+ </excludes>
</resource>
<resource>
<directory>src/main/codegen</directory>
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java
index d63b1a0962d..e2640bfb5ea 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java
@@ -1240,7 +1240,8 @@ public class DataNodeTableOperatorGenerator
ImmutableList.of(),
0,
context.getTypeProvider(),
- metadata)),
+ metadata,
+ context.getMemoryReservationManager())),
columnSchemaList,
database,
table)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java
index 54e475c00d8..65908708394 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.planner;
import org.apache.iotdb.calc.execution.operator.Operator;
import org.apache.iotdb.calc.plan.planner.ITableOperatorGeneratorContext;
+import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager;
import org.apache.iotdb.commons.queryengine.plan.analyze.ITableTypeProvider;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
@@ -314,6 +315,11 @@ public class LocalExecutionPlanContext implements
ITableOperatorGeneratorContext
return typeProvider;
}
+ @Override
+ public MemoryReservationManager getMemoryReservationManager() {
+ return
driverContext.getFragmentInstanceContext().getMemoryReservationContext();
+ }
+
public FragmentInstanceContext getInstanceContext() {
return driverContext.getFragmentInstanceContext();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
index 5ec7102c515..d2fa9bbabde 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
@@ -233,7 +233,8 @@ public class DeleteDevice extends AbstractTraverseDevice {
ImmutableList.of(),
0,
mockTypeProvider,
- metadata))
+ metadata,
+ null))
: null;
return new DeviceBlackListConstructor(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
index 270623b23ee..754cca02092 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
@@ -1574,7 +1574,8 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
ImmutableList.of(),
0,
mockTypeProvider,
- metadata))
+ metadata,
+ null))
: null;
final List<TSDataType> filterOutputDataTypes =
@@ -1601,7 +1602,8 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
filterOutputDataTypes,
inputLocations.size(),
mockTypeProvider,
- metadata);
+ metadata,
+ null);
final List<String> attributeNames =
updateNode.getAssignments().stream()
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java
index f3bf8e507c2..b58d647fcd7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java
@@ -44,8 +44,8 @@ public class DataNodeInternalRPCService extends ThriftService
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
private static final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
- private final AtomicReference<DataNodeInternalRPCServiceImpl> impl = new
AtomicReference<>();
- private DataNodeContext dataNodeContext;
+ protected final AtomicReference<DataNodeInternalRPCServiceImpl> impl = new
AtomicReference<>();
+ protected DataNodeContext dataNodeContext;
private DataNodeInternalRPCService() {}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index 0dec9ce465d..02a95ff5511 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -30,6 +30,7 @@ import
org.apache.iotdb.db.storageengine.rescon.disk.strategy.RandomOnDiskUsable
import org.apache.iotdb.metrics.utils.FileStoreUtils;
import com.google.common.io.BaseEncoding;
+import org.apache.ratis.util.MemoizedCheckedSupplier;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.fileSystem.FSType;
import org.apache.tsfile.utils.FSUtils;
@@ -82,7 +83,10 @@ public class TierManager {
private List<String> copyToTargetDirs;
- private FolderManager copyToFolderManager;
+ // private FolderManager copyToFolderManager;
+
+ private MemoizedCheckedSupplier<FolderManager,
DiskSpaceInsufficientException>
+ copyToFolderManager;
/** total space of each tier, Long.MAX_VALUE when one tier contains remote
storage */
private long[] tierDiskTotalSpace;
@@ -175,11 +179,9 @@ public class TierManager {
.getFile(v,
IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME)
.getPath())
.collect(Collectors.toList());
- try {
- copyToFolderManager = new FolderManager(copyToTargetDirs,
directoryStrategyType);
- } catch (DiskSpaceInsufficientException e) {
- logger.error("All disks of tier {} are full.", tierLevel, e);
- }
+ copyToFolderManager =
+ MemoizedCheckedSupplier.valueOf(
+ () -> new FolderManager(copyToTargetDirs,
directoryStrategyType));
}
objectDirs =
@@ -248,14 +250,7 @@ public class TierManager {
}
public String getNextFolderForCopyToTargetFile() throws
DiskSpaceInsufficientException {
- if (copyToFolderManager == null) {
- throw new DiskSpaceInsufficientException(
- "copyToFolderManager is not initialized. This usually indicates that
folder "
- + "initialization in TierManager.initFolders() failed due to
insufficient disk "
- + "space. Please check disk space and related configuration
before retrying the "
- + "copy-to-target operation.");
- }
- return copyToFolderManager.getNextFolder();
+ return copyToFolderManager.get().getNextFolder();
}
public String getNextFolderForObjectFile() throws
DiskSpaceInsufficientException {
