This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch cacheMetadataIndexNodeOffsetsForQuery in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 48ca1eaf94b9ee762c1b7a4c1f500aa80fd6017c Author: shuwenwei <[email protected]> AuthorDate: Fri Jul 25 16:44:03 2025 +0800 add memory limit --- .../fragment/DeviceMetadataIndexEntryCache.java | 23 ++++++++++++++++++++++ .../fragment/FragmentInstanceContext.java | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/DeviceMetadataIndexEntryCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/DeviceMetadataIndexEntryCache.java index a0c71c40e6e..316cdb4ae02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/DeviceMetadataIndexEntryCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/DeviceMetadataIndexEntryCache.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManag import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; import java.io.IOException; import java.util.ArrayList; @@ -35,11 +36,17 @@ import java.util.Map; import java.util.TreeMap; public class DeviceMetadataIndexEntryCache { + private static final long MAX_CACHED_SIZE = 32 * 1024 * 1024; + private final FragmentInstanceContext context; private TreeMap<IDeviceID, Integer> deviceIndexMap; private final Map<String, long[]> deviceMetadataIndexNodeOffsetsCache = new HashMap<>(); private List<IDeviceID> sortedDevices; private int[] deviceIdxArr; + public DeviceMetadataIndexEntryCache(FragmentInstanceContext context) { + this.context = context; + } + public void addDevices(AbstractDataSourceOperator operator, List<DeviceEntry> deviceEntries) { deviceIndexMap = deviceIndexMap == null ? new TreeMap<>(IDeviceID::compareTo) : deviceIndexMap; int[] operatorDeviceIndexArr = new int[deviceEntries.size()]; @@ -84,6 +91,9 @@ public class DeviceMetadataIndexEntryCache { if (offsets != null) { return offsets; } + if (!reserveMemory()) { + return null; + } TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true); IDeviceID firstDevice = getSortedDevices().get(0); offsets = @@ -110,4 +120,17 @@ public class DeviceMetadataIndexEntryCache { } deviceIndexMap = null; } + + private boolean reserveMemory() { + long costOfOneFile = RamUsageEstimator.sizeOfLongArray(sortedDevices.size()); + if (costOfOneFile * (deviceMetadataIndexNodeOffsetsCache.size() + 1) > MAX_CACHED_SIZE) { + return false; + } + try { + context.getMemoryReservationContext().reserveMemoryCumulatively(costOfOneFile); + return true; + } catch (Exception ignored) { + return false; + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 0844a18b602..cfbde076a8d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -145,8 +145,8 @@ public class FragmentInstanceContext extends QueryContext { private long closedSeqFileNum = 0; private long closedUnseqFileNum = 0; - private DeviceMetadataIndexEntryCache metadataIndexEntryCache = - new DeviceMetadataIndexEntryCache(); + private final DeviceMetadataIndexEntryCache metadataIndexEntryCache = + new DeviceMetadataIndexEntryCache(this); public static FragmentInstanceContext createFragmentInstanceContext( FragmentInstanceId id, FragmentInstanceStateMachine stateMachine, SessionInfo sessionInfo) {
