This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch addMemoryControlForModEntriesInQuery in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dac73b1577bfbc5ece0e26ce90a1eb8022948fd6 Author: shuwenwei <[email protected]> AuthorDate: Mon Aug 11 16:07:24 2025 +0800 add memory control for mod entries in query --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 21 ++++++ .../execution/MemoryEstimationHelper.java | 47 +++++++++++++ .../fragment/FragmentInstanceContext.java | 5 ++ .../execution/fragment/QueryContext.java | 79 ++++++++++++++++------ .../dataregion/modification/DeletionPredicate.java | 15 +++- .../dataregion/modification/IDPredicate.java | 32 ++++++++- .../dataregion/modification/ModEntry.java | 3 +- .../modification/TableDeletionEntry.java | 11 +++ .../dataregion/modification/TreeDeletionEntry.java | 11 +++ .../conf/iotdb-system.properties.template | 7 ++ .../apache/iotdb/commons/path/PathPatternNode.java | 18 ++++- .../apache/iotdb/commons/path/PatternTreeMap.java | 15 +++- 13 files changed, 248 insertions(+), 27 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 75dac521f2f..a075667245d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -417,6 +417,9 @@ public class IoTDBConfig { /** The buffer for sort operation */ private long sortBufferSize = 32 * 1024 * 1024L; + /** Mods cache size limit per query */ + private long modsCacheSizeLimitPerFI = 32 * 1024 * 1024; + /** * The strategy of inner space compaction task. There are just one inner space compaction strategy * SIZE_TIRED_COMPACTION: @@ -4036,6 +4039,14 @@ public class IoTDBConfig { return sortBufferSize; } + public void setModsCacheSizeLimitPerFI(long modsCacheSizeLimitPerFI) { + this.modsCacheSizeLimitPerFI = modsCacheSizeLimitPerFI; + } + + public long getModsCacheSizeLimitPerFI() { + return modsCacheSizeLimitPerFI; + } + public void setSortTmpDir(String sortTmpDir) { this.sortTmpDir = sortTmpDir; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 7a6bfc8aa58..b18eb98b167 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1029,6 +1029,8 @@ public class IoTDBDescriptor { // The buffer for sort operator to calculate loadSortBuffer(properties); + loadModsCacheSizeLimitPerFI(properties); + // tmp filePath for sort operator conf.setSortTmpDir(properties.getProperty("sort_tmp_dir", conf.getSortTmpDir())); @@ -1110,6 +1112,23 @@ public class IoTDBDescriptor { / 2); } + private void loadModsCacheSizeLimitPerFI(TrimProperties properties) { + long defaultValue = + Math.min( + 32 * 1024 * 1024L, + memoryConfig.getOperatorsMemoryManager().getTotalMemorySizeInBytes() + / memoryConfig.getQueryThreadCount() + / 2); + long size = + Long.parseLong( + properties.getProperty( + "mods_cache_size_limit_per_fi_in_bytes", Long.toString(defaultValue))); + if (size <= 0) { + size = defaultValue; + } + conf.setModsCacheSizeLimitPerFI(size); + } + private void reloadConsensusProps(TrimProperties properties) throws IOException { loadIoTConsensusProps(properties); loadIoTConsensusV2Props(properties); @@ -2084,6 +2103,8 @@ public class IoTDBDescriptor { // sort_buffer_size_in_bytes loadSortBuffer(properties); + + loadModsCacheSizeLimitPerFI(properties); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java index f7318865c2e..42fd6731974 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.RamUsageEstimator; @@ -31,6 +32,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; public class MemoryEstimationHelper { @@ -47,6 +49,8 @@ public class MemoryEstimationHelper { RamUsageEstimator.shallowSizeOfInstance(ArrayList.class); private static final long INTEGER_INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(Integer.class); + public static final long TIME_RANGE_INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TimeRange.class); private MemoryEstimationHelper() { // hide the constructor @@ -119,4 +123,47 @@ public class MemoryEstimationHelper { size += INTEGER_INSTANCE_SIZE * integerArrayList.size(); return RamUsageEstimator.alignObjectSize(size); } + + public static long getEstimatedSizeOfStringArrayList(List<String> stringArrayList) { + if (stringArrayList == null) { + return 0L; + } + long size = ARRAY_LIST_INSTANCE_SIZE; + size += + (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + + (long) stringArrayList.size() * (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF; + for (String str : stringArrayList) { + size += RamUsageEstimator.sizeOf(str); + } + return RamUsageEstimator.alignObjectSize(size); + } + + public static long getEstimatedSizeOfAccountableArrayList( + List<? extends Accountable> stringArrayList) { + if (stringArrayList == null) { + return 0L; + } + long size = ARRAY_LIST_INSTANCE_SIZE; + size += + (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + + (long) stringArrayList.size() * (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF; + for (Accountable obj : stringArrayList) { + size += RamUsageEstimator.sizeOfObject(obj); + } + return RamUsageEstimator.alignObjectSize(size); + } + + public static long getEstimatedSizeOfHashSet(Set<?> set) { + if (set == null) { + return 0L; + } else { + long size = + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP + + (long) set.size() * RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY; + for (Object obj : set) { + size += RamUsageEstimator.sizeOfObject(obj); + } + return RamUsageEstimator.alignObjectSize(size); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 8fc462bbd42..e0944b17fc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -791,6 +791,7 @@ public class FragmentInstanceContext extends QueryContext { // release TVList/AlignedTVList owned by current query releaseTVListOwnedByQuery(); + fileModCache = null; dataRegion = null; globalTimeFilter = null; sharedQueryDataSource = null; @@ -967,4 +968,8 @@ public class FragmentInstanceContext extends QueryContext { public boolean ignoreNotExistsDevice() { return ignoreNotExistsDevice; } + + public List<IFullPath> getSourcePaths() { + return sourcePaths; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java index c2f95e39b17..bcd062768f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java @@ -22,6 +22,8 @@ package org.apache.iotdb.db.queryengine.execution.fragment; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -31,6 +33,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory.ModsSeriali import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.RamUsageEstimator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,13 +44,15 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; /** QueryContext contains the shared information with in a query. */ public class QueryContext { private static final Logger LOGGER = LoggerFactory.getLogger(QueryContext.class); + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private QueryStatistics queryStatistics = new QueryStatistics(); /** @@ -55,9 +60,11 @@ public class QueryContext { * use this field because each call of Modification.getModifications() return a copy of the * Modifications, and we do not want it to create multiple copies within a query. */ - private final Map<String, PatternTreeMap<ModEntry, ModsSerializer>> fileModCache = + protected Map<TsFileID, PatternTreeMap<ModEntry, ModsSerializer>> fileModCache = new ConcurrentHashMap<>(); + protected AtomicLong cachedModEntriesSize = new AtomicLong(0); + protected long queryId; private boolean debug; @@ -71,8 +78,6 @@ public class QueryContext { // for tree model, it will be true private boolean ignoreAllNullRows = true; - private final Set<TsFileID> nonExistentModFiles = new CopyOnWriteArraySet<>(); - // referenced TVLists for the query protected final Set<TVList> tvListSet = new HashSet<>(); @@ -92,28 +97,58 @@ public class QueryContext { // if the mods file does not exist, do not add it to the cache protected boolean checkIfModificationExists(TsFileResource tsFileResource) { - if (nonExistentModFiles.contains(tsFileResource.getTsFileID())) { - return false; - } + // The exists state of ModificationFile is maintained in memory, and ModificationFile instance + // is set to the related TsFileResource instance after it is constructed. + return tsFileResource.anyModFileExists(); + } - if (!tsFileResource.anyModFileExists()) { - nonExistentModFiles.add(tsFileResource.getTsFileID()); - return false; + private PatternTreeMap<ModEntry, ModsSerializer> getAllModifications(TsFileResource resource) { + if (!(this instanceof FragmentInstanceContext)) { + return fileModCache.computeIfAbsent( + resource.getTsFileID(), k -> loadAllModifications(resource)); } - return true; + FragmentInstanceContext fragmentInstanceContext = (FragmentInstanceContext) this; + if (fragmentInstanceContext.getSourcePaths().size() == 1) { + return loadAllModifications(resource); + } + + AtomicReference<PatternTreeMap<ModEntry, ModsSerializer>> atomicReference = + new AtomicReference<>(); + PatternTreeMap<ModEntry, ModsSerializer> cachedResult = + fileModCache.computeIfAbsent( + resource.getTsFileID(), + k -> { + PatternTreeMap<ModEntry, ModsSerializer> allMods = loadAllModifications(resource); + atomicReference.set(allMods); + if (cachedModEntriesSize.get() >= config.getModsCacheSizeLimitPerFI()) { + return null; + } + long memCost = RamUsageEstimator.sizeOfObject(allMods); + long alreadyUsedMemoryForCachedModEntries = cachedModEntriesSize.get(); + while (alreadyUsedMemoryForCachedModEntries + memCost + < config.getModsCacheSizeLimitPerFI()) { + if (cachedModEntriesSize.compareAndSet( + alreadyUsedMemoryForCachedModEntries, + alreadyUsedMemoryForCachedModEntries + memCost)) { + fragmentInstanceContext + .getMemoryReservationContext() + .reserveMemoryCumulatively(memCost); + return allMods; + } + alreadyUsedMemoryForCachedModEntries = cachedModEntriesSize.get(); + } + return null; + }); + return cachedResult == null ? atomicReference.get() : cachedResult; } - private PatternTreeMap<ModEntry, ModsSerializer> getAllModifications(TsFileResource resource) { - return fileModCache.computeIfAbsent( - resource.getTsFilePath(), - k -> { - PatternTreeMap<ModEntry, ModsSerializer> modifications = - PatternTreeMapFactory.getModsPatternTreeMap(); - for (ModEntry modification : resource.getAllModEntries()) { - modifications.append(modification.keyOfPatternTree(), modification); - } - return modifications; - }); + private PatternTreeMap<ModEntry, ModsSerializer> loadAllModifications(TsFileResource resource) { + PatternTreeMap<ModEntry, ModsSerializer> modifications = + PatternTreeMapFactory.getModsPatternTreeMap(); + for (ModEntry modification : resource.getAllModEntries()) { + modifications.append(modification.keyOfPatternTree(), modification); + } + return modifications; } public List<ModEntry> getPathModifications( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index d5bbe88501d..5f0272e8da9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@ -18,11 +18,14 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -35,8 +38,10 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -public class DeletionPredicate implements StreamSerializable, BufferSerializable { +public class DeletionPredicate implements StreamSerializable, BufferSerializable, Accountable { + public static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(DeletionPredicate.class); private String tableName; private IDPredicate idPredicate = new NOP(); // an empty list means affecting all columns @@ -180,4 +185,12 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable + measurementNames + '}'; } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + RamUsageEstimator.sizeOf(tableName) + + RamUsageEstimator.sizeOfObject(idPredicate) + + MemoryEstimationHelper.getEstimatedSizeOfStringArrayList(measurementNames); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java index e2de0d8bd91..9c6a464350f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java @@ -18,12 +18,15 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Deserializer; +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -36,7 +39,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -public abstract class IDPredicate implements StreamSerializable, BufferSerializable { +public abstract class IDPredicate implements StreamSerializable, BufferSerializable, Accountable { public int serializedSize() { // type @@ -124,6 +127,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa } public static class NOP extends IDPredicate { + public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NOP.class); public NOP() { super(IDPredicateType.NOP); @@ -158,10 +162,17 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public String toString() { return "NOP"; } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE; + } } public static class FullExactMatch extends IDPredicate { + public static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(FullExactMatch.class); private IDeviceID deviceID; public FullExactMatch(IDeviceID deviceID) { @@ -228,10 +239,17 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public String toString() { return "FullExactMatch{" + "deviceID=" + deviceID + '}'; } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + RamUsageEstimator.sizeOfObject(deviceID); + } } public static class SegmentExactMatch extends IDPredicate { + public static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SegmentExactMatch.class); private String pattern; private int segmentIndex; @@ -318,10 +336,16 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa + segmentIndex + '}'; } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + RamUsageEstimator.sizeOf(pattern); + } } public static class And extends IDPredicate { + public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(And.class); private final List<IDPredicate> predicates = new ArrayList<>(); public And(IDPredicate... predicates) { @@ -405,5 +429,11 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public String toString() { return "And{" + "predicates=" + predicates + '}'; } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableArrayList(predicates); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java index 36b943291b8..349f1b43ec2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.utils.io.StreamSerializable; import org.apache.tsfile.annotations.TreeModel; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.IOException; @@ -33,7 +34,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; public abstract class ModEntry - implements StreamSerializable, BufferSerializable, Comparable<ModEntry> { + implements StreamSerializable, BufferSerializable, Comparable<ModEntry>, Accountable { protected ModType modType; protected TimeRange timeRange; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java index 9dfbe6102a4..858b6645b2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java @@ -20,10 +20,12 @@ package org.apache.iotdb.db.storageengine.dataregion.modification; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.utils.RamUsageEstimator; import java.io.IOException; import java.io.InputStream; @@ -32,6 +34,8 @@ import java.nio.ByteBuffer; import java.util.Objects; public class TableDeletionEntry extends ModEntry { + public static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TableDeletionEntry.class); private DeletionPredicate predicate; public TableDeletionEntry() { @@ -152,4 +156,11 @@ public class TableDeletionEntry extends ModEntry { public DeletionPredicate getPredicate() { return predicate; } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + MemoryEstimationHelper.TIME_RANGE_INSTANCE_SIZE + + RamUsageEstimator.sizeOfObject(predicate); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java index 02fb95680f5..3a755f0744f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java @@ -24,12 +24,14 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; @@ -44,6 +46,8 @@ import java.util.Objects; public class TreeDeletionEntry extends ModEntry { + public static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TreeDeletionEntry.class); private static final Logger LOGGER = LoggerFactory.getLogger(TreeDeletionEntry.class); private MeasurementPath pathPattern; @@ -226,4 +230,11 @@ public class TreeDeletionEntry extends ModEntry { public int hashCode() { return Objects.hash(pathPattern, timeRange); } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + MemoryEstimationHelper.TIME_RANGE_INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfPartialPath(pathPattern); + } } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 7eb20db14f3..1abf605c9d4 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1106,6 +1106,13 @@ batch_size=100000 # Datatype: long sort_buffer_size_in_bytes=0 +# The maximum mod entries size that each FragmentInstance can cache. +# if mods_cache_size_limit_per_fi_in_bytes <= 0, default value will be used, default value = min(32MB, memory for query operators / query_thread_count / 2) +# if mods_cache_size_limit_per_fi_in_bytes > 0, the specified value will be used. +# effectiveMode: hot_reload +# Datatype: long +mods_cache_size_limit_per_fi_in_bytes=0 + # The threshold of operator count in the result set of EXPLAIN ANALYZE, if the number of operator in the result set is larger than this threshold, operator will be merged. # effectiveMode: hot_reload # Datatype: int diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java index b5da17d92bd..3ff41f251a1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java @@ -19,7 +19,9 @@ package org.apache.iotdb.commons.path; +import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -40,8 +42,10 @@ import java.util.function.Supplier; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; -public class PathPatternNode<V, S extends PathPatternNode.Serializer<V>> { +public class PathPatternNode<V, S extends PathPatternNode.Serializer<V>> implements Accountable { + private static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(PathPatternNode.class); private final String name; private final Map<String, PathPatternNode<V, S>> children; private Set<V> valueSet; @@ -273,6 +277,18 @@ public class PathPatternNode<V, S extends PathPatternNode.Serializer<V>> { return node; } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + RamUsageEstimator.sizeOf(name) + + RamUsageEstimator.sizeOfCollection(valueSet) + + RamUsageEstimator.sizeOfCollection(childrenNamesWithNonTrivialWildcard) + + RamUsageEstimator.sizeOfMapWithKnownShallowSize( + children, + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP, + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY); + } + /** * Interface to support serialize and deserialize valueSet. * diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java index 99216d10f45..8e3bbdf5f9a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java @@ -21,6 +21,8 @@ package org.apache.iotdb.commons.path; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; import javax.annotation.concurrent.NotThreadSafe; @@ -34,7 +36,9 @@ import java.util.function.BiConsumer; import java.util.function.Supplier; @NotThreadSafe -public class PatternTreeMap<V, VSerializer extends PathPatternNode.Serializer<V>> { +public class PatternTreeMap<V, VSerializer extends PathPatternNode.Serializer<V>> + implements Accountable { + private final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(PatternTreeMap.class); private final Map<String, PathPatternNode<V, VSerializer>> rootMap; private final Supplier<? extends Set<V>> supplier; private final BiConsumer<V, Set<V>> appendFunction; @@ -269,4 +273,13 @@ public class PatternTreeMap<V, VSerializer extends PathPatternNode.Serializer<V> searchDeviceOverlapped(child, deviceNodes, pos + 1, resultSet); } } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + RamUsageEstimator.sizeOfMapWithKnownShallowSize( + rootMap, + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP, + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY); + } }
