This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new ad8e340d058 Optimize ttl deletion in fast compaction performer
(#16367) (#16383)
ad8e340d058 is described below
commit ad8e340d05830c1d5a28537b45021b090adbd2ab
Author: shuwenwei <[email protected]>
AuthorDate: Tue Sep 16 14:33:04 2025 +0800
Optimize ttl deletion in fast compaction performer (#16367) (#16383)
* optimize ttl deletion in fast compaction performer
* fix ut
* check isEmpty
---
.../performer/impl/FastCompactionPerformer.java | 11 +++--------
.../compaction/execute/utils/CompactionUtils.java | 20 ++++++++++++++++----
.../execute/utils/MultiTsFileDeviceIterator.java | 11 ++---------
.../executor/fast/SeriesCompactionExecutor.java | 6 +++++-
.../utils/writer/AbstractCompactionWriter.java | 11 +++++++++++
.../estimator/AbstractCompactionEstimator.java | 4 ++++
.../RepairUnsortedFileCompactionEstimator.java | 5 +++++
.../impl/NewSizeTieredCompactionSelector.java | 2 +-
.../compaction/selector/impl/SettleSelectorImpl.java | 2 +-
.../apache/iotdb/commons/path/PatternTreeMap.java | 4 ++++
10 files changed, 52 insertions(+), 24 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
index fb99acd99cd..ca1f4459df7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
@@ -142,21 +142,16 @@ public class FastCompactionPerformer
// checked above
//noinspection OptionalGetWithoutIsPresent
sortedSourceFiles.sort(Comparator.comparingLong(x ->
x.getStartTime(device).get()));
+ Deletion ttlDeletion = null;
if (ttl != Long.MAX_VALUE) {
- Deletion ttlDeletion =
+ ttlDeletion =
new Deletion(
new PartialPath(device,
IoTDBConstant.ONE_LEVEL_PATH_WILDCARD),
Long.MAX_VALUE,
Long.MIN_VALUE,
deviceIterator.getTimeLowerBoundForCurrentDevice());
- for (TsFileResource sourceFile : sortedSourceFiles) {
- modificationCache
- .computeIfAbsent(
- sourceFile.getTsFile().getName(),
- k -> PatternTreeMapFactory.getModsPatternTreeMap())
- .append(ttlDeletion.getPath(), ttlDeletion);
- }
}
+ compactionWriter.setTTLDeletion(ttlDeletion);
if (sortedSourceFiles.isEmpty()) {
// device is out of dated in all source files
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index 1ace2bde3a0..d1f0dfc7ede 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.service.metrics.CompactionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -48,6 +49,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
@@ -376,13 +378,23 @@ public class CompactionUtils {
public static List<Modification> getMatchedModifications(
PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer>
patternTreeMap,
IDeviceID deviceID,
- String measurement)
+ String measurement,
+ Deletion ttlDeletion)
throws IllegalPathException {
- if (patternTreeMap == null) {
- return Collections.emptyList();
+ if ((patternTreeMap == null) || patternTreeMap.isEmpty()) {
+ return ttlDeletion == null ? Collections.emptyList() :
Collections.singletonList(ttlDeletion);
}
PartialPath path = CompactionPathUtils.getPath(deviceID, measurement);
- return ModificationFile.sortAndMerge(patternTreeMap.getOverlapped(path));
+ List<Modification> modifications = patternTreeMap.getOverlapped(path);
+ if (ttlDeletion != null) {
+ if (!(modifications instanceof ArrayList)) {
+ List<Modification> newModEntries = new
ArrayList<>(modifications.size() + 1);
+ newModEntries.addAll(modifications);
+ modifications = newModEntries;
+ }
+ modifications.add(ttlDeletion);
+ }
+ return ModificationFile.sortAndMerge(modifications);
}
public static boolean isDiskHasSpace() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 90a507ced53..de8e9575a7c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -451,10 +451,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
}
List<Modification> modificationList =
CompactionUtils.getMatchedModifications(
- modifications, device, valueChunkMetadata.getMeasurementUid());
- if (ttlDeletion != null) {
- modificationList.add(ttlDeletion);
- }
+ modifications, device, valueChunkMetadata.getMeasurementUid(),
ttlDeletion);
modificationForCurDevice.add(
modificationList.isEmpty() ? Collections.emptyList() :
modificationList);
}
@@ -666,11 +663,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
// collect the modifications for current series
List<Modification> modificationForCurrentSeries =
CompactionUtils.getMatchedModifications(
- modificationsInThisResource, device,
currentCompactingSeries);
- // add ttl deletion for current series
- if (ttlDeletion != null) {
- modificationForCurrentSeries.add(ttlDeletion);
- }
+ modificationsInThisResource, device,
currentCompactingSeries, ttlDeletion);
// if there are modifications of current series, apply them to the
chunk metadata
if (!modificationForCurrentSeries.isEmpty()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
index 9bf2e43d4cb..778c2e7f844 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
@@ -478,7 +478,11 @@ public abstract class SeriesCompactionExecutor {
throws IllegalPathException {
PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer>
allModifications =
modificationCacheMap.get(tsFileResource.getTsFile().getName());
- return CompactionUtils.getMatchedModifications(allModifications, deviceId,
measurement);
+ return CompactionUtils.getMatchedModifications(
+ allModifications,
+ deviceId,
+ measurement,
+ compactionWriter.getTTLLowerBoundForCurrentDevice());
}
@SuppressWarnings("squid:S3776")
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
index bf5f816b995..96d4369dcec 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
@@ -26,6 +26,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.exe
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.ChunkMetadataElement;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.flushcontroller.AbstractCompactionFlushController;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.PageHeader;
@@ -99,10 +100,20 @@ public abstract class AbstractCompactionWriter implements
AutoCloseable {
protected String[] measurementId = new String[subTaskNum];
+ protected Deletion ttlDeletionForCurrentDevice;
+
public abstract void startChunkGroup(IDeviceID deviceId, boolean isAlign)
throws IOException;
public abstract void endChunkGroup() throws IOException;
+ public void setTTLDeletion(Deletion ttlDeletion) {
+ this.ttlDeletionForCurrentDevice = ttlDeletion;
+ }
+
+ public Deletion getTTLLowerBoundForCurrentDevice() {
+ return ttlDeletionForCurrentDevice;
+ }
+
public void startMeasurement(String measurement, IChunkWriter chunkWriter,
int subTaskId) {
lastCheckIndex = 0;
lastTimeSet[subTaskId] = false;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
index ba6bfa9722b..36932313260 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
@@ -246,6 +246,10 @@ public abstract class AbstractCompactionEstimator {
return roughFileInfo;
}
+ public boolean supportsRoughEstimation() {
+ return true;
+ }
+
public static void removeFileInfoFromGlobalFileInfoCache(TsFileResource
resource) {
if (resource == null || resource.getTsFile() == null) {
return;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
index eca5bb3dcb0..9316ad3c106 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
@@ -89,4 +89,9 @@ public class RepairUnsortedFileCompactionEstimator extends
AbstractInnerSpaceEst
CompactionScheduleContext context, List<TsFileResource> resources)
throws IOException {
throw new RuntimeException("unimplemented");
}
+
+ @Override
+ public boolean supportsRoughEstimation() {
+ return false;
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
index ee24ea60748..8a26fa4e372 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
@@ -201,7 +201,7 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
performer =
sequence ? context.getSeqCompactionPerformer() :
context.getUnseqCompactionPerformer();
estimator = performer.getInnerSpaceEstimator().orElse(null);
- if (estimator == null) {
+ if (estimator == null || !estimator.supportsRoughEstimation()) {
estimateCompactionTaskMemoryDuringSelection = false;
}
memoryCost = 0;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java
index de81d4f4f7a..8bed5939639 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java
@@ -288,7 +288,7 @@ public class SettleSelectorImpl implements ISettleSelector {
throws IllegalPathException {
List<Modification> deviceModifications =
CompactionUtils.getMatchedModifications(
- modifications, device, AlignedPath.VECTOR_PLACEHOLDER);
+ modifications, device, AlignedPath.VECTOR_PLACEHOLDER, null);
for (Modification modification : deviceModifications) {
if (((Deletion) modification).getTimeRange().contains(startTime,
endTime)) {
return true;
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java
index 4726283b93c..38bddd37ff0 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PatternTreeMap.java
@@ -61,6 +61,10 @@ public class PatternTreeMap<V, VSerializer extends
PathPatternNode.Serializer<V>
this.serializer = serializer;
}
+ public boolean isEmpty() {
+ return root.isLeaf() && root.getValues().isEmpty();
+ }
+
/**
* Append key and value to PatternTreeMap.
*
