This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c364449170 [IOTDB-5726]Select the last sealed seq file for nonOverlap
unseq files to compact in cross compaction (#9501)
c364449170 is described below
commit c364449170f77b0705897bf3ebe26b727e26485f
Author: 周沛辰 <[email protected]>
AuthorDate: Thu Apr 13 15:03:31 2023 +0800
[IOTDB-5726]Select the last sealed seq file for nonOverlap unseq files to
compact in cross compaction (#9501)
---
.../impl/RewriteCrossSpaceCompactionSelector.java | 30 +
.../utils/CrossSpaceCompactionCandidate.java | 15 +-
...eCompactionWithFastPerformerValidationTest.java | 705 ++++++++++++++++++++
...actionWithReadPointPerformerValidationTest.java | 713 ++++++++++++++++++++-
4 files changed, 1458 insertions(+), 5 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index a6ed828827..f96f52ac64 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -154,6 +154,17 @@ public class RewriteCrossSpaceCompactionSelector
implements ICrossSpaceSelector
TsFileResource unseqFile = split.unseqFile.resource;
List<TsFileResource> targetSeqFiles =
split.seqFiles.stream().map(c ->
c.resource).collect(Collectors.toList());
+
+ if (!split.hasOverlap) {
+ LOGGER.info("Unseq file {} does not overlap with any seq files.",
unseqFile);
+ TsFileResource latestSealedSeqFile =
+ getLatestSealedSeqFile(candidate.getSeqFileCandidates());
+ if (latestSealedSeqFile == null) {
+ break;
+ }
+ targetSeqFiles.add(latestSealedSeqFile);
+ }
+
long memoryCost =
compactionEstimator.estimateCrossCompactionMemory(targetSeqFiles,
unseqFile);
if (!canAddToTaskResource(taskResource, unseqFile, targetSeqFiles,
memoryCost)) {
@@ -171,6 +182,25 @@ public class RewriteCrossSpaceCompactionSelector
implements ICrossSpaceSelector
return taskResource;
}
+ private TsFileResource getLatestSealedSeqFile(
+ List<TsFileResourceCandidate> seqResourceCandidateList) {
+ for (int i = seqResourceCandidateList.size() - 1; i >= 0; i--) {
+ TsFileResourceCandidate seqResourceCandidate =
seqResourceCandidateList.get(i);
+ if (seqResourceCandidate.resource.isClosed()) {
+ // We must select the latest sealed and valid seq file to compact
with, in order to avoid
+ // overlapping of the new compacted files with the subsequent seq
files.
+ if (seqResourceCandidate.isValidCandidate) {
+ LOGGER.info(
+ "Select one valid seq file {} for unseq file to compact with.",
+ seqResourceCandidate.resource);
+ return seqResourceCandidate.resource;
+ }
+ break;
+ }
+ }
+ return null;
+ }
+
// TODO: (xingtanzjr) need to confirm whether we should strictly guarantee
the conditions
// If we guarantee the condition strictly, the smallest collection of cross
task resource may not
// satisfied
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
index 2b21345957..bedf2e2fbc 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/utils/CrossSpaceCompactionCandidate.java
@@ -73,6 +73,7 @@ public class CrossSpaceCompactionCandidate {
}
private boolean prepareNextSplit() throws IOException {
+ boolean nextUnseqFileHasOverlap = false;
TsFileResourceCandidate unseqFile = unseqFiles.get(nextUnseqFileIndex);
List<TsFileResourceCandidate> ret = new ArrayList<>();
@@ -104,6 +105,7 @@ public class CrossSpaceCompactionCandidate {
ret.add(seqFile);
seqFile.markAsSelected();
}
+ nextUnseqFileHasOverlap = true;
// if this condition is satisfied, all subsequent seq files is
unnecessary to check
break;
} else if (unseqDeviceInfo.startTime <= seqDeviceInfo.endTime) {
@@ -111,13 +113,14 @@ public class CrossSpaceCompactionCandidate {
ret.add(seqFile);
seqFile.markAsSelected();
}
+ nextUnseqFileHasOverlap = true;
}
}
}
// mark candidates in next split as selected even though it may not be
added to the final
// TaskResource
unseqFile.markAsSelected();
- nextSplit = new CrossCompactionTaskResourceSplit(unseqFile, ret);
+ nextSplit = new CrossCompactionTaskResourceSplit(unseqFile, ret,
nextUnseqFileHasOverlap);
nextUnseqFileIndex++;
return true;
}
@@ -154,6 +157,10 @@ public class CrossSpaceCompactionCandidate {
.collect(Collectors.toList());
}
+ public List<TsFileResourceCandidate> getSeqFileCandidates() {
+ return seqFiles;
+ }
+
public List<TsFileResourceCandidate> getUnseqFileCandidates() {
return unseqFiles;
}
@@ -167,11 +174,15 @@ public class CrossSpaceCompactionCandidate {
public static class CrossCompactionTaskResourceSplit {
public TsFileResourceCandidate unseqFile;
public List<TsFileResourceCandidate> seqFiles;
+ public boolean hasOverlap;
public CrossCompactionTaskResourceSplit(
- TsFileResourceCandidate unseqFile, List<TsFileResourceCandidate>
seqFiles) {
+ TsFileResourceCandidate unseqFile,
+ List<TsFileResourceCandidate> seqFiles,
+ boolean hasOverlap) {
this.unseqFile = unseqFile;
this.seqFiles = seqFiles;
+ this.hasOverlap = hasOverlap;
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index cc0a0d3f18..817fbf11f7 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -19,7 +19,11 @@
package org.apache.iotdb.db.engine.compaction.cross;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.path.AlignedPath;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest;
import
org.apache.iotdb.db.engine.compaction.execute.performer.ICrossCompactionPerformer;
@@ -31,6 +35,7 @@ import
org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.engine.compaction.selector.ICompactionSelector;
import org.apache.iotdb.db.engine.compaction.selector.ICrossSpaceSelector;
import
org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector;
+import
org.apache.iotdb.db.engine.compaction.selector.impl.SizeTieredCompactionSelector;
import
org.apache.iotdb.db.engine.compaction.selector.utils.CrossCompactionTaskResource;
import
org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate;
import
org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils;
@@ -42,7 +47,12 @@ import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
@@ -52,10 +62,13 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
+import static
org.apache.iotdb.tsfile.utils.TsFileGeneratorUtils.alignDeviceOffset;
public class CrossSpaceCompactionWithFastPerformerValidationTest extends
AbstractCompactionTest {
TsFileManager tsFileManager =
@@ -2289,4 +2302,696 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
List<List<TsFileResource>> innerPairs =
innerSelector.selectInnerSpaceTask(targetResources);
Assert.assertEquals(1, innerPairs.size());
}
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles1() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles2() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles3() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(4, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+ createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles4() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 9, 10, 500, 100, 100, 0, 100, false, false);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(2, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(3, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles1() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles2() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles3() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(4, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+ createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles4() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 9, 10, 500, 100, 100, 0, 100, true, false);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new FastCompactionPerformer(false),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(2, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(3, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new FastCompactionPerformer(true),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ public void generateModsFile(
+ List<PartialPath> seriesPaths, TsFileResource resource, long startValue,
long endValue)
+ throws IllegalPathException, IOException {
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (PartialPath path : seriesPaths) {
+ String fullPath =
+ (path instanceof AlignedPath)
+ ? path.getFullPath()
+ + TsFileConstant.PATH_SEPARATOR
+ + ((AlignedPath) path).getMeasurementList().get(0)
+ : path.getFullPath();
+ deleteMap.put(fullPath, new Pair<>(startValue, endValue));
+ }
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
index 3d7a0744cf..c76dfd162d 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
@@ -19,7 +19,11 @@
package org.apache.iotdb.db.engine.compaction.cross;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.path.AlignedPath;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest;
import
org.apache.iotdb.db.engine.compaction.execute.performer.ICrossCompactionPerformer;
@@ -27,13 +31,14 @@ import
org.apache.iotdb.db.engine.compaction.execute.performer.impl.ReadChunkCom
import
org.apache.iotdb.db.engine.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import
org.apache.iotdb.db.engine.compaction.execute.task.CompactionTaskSummary;
import
org.apache.iotdb.db.engine.compaction.execute.task.CrossSpaceCompactionTask;
+import
org.apache.iotdb.db.engine.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.engine.compaction.selector.ICrossSpaceSelector;
import
org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector;
+import
org.apache.iotdb.db.engine.compaction.selector.impl.SizeTieredCompactionSelector;
import
org.apache.iotdb.db.engine.compaction.selector.utils.CrossCompactionTaskResource;
import
org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate;
import
org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils;
-import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.exception.MergeException;
@@ -41,7 +46,12 @@ import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
@@ -51,13 +61,18 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
+import static
org.apache.iotdb.tsfile.utils.TsFileGeneratorUtils.alignDeviceOffset;
+
public class CrossSpaceCompactionWithReadPointPerformerValidationTest
extends AbstractCompactionTest {
- TsFileManager tsFileManager =
- new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath());
+ // TsFileManager tsFileManager =
+ // new TsFileManager(COMPACTION_TEST_SG, "0",
STORAGE_GROUP_DIR.getPath());
private final String oldThreadName = Thread.currentThread().getName();
@@ -2147,4 +2162,696 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
Assert.assertEquals(0, selected.size());
}
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles1() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles2() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles3() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(4, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+ createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles4() throws
Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 9, 10, 500, 100, 100, 0, 100, false, false);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + d + PATH_SEPARATOR +
"s" + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new MeasurementPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR
+ "s" + j,
+ TSDataType.INT64));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(2, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(3, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles1() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles2() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles3() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(4, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+ createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(1, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(2, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ @Test
+ public void testAlignedUnseqFilesNotOverlapWithSeqFiles4() throws Exception {
+
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 9, 10, 500, 100, 100, 0, 100, true, false);
+ createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
+ createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ // delete d0 ~ d5 in seq files
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (int d = 0; d < 5; d++) {
+ for (int m = 0; m < 5; m++) {
+ deleteMap.put(
+ COMPACTION_TEST_SG
+ + PATH_SEPARATOR
+ + "d"
+ + (alignDeviceOffset + d)
+ + PATH_SEPARATOR
+ + "s"
+ + m,
+ new Pair<>(Long.MIN_VALUE, Long.MAX_VALUE));
+ }
+ }
+ for (TsFileResource resource : seqResources) {
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
+
+ List<PartialPath> timeseriesPaths = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 10; j++) {
+ timeseriesPaths.add(
+ new AlignedPath(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i,
+ Collections.singletonList("s" + j),
+ Collections.singletonList(new MeasurementSchema("s" + j,
TSDataType.INT64))));
+ }
+ }
+ Map<PartialPath, List<TimeValuePair>> sourceData =
+ readSourceFiles(timeseriesPaths, Collections.emptyList());
+
+ // inner seq space compact
+ List<List<TsFileResource>> taskResources =
+ new SizeTieredCompactionSelector(COMPACTION_TEST_SG, "0", 0, true,
tsFileManager)
+
.selectInnerSpaceTask(tsFileManager.getOrCreateSequenceListByTimePartition(0));
+ for (List<TsFileResource> taskResource : taskResources) {
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ taskResource,
+ true,
+ new ReadChunkCompactionPerformer(),
+ new AtomicInteger(0),
+ 0L)
+ .start();
+ }
+
+ // select cross compaction
+ ICrossSpaceSelector crossSpaceCompactionSelector =
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionSelector()
+ .createInstance(COMPACTION_TEST_SG, "0", 0, tsFileManager);
+ CrossCompactionTaskResource sourceFiles =
+ crossSpaceCompactionSelector
+ .selectCrossSpaceTask(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0),
+ tsFileManager.getOrCreateUnsequenceListByTimePartition(0))
+ .get(0);
+ Assert.assertEquals(2, sourceFiles.getSeqFiles().size());
+ Assert.assertEquals(3, sourceFiles.getUnseqFiles().size());
+
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ sourceFiles.getSeqFiles(),
+ sourceFiles.getUnseqFiles(),
+ new ReadPointCompactionPerformer(),
+ new AtomicInteger(0),
+ sourceFiles.getTotalMemoryCost(),
+ 0)
+ .start();
+
+ validateSeqFiles(true);
+ validateTargetDatas(sourceData, Collections.emptyList());
+ }
+
+ public void generateModsFile(
+ List<PartialPath> seriesPaths, TsFileResource resource, long startValue,
long endValue)
+ throws IllegalPathException, IOException {
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ for (PartialPath path : seriesPaths) {
+ String fullPath =
+ (path instanceof AlignedPath)
+ ? path.getFullPath()
+ + TsFileConstant.PATH_SEPARATOR
+ + ((AlignedPath) path).getMeasurementList().get(0)
+ : path.getFullPath();
+ deleteMap.put(fullPath, new Pair<>(startValue, endValue));
+ }
+ CompactionFileGeneratorUtils.generateMods(deleteMap, resource, false);
+ }
}
