This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new cc1a4bf7ff7 Compaction create target file with data dir selection
strategy (#12063)
cc1a4bf7ff7 is described below
commit cc1a4bf7ff7de409a3770f4103eb12c780badd71
Author: shuwenwei <[email protected]>
AuthorDate: Fri Apr 19 17:29:03 2024 +0800
Compaction create target file with data dir selection strategy (#12063)
* compaction create target file with data dir selection strategy
* compaction create target file with data dir selection strategy
---
.../execute/task/InnerSpaceCompactionTask.java | 3 +-
.../tsfile/generator/TsFileNameGenerator.java | 99 +++++++++++++++-------
.../InnerSeqCompactionWithFastPerformerTest.java | 9 +-
...nerSeqCompactionWithReadChunkPerformerTest.java | 9 +-
.../utils/CompactionFileGeneratorUtils.java | 10 ++-
5 files changed, 90 insertions(+), 40 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 01399c8d5fe..d2203373d22 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException;
@@ -148,7 +149,7 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
createSummary();
}
- protected void prepare() throws IOException {
+ protected void prepare() throws IOException, DiskSpaceInsufficientException {
targetTsFileResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(
selectedTsFileResourceList, sequence);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
index d9a2ce07c7a..040f93927b9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
@@ -65,13 +65,36 @@ public class TsFileNameGenerator {
int innerSpaceCompactionCount,
int crossSpaceCompactionCount)
throws DiskSpaceInsufficientException {
+ return generateNewTsFilePathWithMkdir(
+ sequence,
+ logicalStorageGroup,
+ virtualStorageGroup,
+ timePartitionId,
+ time,
+ version,
+ innerSpaceCompactionCount,
+ crossSpaceCompactionCount,
+ TsFileConstant.TSFILE_SUFFIX);
+ }
+
+ public static String generateNewTsFilePathWithMkdir(
+ boolean sequence,
+ String logicalStorageGroup,
+ String virtualStorageGroup,
+ long timePartitionId,
+ long time,
+ long version,
+ int innerSpaceCompactionCount,
+ int crossSpaceCompactionCount,
+ String customSuffix)
+ throws DiskSpaceInsufficientException {
String tsFileDir =
generateTsFileDir(sequence, logicalStorageGroup, virtualStorageGroup,
timePartitionId);
fsFactory.getFile(tsFileDir).mkdirs();
return tsFileDir
+ File.separator
+ generateNewTsFileName(
- time, version, innerSpaceCompactionCount,
crossSpaceCompactionCount);
+ time, version, innerSpaceCompactionCount,
crossSpaceCompactionCount, customSuffix);
}
public static String generateTsFileDir(
@@ -93,6 +116,16 @@ public class TsFileNameGenerator {
public static String generateNewTsFileName(
long time, long version, int innerSpaceCompactionCount, int
crossSpaceCompactionCount) {
+ return generateNewTsFileName(
+ time, version, innerSpaceCompactionCount, crossSpaceCompactionCount,
TSFILE_SUFFIX);
+ }
+
+ public static String generateNewTsFileName(
+ long time,
+ long version,
+ int innerSpaceCompactionCount,
+ int crossSpaceCompactionCount,
+ String customSuffix) {
return time
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ version
@@ -100,7 +133,7 @@ public class TsFileNameGenerator {
+ innerSpaceCompactionCount
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ crossSpaceCompactionCount
- + TsFileConstant.TSFILE_SUFFIX;
+ + customSuffix;
}
public static TsFileName getTsFileName(String fileName) throws IOException {
@@ -189,7 +222,7 @@ public class TsFileNameGenerator {
* @throws IOException
*/
public static List<TsFileResource> getCrossCompactionTargetFileResources(
- List<TsFileResource> seqResources) throws IOException {
+ List<TsFileResource> seqResources) throws IOException,
DiskSpaceInsufficientException {
List<TsFileResource> targetFileResources = new ArrayList<>();
for (TsFileResource resource : seqResources) {
TsFileName tsFileName = getTsFileName(resource.getTsFile().getName());
@@ -198,15 +231,16 @@ public class TsFileNameGenerator {
targetFileResources.add(
new TsFileResource(
new File(
- resource.getTsFile().getParent(),
- tsFileName.time
- + FILE_NAME_SEPARATOR
- + tsFileName.version
- + FILE_NAME_SEPARATOR
- + tsFileName.innerCompactionCnt
- + FILE_NAME_SEPARATOR
- + tsFileName.crossCompactionCnt
- + IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX),
+ generateNewTsFilePathWithMkdir(
+ resource.isSeq(),
+ resource.getDatabaseName(),
+ resource.getDataRegionId(),
+ resource.getTimePartition(),
+ tsFileName.time,
+ tsFileName.version,
+ tsFileName.innerCompactionCnt,
+ tsFileName.crossCompactionCnt,
+ IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)),
TsFileResourceStatus.COMPACTING));
}
return targetFileResources;
@@ -222,7 +256,8 @@ public class TsFileNameGenerator {
* @throws IOException
*/
public static TsFileResource getInnerCompactionTargetFileResource(
- List<TsFileResource> tsFileResources, boolean sequence) throws
IOException {
+ List<TsFileResource> tsFileResources, boolean sequence)
+ throws IOException, DiskSpaceInsufficientException {
long minTime = Long.MAX_VALUE;
long maxTime = Long.MIN_VALUE;
long minVersion = Long.MAX_VALUE;
@@ -243,27 +278,29 @@ public class TsFileNameGenerator {
sequence
? new TsFileResource(
new File(
- tsFileResources.get(0).getTsFile().getParent(),
- minTime
- + FILE_NAME_SEPARATOR
- + minVersion
- + FILE_NAME_SEPARATOR
- + (maxInnerMergeCount + 1)
- + FILE_NAME_SEPARATOR
- + maxCrossMergeCount
- + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
+ generateNewTsFilePathWithMkdir(
+ sequence,
+ tsFileResources.get(0).getDatabaseName(),
+ tsFileResources.get(0).getDataRegionId(),
+ tsFileResources.get(0).getTimePartition(),
+ minTime,
+ minVersion,
+ (int) maxInnerMergeCount + 1,
+ (int) maxCrossMergeCount,
+ IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)),
TsFileResourceStatus.COMPACTING)
: new TsFileResource(
new File(
- tsFileResources.get(0).getTsFile().getParent(),
- maxTime
- + FILE_NAME_SEPARATOR
- + maxVersion
- + FILE_NAME_SEPARATOR
- + (maxInnerMergeCount + 1)
- + FILE_NAME_SEPARATOR
- + maxCrossMergeCount
- + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
+ generateNewTsFilePathWithMkdir(
+ sequence,
+ tsFileResources.get(0).getDatabaseName(),
+ tsFileResources.get(0).getDataRegionId(),
+ tsFileResources.get(0).getTimePartition(),
+ maxTime,
+ maxVersion,
+ (int) maxInnerMergeCount + 1,
+ (int) maxCrossMergeCount,
+ IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)),
TsFileResourceStatus.COMPACTING);
resource.setSeq(sequence);
return resource;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
index b8f7f7151bf..539da5cc4eb 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
@@ -171,7 +171,8 @@ public class InnerSeqCompactionWithFastPerformerTest {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
- CompactionFileGeneratorUtils.generateTsFileResource(true,
i + 1);
+ CompactionFileGeneratorUtils.generateTsFileResource(
+ true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
sourceResources.add(tsFileResource);
@@ -466,7 +467,8 @@ public class InnerSeqCompactionWithFastPerformerTest {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
- CompactionFileGeneratorUtils.generateTsFileResource(true, i
+ 1);
+ CompactionFileGeneratorUtils.generateTsFileResource(
+ true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
@@ -791,7 +793,8 @@ public class InnerSeqCompactionWithFastPerformerTest {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
- CompactionFileGeneratorUtils.generateTsFileResource(true,
i + 1);
+ CompactionFileGeneratorUtils.generateTsFileResource(
+ true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
index f9ce303ecb9..aab06143e7b 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
@@ -169,7 +169,8 @@ public class InnerSeqCompactionWithReadChunkPerformerTest {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
- CompactionFileGeneratorUtils.generateTsFileResource(true,
i + 1);
+ CompactionFileGeneratorUtils.generateTsFileResource(
+ true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
sourceResources.add(tsFileResource);
@@ -459,7 +460,8 @@ public class InnerSeqCompactionWithReadChunkPerformerTest {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
- CompactionFileGeneratorUtils.generateTsFileResource(true, i
+ 1);
+ CompactionFileGeneratorUtils.generateTsFileResource(
+ true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
@@ -780,7 +782,8 @@ public class InnerSeqCompactionWithReadChunkPerformerTest {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
- CompactionFileGeneratorUtils.generateTsFileResource(true,
i + 1);
+ CompactionFileGeneratorUtils.generateTsFileResource(
+ true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
index 9c1836077ee..4fcf102225f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.utils;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -69,14 +70,15 @@ public class CompactionFileGeneratorUtils {
}
public static List<TsFileResource> getInnerCompactionTargetTsFileResources(
- List<TsFileResource> fileResources, boolean seq) throws IOException {
+ List<TsFileResource> fileResources, boolean seq)
+ throws IOException, DiskSpaceInsufficientException {
List<TsFileResource> resources = new ArrayList<>();
resources.add(TsFileNameGenerator.getInnerCompactionTargetFileResource(fileResources,
seq));
return resources;
}
public static List<TsFileResource> getCrossCompactionTargetTsFileResources(
- List<TsFileResource> seqFileResources) throws IOException {
+ List<TsFileResource> seqFileResources) throws IOException,
DiskSpaceInsufficientException {
return
TsFileNameGenerator.getCrossCompactionTargetFileResources(seqFileResources);
}
@@ -90,6 +92,8 @@ public class CompactionFileGeneratorUtils {
return new TsFileResource(
new File(
TestConstant.BASE_OUTPUT_PATH
+ .concat(File.separator)
+ .concat("data")
.concat(File.separator)
.concat("sequence")
.concat(File.separator)
@@ -112,6 +116,8 @@ public class CompactionFileGeneratorUtils {
return new TsFileResource(
new File(
TestConstant.BASE_OUTPUT_PATH
+ .concat(File.separator)
+ .concat("data")
.concat(File.separator)
.concat("unsequence")
.concat(File.separator)
