This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch jira_1398 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9b73a13ac26449fdd2de28d5c9fd65cbde8365fa Author: jt <[email protected]> AuthorDate: Thu May 27 11:33:44 2021 +0800 do not select unseq files when there are unmerged old unseq files --- .../merge/selector/MaxFileMergeFileSelector.java | 9 ++- .../engine/merge/MaxFileMergeFileSelectorTest.java | 73 ++++++++++++++++++++++ .../apache/iotdb/db/engine/merge/MergeTest.java | 9 +-- 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java index f8ecdc0..3deb763 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java @@ -168,7 +168,10 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector { useTightBound ? calculateTightMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit) : calculateLooseMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit); - updateSelectedFiles(newCost, unseqFile); + if (!updateSelectedFiles(newCost, unseqFile)) { + // older unseq files must be merged before newer ones + break; + } tmpSelectedSeqFiles.clear(); unseqIndex++; @@ -181,7 +184,7 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector { } } - private void updateSelectedFiles(long newCost, TsFileResource unseqFile) { + private boolean updateSelectedFiles(long newCost, TsFileResource unseqFile) { if (totalCost + newCost < memoryBudget) { selectedUnseqFiles.add(unseqFile); maxSeqFileCost = tempMaxSeqFileCost; @@ -198,7 +201,9 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector { tmpSelectedSeqFiles, newCost, totalCost); + return true; } + return false; } private boolean checkClosed(TsFileResource unseqFile) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java index 14c72d4..789d58d 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java @@ -236,4 +236,77 @@ public class MaxFileMergeFileSelectorTest extends MergeTest { assertEquals(2, result[0].size()); resource.clear(); } + + @Test + public void testSelectContinuousUnseqFile() + throws IOException, WriteProcessException, MergeException { + List<TsFileResource> seqList = new ArrayList<>(); + List<TsFileResource> unseqList = new ArrayList<>(); + try { + // seq files [0,0] [1,1] [2,2] ... [99,99] + int seqFileNum = 99; + for (int i = 0; i < seqFileNum; i++) { + File file = + new File( + TestConstant.BASE_OUTPUT_PATH.concat( + 10 + + "seq" + + IoTDBConstant.FILE_NAME_SEPARATOR + + i + + IoTDBConstant.FILE_NAME_SEPARATOR + + 10 + + IoTDBConstant.FILE_NAME_SEPARATOR + + 0 + + ".tsfile")); + TsFileResource fileResource = new TsFileResource(file); + fileResource.setClosed(true); + prepareFile(fileResource, i, 1, 0); + seqList.add(fileResource); + } + int unseqFileNum = 3; + // 3 unseq files [0,0] [0,99] [99,99] + for (int i = 0; i < unseqFileNum; i++) { + File file = + new File( + TestConstant.BASE_OUTPUT_PATH.concat( + 10 + + "unseq" + + IoTDBConstant.FILE_NAME_SEPARATOR + + i + + IoTDBConstant.FILE_NAME_SEPARATOR + + 10 + + IoTDBConstant.FILE_NAME_SEPARATOR + + 0 + + ".tsfile")); + TsFileResource fileResource = new TsFileResource(file); + fileResource.setClosed(true); + unseqList.add(fileResource); + } + prepareFile(unseqList.get(0), 0, 1, 10); + prepareFile(unseqList.get(1), 0, 100, 20); + prepareFile(unseqList.get(2), 99, 1, 30); + + MergeResource resource = new MergeResource(seqList, unseqList); + // the budget is enough to select unseq0 and unseq2, but not unseq1 + // the first selection should only contain seq0 and unseq0 + IMergeFileSelector mergeFileSelector = new MaxFileMergeFileSelector(resource, 29000); + List[] result = mergeFileSelector.select(); + assertEquals(1, result[0].size()); + assertEquals(1, result[1].size()); + assertEquals(seqList.get(0), result[0].get(0)); + assertEquals(unseqList.get(0), result[1].get(0)); + resource.clear(); + + resource = + new MergeResource( + seqList.subList(1, seqList.size()), unseqList.subList(1, unseqList.size())); + // the second selection should be empty + mergeFileSelector = new MaxFileMergeFileSelector(resource, 29000); + result = mergeFileSelector.select(); + assertEquals(0, result.length); + resource.clear(); + } finally { + removeFiles(seqList, unseqList); + } + } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java index d6e6735..fa705ca 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java @@ -86,7 +86,7 @@ abstract class MergeTest { @After public void tearDown() throws IOException, StorageEngineException { - removeFiles(); + removeFiles(seqResources, unseqResources); seqResources.clear(); unseqResources.clear(); IoTDBDescriptor.getInstance() @@ -186,12 +186,13 @@ abstract class MergeTest { prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000); } - private void removeFiles() throws IOException { - for (TsFileResource tsFileResource : seqResources) { + void removeFiles(List<TsFileResource> seqResList, List<TsFileResource> unseqResList) + throws IOException { + for (TsFileResource tsFileResource : seqResList) { tsFileResource.remove(); tsFileResource.getModFile().remove(); } - for (TsFileResource tsFileResource : unseqResources) { + for (TsFileResource tsFileResource : unseqResList) { tsFileResource.remove(); tsFileResource.getModFile().remove(); }
