This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_new_merge_strategy
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/dev_new_merge_strategy by this
push:
new 58ff4e3 fix imports and tests
58ff4e3 is described below
commit 58ff4e3ace9d6ef24282c38101a1bf110e236afd
Author: jt2594838 <[email protected]>
AuthorDate: Thu Nov 21 21:33:32 2019 +0800
fix imports and tests
---
.../org/apache/iotdb/db/engine/merge/BaseFileSelector.java | 7 +++++++
.../iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java | 6 +++---
.../apache/iotdb/db/sync/sender/manage/SyncFileManager.java | 6 ++++--
.../org/apache/iotdb/db/engine/merge/MergeOverLapTest.java | 1 -
.../org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java | 10 +++++-----
.../org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java | 6 +++---
.../apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java | 6 +++---
7 files changed, 25 insertions(+), 17 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
index 7f38dd6..38ef280 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.MergeException;
import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.db.utils.UpgradeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -109,6 +110,9 @@ public abstract class BaseFileSelector implements
IMergeFileSelector{
while (unseqIndex < resource.getUnseqFiles().size() && timeConsumption <
timeLimit) {
// select next unseq files
TsFileResource unseqFile = resource.getUnseqFiles().get(unseqIndex);
+ if (UpgradeUtils.isNeedUpgrade(unseqFile)) {
+ continue;
+ }
selectOverlappedSeqFiles(unseqFile);
@@ -179,6 +183,9 @@ public abstract class BaseFileSelector implements
IMergeFileSelector{
for (Integer seqFileIdx : tmpSelectedSeqIterable) {
TsFileResource seqFile = resource.getSeqFiles().get(seqFileIdx);
+ if (UpgradeUtils.isNeedUpgrade(seqFile)) {
+ return Long.MAX_VALUE;
+ }
fileReadCost = seqMeasurement.measure(seqFile);
if (fileReadCost > tempMaxSeqFileCost) {
// memory used when read data from a seq file:
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java
index 0bcde8c..96d5b68 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java
@@ -12,7 +12,7 @@ import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.MergeCallback;
import org.apache.iotdb.db.engine.merge.squeeze.recover.SqueezeMergeLogger;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.MetadataErrorException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.utils.MergeUtils;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory;
public class SqueezeMergeTask implements Callable<Void> {
- static final String MERGE_SUFFIX = ".merge.squeeze";
+ public static final String MERGE_SUFFIX = ".merge.squeeze";
private static final Logger logger =
LoggerFactory.getLogger(SqueezeMergeTask.class);
MergeResource resource;
@@ -65,7 +65,7 @@ public class SqueezeMergeTask implements Callable<Void> {
return null;
}
- private void doMerge() throws IOException, MetadataErrorException {
+ private void doMerge() throws IOException, MetadataException {
if (logger.isInfoEnabled()) {
logger.info("{} starts to merge {} seqFiles, {} unseqFiles", taskName,
resource.getSeqFiles().size(), resource.getUnseqFiles().size());
diff --git
a/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java
b/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java
index bb2532b..98ee75d 100644
---
a/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java
@@ -32,7 +32,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.engine.merge.task.MergeTask;
+import org.apache.iotdb.db.engine.merge.inplace.task.InplaceMergeTask;
+import org.apache.iotdb.db.engine.merge.squeeze.task.SqueezeMergeTask;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.sync.conf.SyncSenderDescriptor;
@@ -116,7 +117,8 @@ public class SyncFileManager implements ISyncFileManager {
}
if (new File(file.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX).exists() && !new File(
file.getAbsolutePath() + ModificationFile.FILE_SUFFIX).exists() &&
!new File(
- file.getAbsolutePath() + MergeTask.MERGE_SUFFIX).exists()) {
+ file.getAbsolutePath() + InplaceMergeTask.MERGE_SUFFIX).exists()
&& !new File(
+ file.getAbsolutePath() + SqueezeMergeTask.MERGE_SUFFIX).exists()) {
currentSealedLocalFilesMap.get(sgName).add(file);
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
index 0e83e6a..1857598 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
@@ -170,7 +170,6 @@ public class MergeOverLapTest extends MergeTest {
}
}
assertEquals(expected, cnt);
- assertEquals(1000, cnt);
} finally {
tsFilesReader.close();
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java
index 2fc1be7..073221c 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java
@@ -27,8 +27,8 @@ import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import
org.apache.iotdb.db.engine.merge.inplace.selector.InplaceMaxFileSelector;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
-import org.apache.iotdb.db.engine.merge.selector.MaxFileMergeFileSelector;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.MergeException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
@@ -75,10 +75,10 @@ public class MergeUpgradeTest {
@Test
public void testMergeUpgradeSelect() throws MergeException {
MergeResource resource = new MergeResource(seqResources, unseqResources);
- MaxFileMergeFileSelector mergeFileSelector = new
MaxFileMergeFileSelector(resource,
- Long.MAX_VALUE);
- List[] result = mergeFileSelector.select();
- assertEquals(0, result.length);
+ IMergeFileSelector mergeFileSelector = new
MaxSeriesMergeFileSelector<>(new InplaceMaxFileSelector(resource,
+ Long.MAX_VALUE));
+ mergeFileSelector.select();
+ assertEquals(0, mergeFileSelector.getSelectedSeqFiles().size());
}
private void prepareFiles() throws IOException, WriteProcessException {
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java
index 45b740e..a988b05 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java
@@ -32,9 +32,9 @@ import org.apache.iotdb.db.engine.merge.MergeTest;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.squeeze.task.SqueezeMergeTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.MetadataErrorException;
-import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.exception.path.PathException;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.junit.After;
import org.junit.Before;
@@ -45,7 +45,7 @@ public class MergeLogTest extends MergeTest {
private File tempSGDir;
@Before
- public void setUp() throws IOException, WriteProcessException,
MetadataErrorException, PathErrorException {
+ public void setUp() throws IOException, WriteProcessException,
MetadataException, PathException {
super.setUp();
tempSGDir = new File("tempSG");
tempSGDir.mkdirs();
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java
index 9e63e17..becb3f8 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java
@@ -31,9 +31,9 @@ import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.squeeze.task.SqueezeMergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.MetadataErrorException;
-import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.exception.path.PathException;
import org.apache.iotdb.db.query.context.QueryContext;
import
org.apache.iotdb.db.query.reader.resourceRelated.SeqResourceIterateReader;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
@@ -48,7 +48,7 @@ public class MergeTaskTest extends MergeTest {
private File tempSGDir;
@Before
- public void setUp() throws IOException, WriteProcessException,
MetadataErrorException, PathErrorException {
+ public void setUp() throws IOException, WriteProcessException,
MetadataException, PathException {
super.setUp();
tempSGDir = new File("tempSG");
tempSGDir.mkdirs();
