This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 25a8e98 [IOTDB-1742] Fix test bug caused by restart error of virtual
storage groups (#4083)
25a8e98 is described below
commit 25a8e984eb655429d1cf700788b67f8de2b73cb0
Author: Alan Choo <[email protected]>
AuthorDate: Sat Oct 9 10:03:43 2021 +0800
[IOTDB-1742] Fix test bug caused by restart error of virtual storage groups
(#4083)
---
.../org/apache/iotdb/db/engine/StorageEngine.java | 50 +++++++++++-----------
.../virtualSg/HashVirtualPartitioner.java | 6 ---
.../iotdb/db/integration/IoTDBCompleteIT.java | 9 ++--
.../db/integration/IoTDBLoadExternalTsfileIT.java | 12 +++---
.../db/sync/receiver/load/FileLoaderTest.java | 9 ++--
5 files changed, 41 insertions(+), 45 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 515a0b8..7fcaca9 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -116,7 +116,9 @@ public class StorageEngine implements IService {
* a folder (system/storage_groups/ by default) that persist system info.
Each Storage Processor
* will have a subfolder under the systemDir.
*/
- private final String systemDir;
+ private final String systemDir =
+ FilePathUtils.regularizePath(config.getSystemDir()) + "storage_groups";
+
/** storage group name -> storage group processor */
private final ConcurrentHashMap<PartialPath, VirtualStorageGroupManager>
processorMap =
new ConcurrentHashMap<>();
@@ -134,27 +136,7 @@ public class StorageEngine implements IService {
private List<CloseFileListener> customCloseFileListeners = new ArrayList<>();
private List<FlushListener> customFlushListeners = new ArrayList<>();
- private StorageEngine() {
- systemDir = FilePathUtils.regularizePath(config.getSystemDir()) +
"storage_groups";
-
- // build time Interval to divide time partition
- if (!enablePartition) {
- timePartitionInterval = Long.MAX_VALUE;
- } else {
- initTimePartition();
- }
-
- // create systemDir
- try {
- FileUtils.forceMkdir(SystemFileFactory.INSTANCE.getFile(systemDir));
- } catch (IOException e) {
- throw new StorageEngineFailureException(e);
- }
- // recover upgrade process
- UpgradeUtils.recoverUpgrade();
-
- recover();
- }
+ private StorageEngine() {}
public static StorageEngine getInstance() {
return InstanceHolder.INSTANCE;
@@ -275,6 +257,25 @@ public class StorageEngine implements IService {
@Override
public void start() {
+ // build time Interval to divide time partition
+ if (!enablePartition) {
+ timePartitionInterval = Long.MAX_VALUE;
+ } else {
+ initTimePartition();
+ }
+
+ // create systemDir
+ try {
+ FileUtils.forceMkdir(SystemFileFactory.INSTANCE.getFile(systemDir));
+ } catch (IOException e) {
+ throw new StorageEngineFailureException(e);
+ }
+
+ // recover upgrade process
+ UpgradeUtils.recoverUpgrade();
+
+ recover();
+
ttlCheckThread = Executors.newSingleThreadScheduledExecutor();
ttlCheckThread.scheduleAtFixedRate(
this::checkTTL, TTL_CHECK_INTERVAL, TTL_CHECK_INTERVAL,
TimeUnit.MILLISECONDS);
@@ -369,7 +370,7 @@ public class StorageEngine implements IService {
for (PartialPath storageGroup :
IoTDB.metaManager.getAllStorageGroupPaths()) {
this.releaseWalDirectByteBufferPoolInOneStorageGroup(storageGroup);
}
- this.reset();
+ processorMap.clear();
}
private void stopTimedService(ScheduledExecutorService pool, String
poolName) {
@@ -398,7 +399,7 @@ public class StorageEngine implements IService {
shutdownTimedService(unseqMemtableTimedFlushCheckThread,
"UnseqMemtableTimedFlushCheckThread");
shutdownTimedService(tsFileTimedCloseCheckThread,
"TsFileTimedCloseCheckThread");
recoveryThreadPool.shutdownNow();
- this.reset();
+ processorMap.clear();
}
private void shutdownTimedService(ScheduledExecutorService pool, String
poolName) {
@@ -558,6 +559,7 @@ public class StorageEngine implements IService {
}
/** This function is just for unit test. */
+ @TestOnly
public synchronized void reset() {
for (VirtualStorageGroupManager virtualStorageGroupManager :
processorMap.values()) {
virtualStorageGroupManager.reset();
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java
index 8d9d122..829850b 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.db.engine.storagegroup.virtualSg;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.metadata.PartialPath;
-import org.apache.iotdb.db.utils.TestOnly;
public class HashVirtualPartitioner implements VirtualPartitioner {
@@ -39,11 +38,6 @@ public class HashVirtualPartitioner implements
VirtualPartitioner {
return toStorageGroupId(deviceId);
}
- @TestOnly
- public void setStorageGroupNum(int i) {
- STORAGE_GROUP_NUM = i;
- }
-
@Override
public int getPartitionCount() {
return STORAGE_GROUP_NUM;
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
index eb59ede..8be8005 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.db.integration;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
-import
org.apache.iotdb.db.engine.storagegroup.virtualSg.HashVirtualPartitioner;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
@@ -36,11 +35,14 @@ import java.sql.*;
* IoTDB server should be defined as integration test.
*/
public class IoTDBCompleteIT {
+ private int prevVirtualStorageGroupNum;
@Before
public void setUp() {
// test different partition
- HashVirtualPartitioner.getInstance().setStorageGroupNum(16);
+ prevVirtualStorageGroupNum =
+ IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum();
+ IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(16);
EnvironmentUtils.closeStatMonitor();
EnvironmentUtils.envSetUp();
}
@@ -48,8 +50,7 @@ public class IoTDBCompleteIT {
@After
public void tearDown() throws Exception {
EnvironmentUtils.cleanEnv();
- HashVirtualPartitioner.getInstance()
-
.setStorageGroupNum(IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum());
+
IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(prevVirtualStorageGroupNum);
}
@Test
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
index 2a752a5..5db1788 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.timeindex.TimeIndexLevel;
-import
org.apache.iotdb.db.engine.storagegroup.virtualSg.HashVirtualPartitioner;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.metadata.PartialPath;
@@ -127,21 +126,21 @@ public class IoTDBLoadExternalTsfileIT {
private static final String TEST_D0_S0_STR = "root.test.d0.s0";
private static final String TEST_D0_S1_STR = "root.test.d0.s1";
private static final String TEST_D1_STR = "root.test.d1.g0.s0";
- private static int virtualPartitionNum = 0;
+
+ private int prevVirtualPartitionNum;
private static String[] deleteSqls =
new String[] {"DELETE STORAGE GROUP root.vehicle", "DELETE STORAGE GROUP
root.test"};
@Before
public void setUp() throws Exception {
+ prevVirtualPartitionNum =
IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum();
+ IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(1);
IoTDBDescriptor.getInstance()
.getConfig()
.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
- IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(1);
- HashVirtualPartitioner.getInstance().setStorageGroupNum(1);
EnvironmentUtils.closeStatMonitor();
EnvironmentUtils.envSetUp();
- virtualPartitionNum =
IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum();
Class.forName(Config.JDBC_DRIVER_NAME);
prepareData(insertSequenceSqls);
}
@@ -152,8 +151,7 @@ public class IoTDBLoadExternalTsfileIT {
IoTDBDescriptor.getInstance()
.getConfig()
.setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION);
-
IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(virtualPartitionNum);
-
HashVirtualPartitioner.getInstance().setStorageGroupNum(virtualPartitionNum);
+
IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(prevVirtualPartitionNum);
}
@Test
diff --git
a/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java
b/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java
index 2ce45a0..e44482b 100644
---
a/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import
org.apache.iotdb.db.engine.storagegroup.virtualSg.HashVirtualPartitioner;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -62,10 +61,13 @@ public class FileLoaderTest {
private String dataDir;
private IFileLoader fileLoader;
+ private int prevVirtualPartitionNum;
+
@Before
public void setUp() throws Exception {
+ prevVirtualPartitionNum =
IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum();
+ IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(1);
IoTDBDescriptor.getInstance().getConfig().setSyncEnable(true);
- HashVirtualPartitioner.getInstance().setStorageGroupNum(1);
EnvironmentUtils.closeStatMonitor();
EnvironmentUtils.envSetUp();
dataDir =
@@ -87,8 +89,7 @@ public class FileLoaderTest {
public void tearDown() throws IOException, StorageEngineException {
EnvironmentUtils.cleanEnv();
IoTDBDescriptor.getInstance().getConfig().setSyncEnable(false);
- HashVirtualPartitioner.getInstance()
-
.setStorageGroupNum(IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum());
+
IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(prevVirtualPartitionNum);
}
@Test
