This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ee3319e634 [IOTDB-4772][IOTDB-4402] Avoid multiple IoTDB processes
accessing same dir (#7562)
ee3319e634 is described below
commit ee3319e6340dcbcf021455c249c26348416e9906
Author: Chen YZ <[email protected]>
AuthorDate: Mon Nov 14 13:30:36 2022 +0800
[IOTDB-4772][IOTDB-4402] Avoid multiple IoTDB processes accessing same dir
(#7562)
---
.../apache/iotdb/commons/utils/ProcessIdUtils.java | 35 +++++++
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 33 +++++++
.../db/conf/directories/DirectoryChecker.java | 106 +++++++++++++++++++++
.../java/org/apache/iotdb/db/service/DataNode.java | 8 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 1 +
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 4 +
.../java/org/apache/iotdb/db/service/NewIoTDB.java | 1 +
7 files changed, 184 insertions(+), 4 deletions(-)
diff --git
a/node-commons/src/main/java/org/apache/iotdb/commons/utils/ProcessIdUtils.java
b/node-commons/src/main/java/org/apache/iotdb/commons/utils/ProcessIdUtils.java
new file mode 100644
index 0000000000..3e110670fa
--- /dev/null
+++
b/node-commons/src/main/java/org/apache/iotdb/commons/utils/ProcessIdUtils.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.commons.utils;
+
+import java.lang.management.ManagementFactory;
+
+public class ProcessIdUtils {
+ /**
+ * There exists no platform-independent way that can be guaranteed to work
in all jvm
+ * implementations. ManagementFactory.getRuntimeMXBean().getName() looks
like the best solution,
+ * and typically includes the PID. On linux+windows, it returns a value like
"12345@hostname"
+ * (12345 being the process id).
+ *
+ * @return process id of running Java virtual machine
+ */
+ public static String getProcessId() {
+ return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
index bdb2e7f9f2..b2eb681a2a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
@@ -26,7 +26,10 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.ConfigurationException;
import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.conf.directories.DirectoryChecker;
import org.apache.iotdb.db.metadata.upgrade.MetadataUpgrader;
+import org.apache.iotdb.db.wal.utils.WALMode;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -210,6 +213,36 @@ public class IoTDBStartCheck {
systemProperties.put(DATA_REGION_CONSENSUS_PORT, dataRegionConsensusPort);
}
+ /**
+ * check and create directory before start IoTDB.
+ *
+ * <p>(1) try to create directory, avoid the inability to create directory
at runtime due to lack
+ * of permissions. (2) try to check if the directory is occupied, avoid
multiple IoTDB processes
+ * accessing same director.
+ */
+ public void checkDirectory() throws ConfigurationException, IOException {
+ // check data dirs
+ for (String dataDir : config.getDataDirs()) {
+ DirectoryChecker.getInstance().registerDirectory(new File(dataDir));
+ }
+ // check system dir
+ DirectoryChecker.getInstance().registerDirectory(new
File(config.getSystemDir()));
+ // check WAL dir
+ if (!(config.isClusterMode()
+ && config
+ .getDataRegionConsensusProtocolClass()
+ .equals(ConsensusFactory.RATIS_CONSENSUS))
+ && !config.getWalMode().equals(WALMode.DISABLE)) {
+ for (String walDir : commonConfig.getWalDirs()) {
+ DirectoryChecker.getInstance().registerDirectory(new File(walDir));
+ }
+ }
+ // in cluster mode, check consensus dir
+ if (config.isClusterMode()) {
+ DirectoryChecker.getInstance().registerDirectory(new
File(config.getConsensusDir()));
+ }
+ }
+
/**
* check configuration in system.properties when starting IoTDB
*
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryChecker.java
b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryChecker.java
new file mode 100644
index 0000000000..9ea5bf34b5
--- /dev/null
+++
b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryChecker.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.conf.directories;
+
+import org.apache.iotdb.commons.exception.ConfigurationException;
+import org.apache.iotdb.commons.utils.ProcessIdUtils;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class DirectoryChecker {
+ private static final Logger logger =
LoggerFactory.getLogger(DirectoryChecker.class);
+ private static final String LOCK_FILE_NAME = ".iotdb-lock";
+ private final List<RandomAccessFile> randomAccessFileList = new
ArrayList<>();
+ private final List<File> fileList = new ArrayList<>();
+
+ private DirectoryChecker() {}
+
+ public static DirectoryChecker getInstance() {
+ return DirectoryCheckerHolder.INSTANCE;
+ }
+
+ public void registerDirectory(File dir) throws ConfigurationException,
IOException {
+ if (dir.exists() && !dir.isDirectory()) {
+ throw new ConfigurationException(
+ String.format(
+ "Unable to create directory %s because there is file under the
path, please check configuration and restart.",
+ dir.getAbsolutePath()));
+ } else if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ throw new ConfigurationException(
+ String.format(
+ "Unable to create directory %s, please check configuration and
restart.",
+ dir.getAbsolutePath()));
+ }
+ }
+ File file = new File(dir, LOCK_FILE_NAME);
+ RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw");
+ FileChannel channel = randomAccessFile.getChannel();
+ FileLock lock = null;
+ try {
+ // Try acquiring the lock without blocking. This method returns
+ // null or throws an exception if the file is already locked.
+ lock = channel.tryLock();
+ } catch (OverlappingFileLockException e) {
+ // File is already locked in this thread or virtual machine
+ }
+ // File is already locked other virtual machine
+ if (lock == null) {
+ throw new ConfigurationException(
+ String.format(
+ "Conflict is detected in directory %s, which may be being used
by another IoTDB (ProcessId=%s). Please check configuration and restart.",
+ dir.getAbsolutePath(), randomAccessFile.readLine()));
+ }
+ randomAccessFile.writeBytes(ProcessIdUtils.getProcessId());
+ // add to list
+ fileList.add(file);
+ randomAccessFileList.add(randomAccessFile);
+ }
+
+ public void deregisterAll() {
+ try {
+ for (RandomAccessFile randomAccessFile : randomAccessFileList) {
+ randomAccessFile.close();
+ // it will release lock automatically after close
+ }
+ for (File file : fileList) {
+ FileUtils.delete(file);
+ }
+ } catch (IOException e) {
+ logger.warn("Failed to deregister file lock because {}", e.getMessage(),
e);
+ }
+ }
+
+ private static class DirectoryCheckerHolder {
+ private static final DirectoryChecker INSTANCE = new DirectoryChecker();
+
+ private DirectoryCheckerHolder() {}
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
index dfc90ff5b3..7d5ff0f01e 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
@@ -129,7 +129,11 @@ public class DataNode implements DataNodeMBean {
}
protected void serverCheckAndInit() throws ConfigurationException,
IOException {
+ // set the mpp mode to true
+ config.setMppMode(true);
+ config.setClusterMode(true);
IoTDBStartCheck.getInstance().checkConfig();
+ IoTDBStartCheck.getInstance().checkDirectory();
// TODO: check configuration for data node
for (TEndPoint endPoint : config.getTargetConfigNodeList()) {
@@ -163,7 +167,6 @@ public class DataNode implements DataNodeMBean {
/** initialize the current node and its services */
public boolean initLocalEngines() {
- config.setClusterMode(true);
return true;
}
@@ -175,9 +178,6 @@ public class DataNode implements DataNodeMBean {
// Register services
JMXService.registerMBean(getInstance(), mbeanName);
- // set the mpp mode to true
- config.setMppMode(true);
- config.setClusterMode(true);
}
/** register DataNode with ConfigNode */
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 2c5988c20a..00bd50dd0b 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -71,6 +71,7 @@ public class IoTDB implements IoTDBMBean {
public static void main(String[] args) {
try {
IoTDBStartCheck.getInstance().checkConfig();
+ IoTDBStartCheck.getInstance().checkDirectory();
IoTDBRestServiceCheck.getInstance().checkConfig();
} catch (ConfigurationException | IOException e) {
logger.error("meet error when doing start checking", e);
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
b/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
index 492b87a671..e5ac9e69d8 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.service;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.directories.DirectoryChecker;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.engine.StorageEngineV2;
import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode;
@@ -60,6 +61,9 @@ public class IoTDBShutdownHook extends Thread {
.forEach(id ->
DataRegionConsensusImpl.getInstance().triggerSnapshot(id));
}
+ // clear lock file
+ DirectoryChecker.getInstance().deregisterAll();
+
if (logger.isInfoEnabled()) {
logger.info(
"IoTDB exits. Jvm memory usage: {}",
diff --git a/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java
b/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java
index 53d1f12d52..7996705619 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java
@@ -73,6 +73,7 @@ public class NewIoTDB implements NewIoTDBMBean {
public static void main(String[] args) {
try {
IoTDBStartCheck.getInstance().checkConfig();
+ IoTDBStartCheck.getInstance().checkDirectory();
IoTDBRestServiceCheck.getInstance().checkConfig();
} catch (ConfigurationException | IOException e) {
logger.error("meet error when doing start checking", e);
