This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch fix_load_directory_full in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 39c94ca7d6aadcbbf95ff62a5d185c8c67c56b3b Author: DESKTOP-L0L5GPJ\jt <[email protected]> AuthorDate: Sat Jul 13 20:00:34 2024 +0800 add retry when getting directory space --- .../org/apache/iotdb/commons/utils/IOUtils.java | 32 ++++++++++++ .../apache/iotdb/commons/utils/JVMCommonUtils.java | 58 ++++++++-------------- 2 files changed, 53 insertions(+), 37 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java index f2d9e9d2dac..f96ae8443b9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.auth.entity.Role; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import com.google.common.base.Supplier; import org.apache.tsfile.utils.Pair; import java.io.DataInputStream; @@ -33,6 +34,8 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.function.Function; public class IOUtils { @@ -273,4 +276,33 @@ public class IOUtils { clone.flip(); return clone; } + + /** + * Retry a method at most 'maxRetry' times, each time calling 'retryFunc' and passing the result + * to 'validator'. If 'validator' returns true, returns the result. + * + * @param maxRetry maximum number of retires + * @param retryIntervalMS retry interval in milliseconds + * @param retryFunc function to be retried + * @param validator validating the result of 'retryFunc' + * @return true if the result from 'retryFunc' passes validation, false if all retries fail or is + * interrupted + * @param <T> + */ + public static <T> Optional<T> retryNoException( + int maxRetry, long retryIntervalMS, Supplier<T> retryFunc, Function<T, Boolean> validator) { + for (int i = 0; i < maxRetry; i++) { + T result = retryFunc.get(); + if (Boolean.TRUE.equals(validator.apply(result))) { + return Optional.of(result); + } + try { + Thread.sleep(retryIntervalMS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return Optional.empty(); + } + } + return Optional.empty(); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java index 5649dc052c5..9435d54d2c7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java @@ -67,46 +67,30 @@ public class JVMCommonUtils { public static long getUsableSpace(String dir) { File dirFile = FSFactoryProducer.getFSFactory().getFile(dir); dirFile.mkdirs(); - return dirFile.getFreeSpace(); + return IOUtils.retryNoException(5, 2000L, dirFile::getFreeSpace, space -> space > 0).orElse(0L); } public static double getDiskFreeRatio(String dir) { - File dirFile = - new File(dir) { - public boolean mkdirs() { - if (exists()) { - LOGGER.warn("File {} already exists", dir); - return false; - } - if (mkdir()) { - return true; - } - File canonFile = null; - try { - canonFile = getCanonicalFile(); - } catch (IOException e) { - LOGGER.warn("Cannot get canon file of {} ", dir, e); - return false; - } - - File parent = canonFile.getParentFile(); - if (parent == null) { - LOGGER.warn("Cannot get parent file of {} ", canonFile); - return false; - } - if (!parent.mkdirs() && !parent.exists()) { - LOGGER.warn("Cannot create parent file {} ", parent); - return false; - } - if (!canonFile.mkdir()) { - LOGGER.warn("Cannot create canon file {}", canonFile); - return false; - } - return true; - } - }; - dirFile.mkdirs(); - return 1.0 * dirFile.getFreeSpace() / dirFile.getTotalSpace(); + File dirFile = new File(dir); + if (!dirFile.mkdirs()) { + // This may solve getFreeSpace() == 0? + dirFile = new File(dir); + } + long freeSpace = + IOUtils.retryNoException(5, 2000L, dirFile::getFreeSpace, space -> space > 0).orElse(0L); + if (freeSpace == 0) { + LOGGER.warn("Cannot get free space for {} after retries, please check the disk status", dir); + } + long totalSpace = dirFile.getTotalSpace(); + double ratio = 1.0 * freeSpace / totalSpace; + if (ratio <= diskSpaceWarningThreshold) { + LOGGER.warn( + "{} is above the warning threshold, free space {}, total space {}", + dir, + freeSpace, + totalSpace); + } + return ratio; } public static boolean hasSpace(String dir) {
