This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 78bddba157 [SYSTEMDS-3602] Settings warnings (Memory)
78bddba157 is described below
commit 78bddba157e482222eaf07e3c2b22ad74b3fd2e8
Author: baunsgaard <[email protected]>
AuthorDate: Mon Jul 31 18:40:15 2023 +0200
[SYSTEMDS-3602] Settings warnings (Memory)
This commit adds a warning in case systemds is launched with low
memory settings. The current implementation only works in Linux,
and need some extensions for Windows and macOS.
---
conf/log4j.properties | 1 +
src/main/java/org/apache/sysds/api/DMLScript.java | 3 +
.../org/apache/sysds/utils/SettingsChecker.java | 102 +++++++++++++++++++++
3 files changed, 106 insertions(+)
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 9b751b57ca..befcdaf159 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -22,6 +22,7 @@
log4j.rootLogger=ERROR,console
log4j.logger.org.apache.sysds=ERROR
+log4j.logger.org.apache.sysds.utils.SettingsChecker=WARN
log4j.logger.org.apache.spark=ERROR
log4j.logger.org.apache.hadoop=OFF
diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java
b/src/main/java/org/apache/sysds/api/DMLScript.java
index 795da3ea26..ff70b330ae 100644
--- a/src/main/java/org/apache/sysds/api/DMLScript.java
+++ b/src/main/java/org/apache/sysds/api/DMLScript.java
@@ -86,6 +86,7 @@ import org.apache.sysds.utils.Explain;
import org.apache.sysds.utils.Explain.ExplainCounts;
import org.apache.sysds.utils.Explain.ExplainType;
import org.apache.sysds.utils.NativeHelper;
+import org.apache.sysds.utils.SettingsChecker;
import org.apache.sysds.utils.Statistics;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -279,6 +280,8 @@ public class DMLScript
boolean help = dmlOptions.help;
+ SettingsChecker.check();
+
if (help) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp( "systemds",
dmlOptions.options );
diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java
b/src/main/java/org/apache/sysds/utils/SettingsChecker.java
new file mode 100644
index 0000000000..dd92583a37
--- /dev/null
+++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.utils;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Settings Checker class that contains checks for the JVM setting of systemds
when executed. These checks helps users
+ * configure systemds correctly in case the settings were incorrectly set.
+ */
+public interface SettingsChecker {
+ public static final Log LOG =
LogFactory.getLog(SettingsChecker.class.getName());
+
+ /**
+ * Check if the settings set are correct, otherwise write warnings to a
user.
+ */
+ public static void check() {
+ if(LOG.isWarnEnabled()) {
+ checkMemorySetting();
+ }
+ }
+
+ private static void checkMemorySetting() {
+ long JRE_Mem_Byte = Runtime.getRuntime().maxMemory();
+ long Sys_Mem_Byte = maxMemMachine() * 1024;
+ // Default 500MB
+ final long DefaultJava_500MB = 1024L * 1024 * 500;
+ // 10 GB
+ final long Logging_Limit = 1024L * 1024 * 1024 * 10;
+
+ if(JRE_Mem_Byte <= DefaultJava_500MB) {
+ String st =
byteMemoryToHumanReadableString(JRE_Mem_Byte);
+ LOG.warn("Low memory budget set of: " + st + " this
should most likely be increased");
+ }
+ else if(JRE_Mem_Byte < Logging_Limit && JRE_Mem_Byte * 10 <
Sys_Mem_Byte) {
+ String st =
byteMemoryToHumanReadableString(JRE_Mem_Byte);
+ String sm =
byteMemoryToHumanReadableString(Sys_Mem_Byte);
+ LOG.warn("Low memory budget of total: " + sm + " set
to: " + st);
+ }
+ }
+
+ private static long maxMemMachine() {
+ if("Linux".equals(System.getProperty("os.name"))) {
+ return maxMemMachineLinux();
+ }
+ // TODO add windows.
+ else {
+ return -1;
+ }
+ }
+
+ private static long maxMemMachineLinux() {
+ try(BufferedReader reader = new BufferedReader(new
FileReader("/proc/meminfo"));) {
+ String currentLine = reader.readLine();
+ while(!currentLine.contains("MemTotal:"))
+ currentLine = reader.readLine();
+ return
Long.parseLong(currentLine.split(":")[1].split("kB")[0].strip());
+ }
+ catch(Exception e) {
+ LOG.error(e);
+ return -1;
+ }
+ }
+
+ /**
+ * Converts a number of bytes in a long to a human readable string with
GB, MB, KB and B.
+ *
+ * @param bytes Number of bytes.
+ * @return A human readable string
+ */
+ public static String byteMemoryToHumanReadableString(long bytes) {
+ if(bytes > 1000000000)
+ return String.format("%6d GB", bytes / 1024 / 1024 /
1024);
+ else if(bytes > 1000000)
+ return String.format("%6d MB", bytes / 1024 / 1024);
+ else if(bytes > 1000)
+ return String.format("%6d KB", bytes / 1024);
+ else
+ return String.format("%6d B", bytes);
+ }
+}
