Repository: incubator-systemml Updated Branches: refs/heads/master 4e445d17e -> 29c307c9a
[SYSTEMML-1580] Remove the usage of SYSTEMML_BLAS environment variable from NativeHelper Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/29c307c9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/29c307c9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/29c307c9 Branch: refs/heads/master Commit: 29c307c9ae5f416d2caac7e6e21aeb8d375dd8e5 Parents: 4e445d1 Author: Niketan Pansare <npan...@us.ibm.com> Authored: Wed May 3 15:15:18 2017 -0700 Committer: Niketan Pansare <npan...@us.ibm.com> Committed: Wed May 3 15:15:18 2017 -0700 ---------------------------------------------------------------------- conf/SystemML-config.xml.template | 4 +-- docs/native-backend.md | 8 ++--- .../java/org/apache/sysml/conf/DMLConfig.java | 2 +- .../cp/AggregateBinaryCPInstruction.java | 6 ++-- .../cp/ConvolutionCPInstruction.java | 4 +-- .../spark/ConvolutionSPInstruction.java | 5 ++- .../sysml/runtime/matrix/data/MatrixBlock.java | 8 ++--- .../org/apache/sysml/utils/NativeHelper.java | 38 ++++++++++---------- 8 files changed, 33 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/conf/SystemML-config.xml.template ---------------------------------------------------------------------- diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template index 8a6ac54..8092ca6 100644 --- a/conf/SystemML-config.xml.template +++ b/conf/SystemML-config.xml.template @@ -66,8 +66,8 @@ <!-- if codegen.enabled, compile literals as constants: 1..heuristic, 2..always --> <codegen.literals>1</codegen.literals> - <!-- enables native blas for matrix multiplication and convolution, experimental feature --> - <native.blas>false</native.blas> + <!-- enables native blas for matrix multiplication and convolution, experimental feature (options: auto, mkl, openblas, none) --> + <native.blas>auto</native.blas> <!-- prints extra statistics information for GPU --> <systemml.stats.extraGPU>false</systemml.stats.extraGPU> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/docs/native-backend.md ---------------------------------------------------------------------- diff --git a/docs/native-backend.md b/docs/native-backend.md index c64c2fe..6207932 100644 --- a/docs/native-backend.md +++ b/docs/native-backend.md @@ -37,7 +37,9 @@ rather than SystemML's internal Java library for performing single-node operations such matrix multiplication, convolution, etc. To allow SystemML to use native BLAS rather than internal Java library, -please set the configuration property `native.blas` to `true`. +please set the configuration property `native.blas` to `auto`. +Other possible options are: `mkl`, `openblas` and `none`. +The first two options will only attempt to use the respective BLAS libraries. By default, SystemML will first attempt to use Intel MKL (if installed) and then OpenBLAS (if installed). @@ -153,10 +155,6 @@ Make sure that this path is accessible to Java as per instructions provided in t This section describes how to compile shared libraries in the folder `src/main/cpp/lib`. This is required when the developer makes changes to cpp directory or while validating the source package during the release process. -To force SystemML to use OpenBLAS instead of Intel MKL if both are installed, -please set the environment variable `SYSTEMML_BLAS` to `openblas`. -This environment variable is used internally for testing and is not required for users. - ## Intro to CMake If you are familiar with cmake, skip this section. http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/src/main/java/org/apache/sysml/conf/DMLConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java index 5cc496e..f8577f3 100644 --- a/src/main/java/org/apache/sysml/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java @@ -116,7 +116,7 @@ public class DMLConfig _defaultVals.put(CODEGEN, "false" ); _defaultVals.put(CODEGEN_PLANCACHE, "true" ); _defaultVals.put(CODEGEN_LITERALS, "1" ); - _defaultVals.put(NATIVE_BLAS, "false" ); + _defaultVals.put(NATIVE_BLAS, "auto" ); _defaultVals.put(EXTRA_GPU_STATS, "false" ); _defaultVals.put(EXTRA_DNN_STATS, "false" ); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java index 59da4ac..3bcdbdb 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateBinaryCPInstruction.java @@ -19,8 +19,6 @@ package org.apache.sysml.runtime.instructions.cp; -import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; @@ -33,6 +31,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; +import org.apache.sysml.utils.NativeHelper; public class AggregateBinaryCPInstruction extends BinaryCPInstruction { @@ -81,8 +80,7 @@ public class AggregateBinaryCPInstruction extends BinaryCPInstruction if( matBlock2 instanceof CompressedMatrixBlock ) soresBlock = (MatrixBlock) (matBlock2.aggregateBinaryOperations(matBlock1, matBlock2, new MatrixBlock(), ab_op)); else { - boolean enableNative = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.NATIVE_BLAS); - soresBlock = (MatrixBlock) (matBlock1.aggregateBinaryOperations(matBlock1, matBlock2, new MatrixBlock(), ab_op, enableNative)); + soresBlock = (MatrixBlock) (matBlock1.aggregateBinaryOperations(matBlock1, matBlock2, new MatrixBlock(), ab_op, NativeHelper.isNativeLibraryLoaded())); } //release inputs/outputs http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java index c794932..d428ed9 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java @@ -22,8 +22,6 @@ package org.apache.sysml.runtime.instructions.cp; import java.util.ArrayList; import java.util.Arrays; -import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.functionobjects.SwapIndex; @@ -342,7 +340,7 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction int Q = (int) ConvolutionUtils.getQ(W, S, stride_w, pad_w); ConvolutionParameters params = new ConvolutionParameters(N, C, H, W, K, R, S, stride_h, stride_w, pad_h, pad_w, _numThreads); - params.enableNative = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.NATIVE_BLAS) && NativeHelper.isNativeLibraryLoaded(); + params.enableNative = NativeHelper.isNativeLibraryLoaded(); if (instOpcode.equalsIgnoreCase("maxpooling") || instOpcode.equalsIgnoreCase("relu_maxpooling")) { if(matBlock.isEmptyBlock()) { outputBlock = new MatrixBlock(N, C*P*Q, true); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java index f687cc0..fcf90e9 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java @@ -25,8 +25,6 @@ import java.util.Iterator; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.spark.broadcast.Broadcast; -import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; @@ -49,6 +47,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.matrix.operators.ReorgOperator; import org.apache.sysml.runtime.util.ConvolutionUtils; +import org.apache.sysml.utils.NativeHelper; import scala.Tuple2; @@ -284,7 +283,7 @@ public class ConvolutionSPInstruction extends UnarySPInstruction { int Q = (int) ConvolutionUtils.getQ(W, S, stride_w, pad_w); ConvolutionParameters params = new ConvolutionParameters(numRowsPerBlock, C, H, W, K, R, S, stride_h, stride_w, pad_h, pad_w, 1); - boolean enableNativeBLAS = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.NATIVE_BLAS); + boolean enableNativeBLAS = NativeHelper.isNativeLibraryLoaded(); JavaPairRDD<MatrixIndexes,MatrixBlock> out = inputRDD.mapPartitionsToPair(new RDDConv2dMapMMFunction(filterBroadcast, params, instOpcode, biasBroadcast, mcRdd.getRows(), enableNativeBLAS), true); //put output RDD handle into symbol table http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index 8fac947..4172e44 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -35,7 +35,6 @@ import java.util.stream.LongStream; import org.apache.commons.math3.random.Well1024a; import org.apache.hadoop.io.DataInputBuffer; import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.Hop.OpOp2; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.lops.MMTSJ.MMTSJType; @@ -86,6 +85,7 @@ import org.apache.sysml.runtime.util.FastBufferedDataInputStream; import org.apache.sysml.runtime.util.FastBufferedDataOutputStream; import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.utils.NativeHelper; @@ -4878,8 +4878,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab public MatrixValue aggregateBinaryOperations(MatrixIndexes m1Index, MatrixValue m1Value, MatrixIndexes m2Index, MatrixValue m2Value, MatrixValue result, AggregateBinaryOperator op ) throws DMLRuntimeException { - boolean enableNativeBLAS = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.NATIVE_BLAS); - return aggregateBinaryOperations(m1Value, m2Value, result, op, enableNativeBLAS); + return aggregateBinaryOperations(m1Value, m2Value, result, op, NativeHelper.isNativeLibraryLoaded()); } public MatrixValue aggregateBinaryOperations(MatrixIndexes m1Index, MatrixValue m1Value, MatrixIndexes m2Index, MatrixValue m2Value, @@ -4889,8 +4888,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab } public MatrixValue aggregateBinaryOperations(MatrixValue m1Value, MatrixValue m2Value, MatrixValue result, AggregateBinaryOperator op) throws DMLRuntimeException { - boolean enableNativeBLAS = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.NATIVE_BLAS); - return aggregateBinaryOperations(m1Value, m2Value, result, op, enableNativeBLAS); + return aggregateBinaryOperations(m1Value, m2Value, result, op, NativeHelper.isNativeLibraryLoaded()); } public MatrixValue aggregateBinaryOperations(MatrixValue m1Value, MatrixValue m2Value, MatrixValue result, AggregateBinaryOperator op, boolean nativeMatMult) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/29c307c9/src/main/java/org/apache/sysml/utils/NativeHelper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java index 7de52fd..129824b 100644 --- a/src/main/java/org/apache/sysml/utils/NativeHelper.java +++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java @@ -56,6 +56,8 @@ public class NativeHelper { private static boolean attemptedLoading = false; + private static String hintOnFailures = ""; + // Performing loading in a method instead of a static block will throw a detailed stack trace in case of fatal errors private static void init() { // Only Linux supported for BLAS @@ -66,8 +68,6 @@ public class NativeHelper { // again and again especially in the parfor (hence the double-checking with synchronized). if(!attemptedLoading) { DMLConfig dmlConfig = ConfigurationManager.getDMLConfig(); - String userSpecifiedBLAS = System.getenv("SYSTEMML_BLAS"); - userSpecifiedBLAS = (userSpecifiedBLAS == null) ? "" : userSpecifiedBLAS.trim().toLowerCase(); // ------------------------------------------------------------------------------------- // We allow BLAS to be enabled or disabled or explicitly selected in one of the two ways: // 1. DML Configuration: native.blas (boolean flag) @@ -76,13 +76,12 @@ public class NativeHelper { // The option 2 is useful for two reasons: // - Developer testing of different BLAS // - Provides fine-grained control. Certain machines could use mkl while others use openblas, etc. - boolean enabledViaConfig = (dmlConfig == null) ? true : dmlConfig.getBooleanValue(DMLConfig.NATIVE_BLAS); - boolean enabledViaEnvironmentVariable = userSpecifiedBLAS.equals("") || userSpecifiedBLAS.equals("mkl") || userSpecifiedBLAS.equals("openblas"); - - if(enabledViaConfig && enabledViaEnvironmentVariable) { + String userSpecifiedBLAS = (dmlConfig == null) ? "auto" : dmlConfig.getTextValue(DMLConfig.NATIVE_BLAS).trim().toLowerCase(); + + if(userSpecifiedBLAS.equals("auto") || userSpecifiedBLAS.equals("mkl") || userSpecifiedBLAS.equals("openblas")) { long start = System.nanoTime(); if(!supportedArchitectures.containsKey(SystemUtils.OS_ARCH)) { - LOG.warn("Unsupported architecture for native BLAS:" + SystemUtils.OS_ARCH); + LOG.info("Unsupported architecture for native BLAS:" + SystemUtils.OS_ARCH); return; } synchronized(NativeHelper.class) { @@ -92,23 +91,24 @@ public class NativeHelper { // By default, we will native.blas=true and we will attempt to load MKL first. // If MKL is not enabled then we try to load OpenBLAS. // If both MKL and OpenBLAS are not available we fall back to Java BLAS. - if(userSpecifiedBLAS.equalsIgnoreCase("")) { + if(userSpecifiedBLAS.equals("auto")) { blasType = isMKLAvailable() ? "mkl" : isOpenBLASAvailable() ? "openblas" : null; if(blasType == null) - LOG.info("Unable to load either MKL or OpenBLAS. Please set "); + LOG.info("Unable to load either MKL or OpenBLAS due to " + hintOnFailures); } - else if(userSpecifiedBLAS.equalsIgnoreCase("mkl")) { + else if(userSpecifiedBLAS.equals("mkl")) { blasType = isMKLAvailable() ? "mkl" : null; if(blasType == null) - LOG.info("Unable to load MKL"); + LOG.info("Unable to load MKL due to " + hintOnFailures); } - else if(userSpecifiedBLAS.equalsIgnoreCase("openblas")) { + else if(userSpecifiedBLAS.equals("openblas")) { blasType = isOpenBLASAvailable() ? "openblas" : null; if(blasType == null) - LOG.info("Unable to load OpenBLAS"); + LOG.info("Unable to load OpenBLAS due to " + hintOnFailures); } else { - LOG.info("Unsupported BLAS:" + userSpecifiedBLAS); + // Only thrown at development time. + throw new RuntimeException("Unsupported BLAS:" + userSpecifiedBLAS); } // ============================================================================= if(blasType != null && loadLibraryHelper("libsystemml_" + blasType + "-Linux-x86_64.so")) { @@ -141,14 +141,11 @@ public class NativeHelper { } } double timeToLoadInMilliseconds = (System.nanoTime()-start)*1e-6; - if(timeToLoadInMilliseconds > 100) + if(timeToLoadInMilliseconds > 1000) LOG.warn("Time to load native blas: " + timeToLoadInMilliseconds + " milliseconds."); } else { - if(enabledViaConfig) - LOG.warn("Using internal Java BLAS as native BLAS support is disabled by the configuration 'native.blas'."); - else - LOG.warn("Using internal Java BLAS as native BLAS support is disabled by the environment variable 'SYSTEMML_BLAS=" + userSpecifiedBLAS + "'."); + LOG.warn("Using internal Java BLAS as native BLAS support the configuration 'native.blas'=" + userSpecifiedBLAS + "."); } attemptedLoading = true; } @@ -180,6 +177,7 @@ public class NativeHelper { // Set environment variable MKL_THREADING_LAYER to GNU on Linux for performance if(!loadLibraryHelper("libpreload_systemml-Linux-x86_64.so")) { LOG.debug("Unable to load preload_systemml (required for loading MKL-enabled SystemML library)"); + hintOnFailures = hintOnFailures + " libpreload_systemml-Linux-x86_64.so"; return false; } // The most reliable way in my investigation to ensure that MKL runs smoothly with OpenMP (used by conv2d*) @@ -204,6 +202,8 @@ public class NativeHelper { return true; } catch (UnsatisfiedLinkError e) { + if(!hintOnFailures.contains(blas)) + hintOnFailures = hintOnFailures + blas + " "; if(optionalMsg != null) LOG.debug("Unable to load " + blas + "(" + optionalMsg + "):" + e.getMessage()); else
