Repository: systemml Updated Branches: refs/heads/master 5df6ab6dd -> 62b5725d8
[SYSTEMML-1630] Allow user to specify custom path to BLAS libraries - This feature is useful in the cloud environment where the user doesnot have sudo permission or where setting environment variables such as LD_LIBRARY_PATH is difficult. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/62b5725d Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/62b5725d Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/62b5725d Branch: refs/heads/master Commit: 62b5725d82c5a24f05d6178a7498a9124746bf36 Parents: 5df6ab6 Author: Niketan Pansare <npan...@us.ibm.com> Authored: Tue Nov 14 22:10:09 2017 -0800 Committer: Niketan Pansare <npan...@us.ibm.com> Committed: Tue Nov 14 22:14:23 2017 -0800 ---------------------------------------------------------------------- .../org/apache/sysml/utils/NativeHelper.java | 31 +++++++++++++++++--- src/main/python/systemml/mlcontext.py | 20 ++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/62b5725d/src/main/java/org/apache/sysml/utils/NativeHelper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java index 63eea16..b57cfd6 100644 --- a/src/main/java/org/apache/sysml/utils/NativeHelper.java +++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java @@ -48,6 +48,7 @@ public class NativeHelper { public static String blasType; private static int maxNumThreads = -1; private static boolean setMaxNumThreads = false; + private static String customLibPath = null; static { // Note: we only support 64 bit Java on x86 and AMD machine supportedArchitectures.put("x86_64", "x86_64"); @@ -58,15 +59,20 @@ public class NativeHelper { private static String hintOnFailures = ""; + public static void setBLASPath(String path) { + customLibPath = path; + init(true); + } + // Performing loading in a method instead of a static block will throw a detailed stack trace in case of fatal errors - private static void init() { + private static void init(boolean forcedInit) { // Only Linux supported for BLAS if(!SystemUtils.IS_OS_LINUX) return; // attemptedLoading variable ensures that we don't try to load SystemML and other dependencies // again and again especially in the parfor (hence the double-checking with synchronized). - if(!attemptedLoading) { + if(!attemptedLoading || forcedInit) { DMLConfig dmlConfig = ConfigurationManager.getDMLConfig(); // ------------------------------------------------------------------------------------- // We allow BLAS to be enabled or disabled or explicitly selected in one of the two ways: @@ -85,7 +91,7 @@ public class NativeHelper { return; } synchronized(NativeHelper.class) { - if(!attemptedLoading) { + if(!attemptedLoading || forcedInit) { // ----------------------------------------------------------------------------- // ============================================================================= // By default, we will native.blas=true and we will attempt to load MKL first. @@ -152,7 +158,7 @@ public class NativeHelper { } public static boolean isNativeLibraryLoaded() { - init(); + init(false); if(maxNumThreads == -1) maxNumThreads = OptimizerUtils.getConstrainedNumThreads(-1); if(isSystemMLLoaded && !setMaxNumThreads && maxNumThreads != -1) { @@ -183,6 +189,22 @@ public class NativeHelper { } private static boolean loadBLAS(String blas, String optionalMsg) { + // First attempt to load from custom library path + if(customLibPath != null) { + String libPath = customLibPath + File.separator + System.mapLibraryName(blas); + try { + System.load(libPath); + // Print to stdout as this feature is intended for cloud environment + System.out.println("Loaded the library:" + libPath); + return true; + } + catch (UnsatisfiedLinkError e1) { + // Print to stdout as this feature is intended for cloud environment + System.out.println("Unable to load " + libPath + ":" + e1.getMessage()); + } + } + + // Then try loading using loadLibrary try { System.loadLibrary(blas); return true; @@ -197,6 +219,7 @@ public class NativeHelper { return false; } } + private static boolean loadLibraryHelper(String path) { InputStream in = null; OutputStream out = null; http://git-wip-us.apache.org/repos/asf/systemml/blob/62b5725d/src/main/python/systemml/mlcontext.py ---------------------------------------------------------------------- diff --git a/src/main/python/systemml/mlcontext.py b/src/main/python/systemml/mlcontext.py index 54e1969..af1a57e 100644 --- a/src/main/python/systemml/mlcontext.py +++ b/src/main/python/systemml/mlcontext.py @@ -22,7 +22,7 @@ # Methods to create Script object script_factory_methods = [ 'dml', 'pydml', 'dmlFromResource', 'pydmlFromResource', 'dmlFromFile', 'pydmlFromFile', 'dmlFromUrl', 'pydmlFromUrl' ] # Utility methods -util_methods = [ 'jvm_stdout', '_java2py', 'getHopDAG' ] +util_methods = [ 'jvm_stdout', '_java2py', 'getHopDAG', 'setBLASPath' ] __all__ = ['MLResults', 'MLContext', 'Script', 'Matrix' ] + script_factory_methods + util_methods import os @@ -64,6 +64,24 @@ def _get_spark_context(): else: raise Exception('Expected spark context to be created.') + + +def setBLASPath(path): + """ + This method useful in the cloud environment where the user + doesnot have sudo permission or where setting environment variables + such as LD_LIBRARY_PATH is difficult. + + Parameters + ---------- + path: String + Custom path where the BLAS libraries where located. + """ + sc = _get_spark_context() + sc._jvm.org.apache.sysml.utils.NativeHelper.setBLASPath(path) + + + # This is useful utility class to get the output of the driver JVM from within a Jupyter notebook # Example usage: # with jvm_stdout():
