Repository: incubator-systemml Updated Branches: refs/heads/master 8324b69f1 -> 47c2dd10f
[MINOR] Updated documentation and improved log messages - Also, BLAS is disabled by default. We can enable it after more rigorous testing. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/47c2dd10 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/47c2dd10 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/47c2dd10 Branch: refs/heads/master Commit: 47c2dd10f43e3f348c4372a8a7b34fa04705930a Parents: 8324b69 Author: Niketan Pansare <npan...@us.ibm.com> Authored: Tue May 2 15:48:49 2017 -0800 Committer: Niketan Pansare <npan...@us.ibm.com> Committed: Tue May 2 16:48:49 2017 -0700 ---------------------------------------------------------------------- conf/SystemML-config.xml.template | 2 +- docs/beginners-guide-caffe2dml.md | 34 ++++++++++++++++++ docs/index.md | 1 + docs/native-backend.md | 35 +++++++++++------- .../sysml/api/mlcontext/ScriptExecutor.java | 12 +++---- .../java/org/apache/sysml/conf/DMLConfig.java | 2 +- .../org/apache/sysml/utils/NativeHelper.java | 38 +++++++++++++++----- .../org/apache/sysml/api/dl/Caffe2DML.scala | 9 ++--- .../org/apache/sysml/api/dl/CaffeNetwork.scala | 6 ++-- .../scala/org/apache/sysml/api/dl/Utils.scala | 12 ------- 10 files changed, 102 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/conf/SystemML-config.xml.template ---------------------------------------------------------------------- diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template index 0ccf7da..8a6ac54 100644 --- a/conf/SystemML-config.xml.template +++ b/conf/SystemML-config.xml.template @@ -67,7 +67,7 @@ <codegen.literals>1</codegen.literals> <!-- enables native blas for matrix multiplication and convolution, experimental feature --> - <native.blas>true</native.blas> + <native.blas>false</native.blas> <!-- prints extra statistics information for GPU --> <systemml.stats.extraGPU>false</systemml.stats.extraGPU> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/docs/beginners-guide-caffe2dml.md ---------------------------------------------------------------------- diff --git a/docs/beginners-guide-caffe2dml.md b/docs/beginners-guide-caffe2dml.md index cfcc0cb..dea53fd 100644 --- a/docs/beginners-guide-caffe2dml.md +++ b/docs/beginners-guide-caffe2dml.md @@ -31,6 +31,40 @@ limitations under the License. Caffe2DML is an experimental API that converts an Caffe specification to DML. +## Example: Train Lenet + +1. Install `mlextend` package to get MNIST data: `pip install mlxtend`. +2. (Optional but recommended) Follow the steps mentioned in [the user guide]([the user guide of native backend](http://apache.github.io/incubator-systemml/native-backend)) and install Intel MKL. +3. Install [SystemML](http://apache.github.io/incubator-systemml/beginners-guide-python#install-systemml). +4. Invoke PySpark shell: `pyspark --conf spark.executorEnv.LD_LIBRARY_PATH=/path/to/blas-n-other-dependencies`. + +```bash +# Download the MNIST dataset +from mlxtend.data import mnist_data +import numpy as np +from sklearn.utils import shuffle +X, y = mnist_data() +X, y = shuffle(X, y) + +# Split the data into training and test +n_samples = len(X) +X_train = X[:int(.9 * n_samples)] +y_train = y[:int(.9 * n_samples)] +X_test = X[int(.9 * n_samples):] +y_test = y[int(.9 * n_samples):] + +# Download the Lenet network +import urllib +urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/lenet/mnist/lenet.proto', 'lenet.proto') +urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/lenet/mnist/lenet_solver.proto', 'lenet_solver.proto') + +# Train Lenet On MNIST using scikit-learn like API +from systemml.mllearn import Caffe2DML +lenet = Caffe2DML(sqlCtx, solver='lenet_solver.proto', input_shape=(1, 28, 28)).set(debug=True).setStatistics(True) +lenet.fit(X_train, y_train) +y_predicted = lenet.predict(X_test) +``` + ## Frequently asked questions - How to set batch size ? http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/docs/index.md ---------------------------------------------------------------------- diff --git a/docs/index.md b/docs/index.md index c84e7b7..080dfd2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -57,6 +57,7 @@ machine in R-like and Python-like declarative languages. in Standalone Mode. * [JMLC](jmlc) - Java Machine Learning Connector. * See [Java Machine Learning Connector (JMLC)](jmlc) for more information. +* *Experimental* [Caffe2DML API](http://apache.github.io/incubator-systemml/beginners-guide-caffe2dml) for Deep Learning. ## Language Guides http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/docs/native-backend.md ---------------------------------------------------------------------- diff --git a/docs/native-backend.md b/docs/native-backend.md index 86a1340..c64c2fe 100644 --- a/docs/native-backend.md +++ b/docs/native-backend.md @@ -1,3 +1,8 @@ +--- +layout: global +title: Using SystemML with Native BLAS support +description: Using SystemML with Native BLAS support +--- <!-- {% comment %} Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,6 +22,11 @@ limitations under the License. {% endcomment %} --> +* This will become a table of contents (this text will be scraped). +{:toc} + +<br/> + # User Guide By default, SystemML implements all its matrix operations in Java. @@ -25,16 +35,16 @@ This simplifies deployment especially in a distributed environment. In some cases (such as deep learning), the user might want to use native BLAS rather than SystemML's internal Java library for performing single-node operations such matrix multiplication, convolution, etc. + +To allow SystemML to use native BLAS rather than internal Java library, +please set the configuration property `native.blas` to `true`. + By default, SystemML will first attempt to use Intel MKL (if installed) and then OpenBLAS (if installed). If both Intel MKL and OpenBLAS are not available, SystemML falls back to its internal Java library. -To force SystemML to use internal Java library rather than native BLAS, -please set the configuration property `native.blas` to `false`. - -The current version of SystemML only supports BLAS on Linux machines. - +The current version of SystemML only supports BLAS on **Linux** machines. ## Step 1: Install BLAS @@ -95,19 +105,20 @@ sudo ln -s /lib64/libgomp.so.1 /lib64/libgomp.so ## Step 3: Provide the location of the native libraries -1. Add the location of the native libraries (i.e. BLAS and other dependencies) +1. Pass the location of the native libraries using command-line options: + +- [Spark](http://spark.apache.org/docs/latest/configuration.html): `--conf spark.executorEnv.LD_LIBRARY_PATH=/path/to/blas-n-other-dependencies` +- Java: `-Djava.library.path=/path/to/blas-n-other-dependencies` + +2. Alternatively, you can add the location of the native libraries (i.e. BLAS and other dependencies) to the environment variable `LD_LIBRARY_PATH` (on Linux). -If you want to use SystemML with Spark, please add the following line to `spark-env.sh` +If you want to use SystemML with Spark, please add the following line to `spark-env.sh` +(or to the bash profile). ```bash export LD_LIBRARY_PATH=/path/to/blas-n-other-dependencies - # Or export SPARK_LIBRARY_PATH=/path/to/blas-n-other-dependencies ``` -2. Alternatively, you can pass the location of the native libraries using command-line options: - -- Java: `-Djava.library.path=/path/to/blas-n-other-dependencies` -- [Spark](http://spark.apache.org/docs/latest/configuration.html): `--driver-library-path` ## Common issues on Linux http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index 56beef3..c70fbcf 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -256,12 +256,12 @@ public class ScriptExecutor { */ protected void setGlobalFlags() { oldStatistics = DMLScript.STATISTICS; - DMLScript.STATISTICS = statistics; - oldForceGPU = DMLScript.FORCE_ACCELERATOR; - DMLScript.FORCE_ACCELERATOR = forceGPU; - oldGPU = DMLScript.USE_ACCELERATOR; - DMLScript.USE_ACCELERATOR = gpu; - DMLScript.STATISTICS_COUNT = statisticsMaxHeavyHitters; + DMLScript.STATISTICS = statistics; + oldForceGPU = DMLScript.FORCE_ACCELERATOR; + DMLScript.FORCE_ACCELERATOR = forceGPU; + oldGPU = DMLScript.USE_ACCELERATOR; + DMLScript.USE_ACCELERATOR = gpu; + DMLScript.STATISTICS_COUNT = statisticsMaxHeavyHitters; } /** http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/java/org/apache/sysml/conf/DMLConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java index 84bea42..5cc496e 100644 --- a/src/main/java/org/apache/sysml/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java @@ -116,7 +116,7 @@ public class DMLConfig _defaultVals.put(CODEGEN, "false" ); _defaultVals.put(CODEGEN_PLANCACHE, "true" ); _defaultVals.put(CODEGEN_LITERALS, "1" ); - _defaultVals.put(NATIVE_BLAS, "true" ); + _defaultVals.put(NATIVE_BLAS, "false" ); _defaultVals.put(EXTRA_GPU_STATS, "false" ); _defaultVals.put(EXTRA_DNN_STATS, "false" ); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/java/org/apache/sysml/utils/NativeHelper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java index 2b997ed..303b7a8 100644 --- a/src/main/java/org/apache/sysml/utils/NativeHelper.java +++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import java.util.HashMap; +import java.util.Vector; import java.io.InputStream; import java.io.OutputStream; import java.io.File; @@ -94,24 +95,45 @@ public class NativeHelper { if(userSpecifiedBLAS.equalsIgnoreCase("")) { blasType = isMKLAvailable() ? "mkl" : isOpenBLASAvailable() ? "openblas" : null; if(blasType == null) - LOG.warn("Unable to load either MKL or OpenBLAS"); + LOG.info("Unable to load either MKL or OpenBLAS. Please set "); } else if(userSpecifiedBLAS.equalsIgnoreCase("mkl")) { blasType = isMKLAvailable() ? "mkl" : null; if(blasType == null) - LOG.warn("Unable to load MKL"); + LOG.info("Unable to load MKL"); } else if(userSpecifiedBLAS.equalsIgnoreCase("openblas")) { blasType = isOpenBLASAvailable() ? "openblas" : null; if(blasType == null) - LOG.warn("Unable to load OpenBLAS"); + LOG.info("Unable to load OpenBLAS"); } else { - LOG.warn("Unsupported BLAS:" + userSpecifiedBLAS); + LOG.info("Unsupported BLAS:" + userSpecifiedBLAS); } // ============================================================================= if(blasType != null && loadLibraryHelper("libsystemml_" + blasType + "-Linux-x86_64.so")) { - LOG.info("Using native blas: " + blasType); + String blasPathAndHint = ""; + // ------------------------------------------------------------ + // This logic gets the list of native libraries that are loaded + try { + java.lang.reflect.Field loadedLibraryNamesField = ClassLoader.class.getDeclaredField("loadedLibraryNames"); + loadedLibraryNamesField.setAccessible(true); + @SuppressWarnings("unchecked") + Vector<String> libraries = (Vector<String>) loadedLibraryNamesField.get(ClassLoader.getSystemClassLoader()); + LOG.debug("List of native libraries loaded:" + libraries); + for(String library : libraries) { + if(library.endsWith("libmkl_rt.so")) + blasPathAndHint = " from the path " + library; + else if(library.endsWith("libopenblas.so")) { + blasPathAndHint = " from the path " + library + ". Hint: Please make sure that the libopenblas.so is built with GNU OpenMP threading (ldd " + library + " | grep libgomp)."; + } + } + } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { + LOG.debug("Error while finding list of native libraries:" + e.getMessage()); + } + // ------------------------------------------------------------ + + LOG.info("Using native blas: " + blasType + blasPathAndHint); isSystemMLLoaded = true; } } @@ -155,7 +177,7 @@ public class NativeHelper { // ------------------------------------------------------------ // Set environment variable MKL_THREADING_LAYER to GNU on Linux for performance if(!loadLibraryHelper("libpreload_systemml-Linux-x86_64.so")) { - LOG.warn("Unable to load preload_systemml (required for loading MKL-enabled SystemML library)"); + LOG.debug("Unable to load preload_systemml (required for loading MKL-enabled SystemML library)"); return false; } // The most reliable way in my investigation to ensure that MKL runs smoothly with OpenMP (used by conv2d*) @@ -181,9 +203,9 @@ public class NativeHelper { } catch (UnsatisfiedLinkError e) { if(optionalMsg != null) - LOG.warn("Unable to load " + blas + "(" + optionalMsg + "):" + e.getMessage()); + LOG.debug("Unable to load " + blas + "(" + optionalMsg + "):" + e.getMessage()); else - LOG.warn("Unable to load " + blas + ":" + e.getMessage()); + LOG.debug("Unable to load " + blas + ":" + e.getMessage()); return false; } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala index 7ab9160..22de112 100644 --- a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala +++ b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala @@ -53,13 +53,10 @@ import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyze object Caffe2DML { - val LOG = LogFactory.getLog(classOf[Caffe2DML].getName()) - def fileSep():String = { if(File.separator.equals("\\")) "\\\\" else File.separator } - def setNNLibraryPath(path:String):Unit = { prefix = path + fileSep + "nn"} + val LOG = LogFactory.getLog(classOf[Caffe2DML].getName()) // ------------------------------------------------------------------------ - var prefix = Utils.getPrefix() - def layerDir = prefix + fileSep + "layers" + fileSep - def optimDir = prefix + fileSep + "optim" + fileSep + def layerDir = "nn/layers/" + def optimDir = "nn/optim/" // Naming conventions: val X = "X"; val y = "y"; val batchSize = "BATCH_SIZE"; val numImages = "num_images"; val numValidationImages = "num_validation" http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala b/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala index e585e30..73490b4 100644 --- a/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala +++ b/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala @@ -136,14 +136,14 @@ class CaffeNetwork(netFilePath:String, val currentPhase:Phase, // The bottom layers are the layers available in the getBottomList (from Caffe .proto files) private val _bottomLayers:Map[String, Set[String]] = convertTupleListToMap( _caffeLayerParams.flatMap(l => expandBottomList(l.getName, l.getBottomList))) - CaffeNetwork.LOG.info("Bottom layers:" + _bottomLayers) + CaffeNetwork.LOG.debug("Bottom layers:" + _bottomLayers) // Find the top layers by reversing the bottom list private val _topLayers:Map[String, Set[String]] = convertTupleListToMap(flipKeyValues(_bottomLayers.toList)) - CaffeNetwork.LOG.info("Top layers:" + _topLayers) + CaffeNetwork.LOG.debug("Top layers:" + _topLayers) private val _layers: Map[String, CaffeLayer] = _caffeLayerParams.map(l => l.getName -> convertLayerParameterToCaffeLayer(l)).toMap - CaffeNetwork.LOG.info("Layers:" + _layers) + CaffeNetwork.LOG.debug("Layers:" + _layers) private val _layerIDs: Map[String, Int] = _layers.entrySet().map(x => x.getKey -> x.getValue.id).toMap http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/scala/org/apache/sysml/api/dl/Utils.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/dl/Utils.scala b/src/main/scala/org/apache/sysml/api/dl/Utils.scala index b9d6d33..5181c9b 100644 --- a/src/main/scala/org/apache/sysml/api/dl/Utils.scala +++ b/src/main/scala/org/apache/sysml/api/dl/Utils.scala @@ -76,18 +76,6 @@ object Utils { } } - - def getPrefix():String = { - val f = new File("nn") - if(f.exists() && f.isDirectory()) { - Caffe2DML.LOG.info("Since nn directory exists in current folder, using it.") - return "nn" - } - else { - // TODO: Extract from the jar - throw new RuntimeException("In current version, we require that you download the nn folder into current directory from https://github.com/apache/incubator-systemml/tree/master/scripts/staging/SystemML-NN";) - } - } // -------------------------------------------------------------- // Caffe utility functions
