This is an automated email from the ASF dual-hosted git repository.
niketanpansare pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push:
new b49ac76 [SYSTEMML-540] Invoke update immediately after backward call
at the script-level.
b49ac76 is described below
commit b49ac760c180baa0582c2168c4b58fb3c0108bc4
Author: Niketan Pansare <npan...@us.ibm.com>
AuthorDate: Sat Mar 23 21:51:45 2019 -0700
[SYSTEMML-540] Invoke update immediately after backward call at the
script-level.
- This reduces the chance of unnecessary evictions especially when there
are statement block cuts.
- The configuration property `perform_fused_backward_update` allows the
user to toggle this behavior and control the script-generation process.
- Also, updated the release creation document to ensure that untracked
files are not included in the artifacts.
- For Resnet-200, the eviction time was reduced from 173.488 seconds to
60.048 seconds with minibatch size of 96.
---
docs/release-creation-process.md | 3 +
src/main/python/systemml/mllearn/estimators.py | 8 ++-
.../scala/org/apache/sysml/api/dl/Caffe2DML.scala | 76 ++++++++++++++++++----
.../scala/org/apache/sysml/api/dl/CaffeLayer.scala | 2 +
4 files changed, 76 insertions(+), 13 deletions(-)
diff --git a/docs/release-creation-process.md b/docs/release-creation-process.md
index 3115390..bf05000 100644
--- a/docs/release-creation-process.md
+++ b/docs/release-creation-process.md
@@ -42,6 +42,9 @@ Step 1: Prepare the release.
# Extract latest code to a directory
<GitRepoHome>
+
+ # Check if there are any untracked files (created by the unit tests)
and remove them to avoid packing them in the artifacts
+ git status
# Go to dev/release directory
cd <GitRepoHome>/dev/release
diff --git a/src/main/python/systemml/mllearn/estimators.py
b/src/main/python/systemml/mllearn/estimators.py
index 8d1e164..456280b 100644
--- a/src/main/python/systemml/mllearn/estimators.py
+++ b/src/main/python/systemml/mllearn/estimators.py
@@ -922,7 +922,8 @@ class Caffe2DML(BaseSystemMLClassifier):
self.estimator.setWeightsToIgnore(ignore_weights)
def set(self, debug=None, train_algo=None, test_algo=None,
parallel_batches=None,
- output_activations=None, perform_one_hot_encoding=None,
parfor_parameters=None, inline_nn_library=None, use_builtin_lstm_fn=None):
+ output_activations=None, perform_one_hot_encoding=None,
parfor_parameters=None, inline_nn_library=None, use_builtin_lstm_fn=None,
+ perform_fused_backward_update=None):
"""
Set input to Caffe2DML
@@ -933,10 +934,11 @@ class Caffe2DML(BaseSystemMLClassifier):
test_algo: can be minibatch, batch, allreduce_parallel_batches or
allreduce (default: minibatch)
parallel_batches: number of parallel batches
output_activations: (developer flag) directory to output activations
of each layer as csv while prediction. To be used only in batch mode (default:
None)
- perform_one_hot_encoding: should perform one-hot encoding in DML using
table function (default: False)
+ perform_one_hot_encoding: should perform one-hot encoding in DML using
table function (default: True)
parfor_parameters: dictionary for parfor parameters when using
allreduce-style algorithms (default: "")
inline_nn_library: whether to inline the NN library when generating
DML using Caffe2DML (default: False)
use_builtin_lstm_fn: whether to use builtin lstm function for LSTM
layer (default: True)
+ perform_fused_backward_update: whether to perform update immediately
after backward pass at the script level. Supported for minibatch and batch
algorithms. (default: True)
"""
if debug is not None:
self.estimator.setInput("$debug", str(debug).upper())
@@ -950,6 +952,8 @@ class Caffe2DML(BaseSystemMLClassifier):
self.estimator.setInput("$parallel_batches", str(parallel_batches))
if use_builtin_lstm_fn is not None:
self.estimator.setInput("$use_builtin_lstm_fn",
str(use_builtin_lstm_fn).upper())
+ if perform_fused_backward_update is not None:
+ self.estimator.setInput("$perform_fused_backward_update",
str(perform_fused_backward_update).upper())
if output_activations is not None:
self.estimator.setInput(
"$output_activations",
diff --git a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
index e480dfc..c5a20db 100644
--- a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
+++ b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
@@ -304,14 +304,27 @@ class Caffe2DML(val sc: SparkContext,
net.getLayers.map(layer => {net.getCaffeLayer(layer).debugLayer = isDebug})
net.getLayers.map(layer => {net.getCaffeLayer(layer).caffe2dmlObj = this})
net.getLayers.filter(layer =>
net.getCaffeLayer(layer).isInstanceOf[LSTM]).map(layer => {
- if (inputs.containsKey("$use_builtin_lstm_fn"))
-
net.getCaffeLayer(layer).asInstanceOf[LSTM].useBuiltinFunction(inputs.get("$use_builtin_lstm_fn").toLowerCase.toBoolean)
+
net.getCaffeLayer(layer).asInstanceOf[LSTM].useBuiltinFunction(getInputBooleanValue("$use_builtin_lstm_fn"))
})
}
// Comma is included
def getParforParameters():String = if
(inputs.containsKey("$parfor_parameters")) inputs.get("$parfor_parameters")
else ""
+ def getInputBooleanValue(key:String):Boolean = {
+ if(inputs.containsKey(key))
+ return inputs.get(key).toLowerCase.toBoolean
+ else {
+ key match {
+ case "$debug" => false
+ case "$perform_one_hot_encoding" => true
+ case "$inline_nn_library" => false
+ case "$use_builtin_lstm_fn" => true
+ case "$perform_fused_backward_update" => true
+ case _ => throw new DMLRuntimeException("Unsupported input:" + key)
+ }
+ }
+ }
//
================================================================================================
// The below method parses the provided network and solver file and
generates DML script.
def getTrainingScript(isSingleNode: Boolean): (Script, String, String) = {
@@ -320,16 +333,18 @@ class Caffe2DML(val sc: SparkContext,
reset // Reset the state of DML generator for training script.
// Flags passed by user
- val DEBUG_TRAINING = if (inputs.containsKey("$debug"))
inputs.get("$debug").toLowerCase.toBoolean else false
- Caffe2DML.INLINE_NN_LIBRARY = if
(inputs.containsKey("$inline_nn_library"))
inputs.get("$inline_nn_library").toLowerCase.toBoolean else false
+ val DEBUG_TRAINING = getInputBooleanValue("$debug")
+ Caffe2DML.INLINE_NN_LIBRARY = getInputBooleanValue("$inline_nn_library")
assign(tabDMLScript, "debug", if (DEBUG_TRAINING) "TRUE" else "FALSE")
setDebugFlags(DEBUG_TRAINING)
appendHeaders(net, solver, true) // Appends DML corresponding to source
and externalFunction statements.
- val performOneHotEncoding =
!inputs.containsKey("$perform_one_hot_encoding") ||
inputs.get("$perform_one_hot_encoding").toBoolean
+ val performOneHotEncoding =
getInputBooleanValue("$perform_one_hot_encoding")
readInputData(net, true, performOneHotEncoding) // Read X_full and
y_full
// Initialize the layers and solvers. Reads weights and bias if $weights
is set.
initWeights(net, solver, inputs.containsKey("$weights"), layersToIgnore)
+
+ val performFusedBackwardUpdate =
getInputBooleanValue("$perform_fused_backward_update")
// Split into training and validation set
// Initializes Caffe2DML.X, Caffe2DML.y, Caffe2DML.XVal, Caffe2DML.yVal
and Caffe2DML.numImages
@@ -355,7 +370,13 @@ class Caffe2DML(val sc: SparkContext,
getTrainingBatch(tabDMLScript)
// -------------------------------------------------------
// Perform forward, backward and update on minibatch
- forward; backward; update
+ forward;
+ if(performFusedBackwardUpdate) {
+ backwardUpdate
+ }
+ else {
+ backward; update
+ }
// -------------------------------------------------------
if(solverParam.getDisplay > 0) {
ifBlock("iter %% " + solverParam.getDisplay + " == 0") {
@@ -385,7 +406,13 @@ class Caffe2DML(val sc: SparkContext,
assign(tabDMLScript, "yb", Caffe2DML.y)
// -------------------------------------------------------
// Perform forward, backward and update on entire dataset
- forward; backward; update
+ forward
+ if(performFusedBackwardUpdate) {
+ backwardUpdate
+ }
+ else {
+ backward; update
+ }
// -------------------------------------------------------
if(solverParam.getDisplay > 0) {
// Show training/validation loss every epoch
@@ -416,6 +443,10 @@ class Caffe2DML(val sc: SparkContext,
rightIndexing(tabDMLScript, "Xb", Caffe2DML.X, "beg", "end")
rightIndexing(tabDMLScript, "yb", Caffe2DML.y, "beg", "end")
forward; backward
+ if(performFusedBackwardUpdate &&
inputs.containsKey("$perform_fused_backward_update")) {
+ // Warn user only if the user explicitly ask for it
+ Caffe2DML.LOG.warn("Fused backward update is not supported for
allreduce_parallel_batches")
+ }
flattenGradients
if(solverParam.getDisplay > 0) {
ifBlock("(iter + j - 1) %% " + solverParam.getDisplay + " ==
0") {
@@ -447,6 +478,10 @@ class Caffe2DML(val sc: SparkContext,
assign(tabDMLScript, "Xb", Caffe2DML.X + "[j,]")
assign(tabDMLScript, "yb", Caffe2DML.y + "[j,]")
forward; backward
+ if(performFusedBackwardUpdate &&
inputs.containsKey("$perform_fused_backward_update")) {
+ // Warn user only if the user explicitly ask for it
+ Caffe2DML.LOG.warn("Fused backward update is not supported for
allreduce_parallel_batches")
+ }
flattenGradients
}
aggregateAggGradients
@@ -513,7 +548,7 @@ class Caffe2DML(val sc: SparkContext,
}
private def displayTrainingLoss(lossLayer: IsLossLayer,
performOneHotEncoding:Boolean): Unit = {
- val DEBUG_TRAINING = if (inputs.containsKey("$debug"))
inputs.get("$debug").toLowerCase.toBoolean else false
+ val DEBUG_TRAINING = getInputBooleanValue("$debug")
tabDMLScript.append("# Compute training loss & accuracy\n")
assign(tabDMLScript, "loss", "0"); assign(tabDMLScript, "accuracy", "0")
lossLayer.computeLoss(dmlScript, numTabs)
@@ -619,6 +654,25 @@ class Caffe2DML(val sc: SparkContext,
tabDMLScript.append("# Update the parameters\n")
net.getLayers.map(layer => solver.update(tabDMLScript,
net.getCaffeLayer(layer)))
}
+ private def backwardUpdate(): Unit = {
+ tabDMLScript.append("# Perform backward pass and update the parameters\n")
+ var skipedLayer:String = null
+ for(layer <- net.getLayers.reverse) {
+ val caffeLayer = net.getCaffeLayer(layer)
+ caffeLayer.backward(tabDMLScript, "")
+ if(caffeLayer.isInstanceOf[Scale] &&
caffeLayer.asInstanceOf[Scale].isPartOfBatchNorm) {
+ skipedLayer = layer // Skip update
+ }
+ else {
+ solver.update(tabDMLScript, caffeLayer) // Perform update of the
current layer
+ if(skipedLayer != null) {
+ // And then update the skipped layer (if any)
+ solver.update(tabDMLScript, net.getCaffeLayer(skipedLayer))
+ skipedLayer = null
+ }
+ }
+ }
+ }
private def initializeGradients(parallel_batches: String): Unit = {
tabDMLScript.append("# Data structure to store gradients computed in
parallel\n")
if(Caffe2DML.USE_PLUS_EQ) {
@@ -730,13 +784,13 @@ class Caffe2DMLModel(val numClasses: String, val sc:
SparkContext, val solver: C
reset // Reset the state of DML generator for training script.
- val DEBUG_PREDICTION = if (estimator.inputs.containsKey("$debug"))
estimator.inputs.get("$debug").toLowerCase.toBoolean else false
- Caffe2DML.INLINE_NN_LIBRARY = if
(estimator.inputs.containsKey("$inline_nn_library"))
estimator.inputs.get("$inline_nn_library").toLowerCase.toBoolean else false
+ val DEBUG_PREDICTION = estimator.getInputBooleanValue("$debug")
+ Caffe2DML.INLINE_NN_LIBRARY =
estimator.getInputBooleanValue("$inline_nn_library")
assign(tabDMLScript, "debug", if (DEBUG_PREDICTION) "TRUE" else "FALSE")
estimator.setDebugFlags(DEBUG_PREDICTION)
appendHeaders(net, solver, false) // Appends DML corresponding to source
and externalFunction statements.
- val performOneHotEncoding =
!estimator.inputs.containsKey("$perform_one_hot_encoding") ||
estimator.inputs.get("$perform_one_hot_encoding").toBoolean
+ val performOneHotEncoding =
estimator.getInputBooleanValue("$perform_one_hot_encoding")
readInputData(net, false, performOneHotEncoding) // Read X_full
and y_full
assign(tabDMLScript, "X", "X_full")
diff --git a/src/main/scala/org/apache/sysml/api/dl/CaffeLayer.scala
b/src/main/scala/org/apache/sysml/api/dl/CaffeLayer.scala
index 62323d1..bf86f38 100644
--- a/src/main/scala/org/apache/sysml/api/dl/CaffeLayer.scala
+++ b/src/main/scala/org/apache/sysml/api/dl/CaffeLayer.scala
@@ -370,6 +370,7 @@ class BatchNorm(val param: LayerParameter, val id: Int, val
net: CaffeNetwork) e
val topLayers = net.getTopLayers(param.getName).map(l =>
net.getCaffeLayer(l)).toList
if (topLayers.length != 1 && !topLayers(0).isInstanceOf[Scale]) throw
new LanguageException("Only one top layer of type Scale allowed for BatchNorm")
scaleLayer = topLayers(0).asInstanceOf[Scale]
+ scaleLayer.isPartOfBatchNorm = true
}
def numChannels = bottomLayerOutputShape._1
def Hin = bottomLayerOutputShape._2
@@ -385,6 +386,7 @@ class Scale(val param: LayerParameter, val id: Int, val
net: CaffeNetwork) exten
override def backward(dmlScript: StringBuilder, outSuffix: String): Unit =
assignDoutToDX(dmlScript, outSuffix)
override def weightShape(): Array[Int] =
Array(bottomLayerOutputShape._1.toInt, 1)
override def biasShape(): Array[Int] =
Array(bottomLayerOutputShape._1.toInt, 1)
+ var isPartOfBatchNorm = false
}
// ------------------------------------------------------------------
