This is an automated email from the ASF dual-hosted git repository.
masahi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-tvm.git
The following commit(s) were added to refs/heads/master by this push:
new 9f7745e [Relay][Frontend][Onnx] GRU Layer Support (#6020)
9f7745e is described below
commit 9f7745e7c265787d02af8a044a397ab788f9b6e0
Author: Josh Fromm <[email protected]>
AuthorDate: Sun Jul 12 05:26:02 2020 -0700
[Relay][Frontend][Onnx] GRU Layer Support (#6020)
* GRU debugging and testing added to onnx frontend.
* All tests working and code formatted.
* Fix lint issues.
* Add a test case and changed RNN argument parsing.
* Small refactor.
---
python/tvm/relay/frontend/onnx.py | 140 ++++++++++++-
tests/python/frontend/onnx/test_forward.py | 311 ++++++++++++++++++++++-------
2 files changed, 366 insertions(+), 85 deletions(-)
diff --git a/python/tvm/relay/frontend/onnx.py
b/python/tvm/relay/frontend/onnx.py
index 224428b..1568c97 100644
--- a/python/tvm/relay/frontend/onnx.py
+++ b/python/tvm/relay/frontend/onnx.py
@@ -60,6 +60,8 @@ class onnx_input():
def __getitem__(self, item):
if isinstance(item, int):
+ if item > (len(self.input_keys) - 1):
+ return None
return self.input_dict[self.input_keys[item]]
if isinstance(item, str):
if item not in self.input_keys:
@@ -1493,8 +1495,8 @@ class Expand(OnnxOpConverter):
return _op.broadcast_to(inputs[0], shape=tuple(shape))
-class LSTM(OnnxOpConverter):
- """ Operator converter for LSTM.
+class RNN(OnnxOpConverter):
+ """ Operator converter for RNNs such as LSTM and GRU.
"""
@classmethod
@@ -1528,18 +1530,23 @@ class LSTM(OnnxOpConverter):
]
return activation.decode("utf-8") in needs_beta
+
+class LSTM(RNN):
+ """Operator converter for LSTM
+ """
+
@classmethod
def _impl_v7(cls, inputs, attr, params):
# Unpack inputs, note that if optional and not provided then value
will be None.
X = inputs[0]
W = inputs[1]
R = inputs[2]
- B = inputs['B']
+ B = inputs[3]
# Sequence length currently unused as it can be inferred from shapes.
#sequence_lens = inputs['sequence_lens']
- h_0 = inputs['initial_h']
- c_0 = inputs['initial_c']
- P = inputs['P']
+ h_0 = inputs[5]
+ c_0 = inputs[6]
+ P = inputs[7]
num_directions = infer_shape(W)[0]
W_dtype = infer_type(W).type_annotation.dtype
@@ -1577,7 +1584,8 @@ class LSTM(OnnxOpConverter):
if 'activations' in attr:
activations = attr['activations']
if len(activations) != 3:
- raise NotImplementedError("LSTM assumes 3 activation functions
are provided")
+ raise NotImplementedError(
+ "LSTM assumes 3 activation functions are provided")
alpha_loc = 0
alphas = attr.get('activation_alpha', [])
if isinstance(alphas, float):
@@ -1591,10 +1599,12 @@ class LSTM(OnnxOpConverter):
alpha = None
beta = None
activation = activations[i]
- if cls._activation_needs_alpha(activation) and len(alphas) >
alpha_loc:
+ if cls._activation_needs_alpha(
+ activation) and len(alphas) > alpha_loc:
alpha = alphas[alpha_loc]
alpha_loc += 1
- if cls._activation_needs_beta(activation) and len(betas) >
beta_loc:
+ if cls._activation_needs_beta(
+ activation) and len(betas) > beta_loc:
beta = betas[beta_loc]
beta_loc += 1
acts.append(cls._activation_helper(activation, alpha, beta))
@@ -1638,6 +1648,117 @@ class LSTM(OnnxOpConverter):
return _expr.TupleWrapper(_expr.Tuple((output, H_t, C_t)), 3)
+class GRU(RNN):
+ """Operator convert for GRU
+ """
+
+ @classmethod
+ def _impl_v7(cls, inputs, attr, params):
+ # Unpack inputs, note that if optional and not provided then value
will be None.
+ X = inputs[0]
+ W = inputs[1]
+ R = inputs[2]
+ B = inputs[3]
+ # Sequence length currently unused as it can be inferred from shapes.
+ #sequence_lens = inputs['sequence_lens']
+ h_0 = inputs[5]
+ linear_before_reset = attr.get('linear_before_reset', 0)
+
+ num_directions = infer_shape(W)[0]
+ W_dtype = infer_type(W).type_annotation.dtype
+
+ if num_directions != 1:
+ raise NotImplementedError("Bidirectional GRUs not yet supported.")
+ # Remove num_directions axis from weights.
+ W = _op.squeeze(W, axis=[0])
+ R = _op.squeeze(R, axis=[0])
+ if B is not None:
+ B = _op.squeeze(B, axis=[0])
+
+ X_shape = infer_shape(X)
+ hidden_size = infer_shape(R)[-1]
+ batch_size = X_shape[1]
+
+ # Initialize state if not provided.
+ # Otherwise remove bidirectional axis.
+ if h_0 is None:
+ h_0 = _op.zeros((batch_size, hidden_size), W_dtype)
+ else:
+ h_0 = _op.squeeze(h_0, axis=[0])
+
+ H_t = h_0
+ h_list = []
+
+ if 'activations' in attr:
+ activations = attr['activations']
+ if len(activations) != 2:
+ raise NotImplementedError(
+ "GRU assumes 2 activation functions are provided")
+ alpha_loc = 0
+ alphas = attr.get('activation_alpha', [])
+ if isinstance(alphas, float):
+ alphas = [alphas]
+ beta_loc = 0
+ betas = attr.get('activation_beta', [])
+ if isinstance(betas, float):
+ betas = [betas]
+ acts = []
+ for i in range(2):
+ alpha = None
+ beta = None
+ activation = activations[i]
+ if cls._activation_needs_alpha(
+ activation) and len(alphas) > alpha_loc:
+ alpha = alphas[alpha_loc]
+ alpha_loc += 1
+ if cls._activation_needs_beta(
+ activation) and len(betas) > beta_loc:
+ beta = betas[beta_loc]
+ beta_loc += 1
+ acts.append(cls._activation_helper(activation, alpha, beta))
+ f_act, g_act = acts
+ else:
+ f_act = _op.sigmoid
+ g_act = _op.tanh
+
+ X_steps = _op.split(X, indices_or_sections=X_shape[0], axis=0)
+ for step in X_steps:
+ step = _op.squeeze(step, axis=[0])
+ current = _op.nn.dense(step, W)
+ cz, cr, ch = _op.split(current, 3, axis=1)
+ rz, rr, rh = _op.split(R, 3, axis=0)
+ z = cz + _op.nn.dense(H_t, rz)
+ r = cr + _op.nn.dense(H_t, rr)
+ if B is not None:
+ WB, RB = _op.split(B, 2)
+ wbz, wbr, wbh = _op.split(WB, 3, axis=-1)
+ rbz, rbr, rbh = _op.split(RB, 3, axis=-1)
+ z += wbz + rbz
+ r += wbr + rbr
+ if linear_before_reset:
+ h = ch + (r * (_op.nn.dense(H_t, rh) + rbh)) + wbh
+ else:
+ h = ch + _op.nn.dense((r * H_t), rh) + wbh + rbh
+ else:
+ if linear_before_reset:
+ h = ch + (r * (_op.nn.dense(H_t, rh)))
+ else:
+ h = ch + _op.nn.dense((r * H_t), rh)
+
+ z = f_act(z)
+ r = f_act(r)
+ h = g_act(h)
+
+ H_t = ((_expr.const(1, dtype=W_dtype) - z) * h) + (z * H_t)
+ h_list.append(_op.expand_dims(H_t, axis=0))
+ # Concatenate outputs and add back in direction axis.
+ concatenated = _op.concatenate(h_list, 0)
+ output = _op.expand_dims(concatenated, axis=1)
+ H_t = _op.expand_dims(H_t, axis=0)
+
+ return _expr.TupleWrapper(_expr.Tuple((output, H_t)), 2)
+
+
class Resize(OnnxOpConverter):
"""Operator converter for Resize
"""
@@ -1859,6 +1980,7 @@ def _get_convert_map(opset):
'LRN': LRN.get_converter(opset),
# Recurrent Layers
'LSTM': LSTM.get_converter(opset),
+ 'GRU': GRU.get_converter(opset),
# defs/vision
'MaxRoiPool': MaxRoiPool.get_converter(opset),
diff --git a/tests/python/frontend/onnx/test_forward.py
b/tests/python/frontend/onnx/test_forward.py
index 5c472be..8654bf0 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -2573,19 +2573,30 @@ def test_lppool():
pads=None, out_shape=[1, 1, 16, 16, 16],
auto_pad='SAME_UPPER')
-def verify_lstm(seq_length,
- batch_size,
- input_size,
- hidden_size,
- use_bias=False,
- activations=None,
- alphas=None,
- betas=None,
- use_initial_state=False,
- use_peep=False):
- x_np = np.random.uniform(size=(seq_length, batch_size,
input_size)).astype('float32')
- w_np = np.random.uniform(size=(1, 4 * hidden_size,
input_size)).astype('float32')
- r_np = np.random.uniform(size=(1, 4 * hidden_size,
hidden_size)).astype('float32')
+def verify_rnn(seq_length,
+ batch_size,
+ input_size,
+ hidden_size,
+ rnn_type='LSTM',
+ use_bias=False,
+ activations=None,
+ alphas=None,
+ betas=None,
+ use_initial_state=False,
+ use_peep=False,
+ linear_before_reset=False):
+ if rnn_type == 'LSTM':
+ multiplier = 4
+ elif rnn_type == 'GRU':
+ multiplier = 3
+ else:
+ raise NotImplementedError("%s RNNs not yet supported." % rnn_type)
+ x_np = np.random.uniform(size=(seq_length, batch_size,
+ input_size)).astype('float32')
+ w_np = np.random.uniform(size=(1, multiplier * hidden_size,
+ input_size)).astype('float32')
+ r_np = np.random.uniform(size=(1, multiplier * hidden_size,
+ hidden_size)).astype('float32')
input_names = ["X", "W", "R"]
input_tensors = [
helper.make_tensor_value_info("X", TensorProto.FLOAT,
list(x_np.shape)),
@@ -2595,78 +2606,87 @@ def verify_lstm(seq_length,
input_values = [x_np, w_np, r_np]
if use_bias:
- b_np = np.random.uniform(size=(1, 8 * hidden_size)).astype('float32')
+ b_np = np.random.uniform(size=(1, multiplier * 2 *
+ hidden_size)).astype('float32')
input_names.append("B")
input_tensors.append(
- helper.make_tensor_value_info("B", TensorProto.FLOAT, [1, 8 *
hidden_size]))
+ helper.make_tensor_value_info("B", TensorProto.FLOAT,
+ [1, multiplier * 2 * hidden_size]))
input_values.append(b_np)
if use_initial_state:
assert use_bias == True, "Initial states must have bias specified."
sequence_np = np.repeat(seq_length, batch_size).astype('int32')
input_names.append("sequence_lens")
- input_tensors.append(helper.make_tensor_value_info("sequence_lens",
TensorProto.INT32, [batch_size]))
+ input_tensors.append(
+ helper.make_tensor_value_info("sequence_lens", TensorProto.INT32,
+ [batch_size]))
input_values.append(sequence_np)
- initial_h_np = np.random.uniform(size=(1, batch_size,
hidden_size)).astype('float32')
+ initial_h_np = np.random.uniform(size=(1, batch_size,
+ hidden_size)).astype('float32')
input_names.append("initial_h")
input_tensors.append(
helper.make_tensor_value_info("initial_h", TensorProto.FLOAT,
[1, batch_size, hidden_size]))
input_values.append(initial_h_np)
- initial_c_np = np.random.uniform(size=(1, batch_size,
hidden_size)).astype('float32')
- input_names.append("initial_c")
- input_tensors.append(
- helper.make_tensor_value_info("initial_c", TensorProto.FLOAT,
- [1, batch_size, hidden_size]))
- input_values.append(initial_c_np)
+ if rnn_type == 'LSTM':
+ initial_c_np = np.random.uniform(
+ size=(1, batch_size, hidden_size)).astype('float32')
+ input_names.append("initial_c")
+ input_tensors.append(
+ helper.make_tensor_value_info("initial_c", TensorProto.FLOAT,
+ [1, batch_size, hidden_size]))
+ input_values.append(initial_c_np)
- if use_peep:
+ if use_peep and rnn_type == 'LSTM':
assert use_initial_state == True, "Peepholes require initial state to
be specified."
p_np = np.random.uniform(size=(1, 3 * hidden_size)).astype('float32')
input_names.append("P")
input_tensors.append(
- helper.make_tensor_value_info("P", TensorProto.FLOAT, [1, 3 *
hidden_size]))
+ helper.make_tensor_value_info("P", TensorProto.FLOAT,
+ [1, 3 * hidden_size]))
input_values.append(p_np)
Y_shape = [seq_length, 1, batch_size, hidden_size]
Y_h_shape = [1, batch_size, hidden_size]
- Y_c_shape = [1, batch_size, hidden_size]
-
- if activations is None:
- lstm_node = helper.make_node(
- 'LSTM', inputs=input_names, outputs=["Y", "Y_h", "Y_c"],
hidden_size=hidden_size)
- elif alphas is None:
- lstm_node = helper.make_node(
- 'LSTM',
- inputs=input_names,
- outputs=["Y", "Y_h", "Y_c"],
- hidden_size=hidden_size,
- activations=activations)
- else:
- lstm_node = helper.make_node(
- 'LSTM',
- inputs=input_names,
- outputs=["Y", "Y_h", "Y_c"],
- hidden_size=hidden_size,
- activations=activations,
- activation_alpha=alphas,
- activation_beta=betas)
-
- graph = helper.make_graph([lstm_node],
- "lstm_test",
+ outputs = ["Y", "Y_h"]
+ graph_outputs = [
+ helper.make_tensor_value_info("Y", TensorProto.FLOAT, list(Y_shape)),
+ helper.make_tensor_value_info("Y_h", TensorProto.FLOAT,
list(Y_h_shape))
+ ]
+ output_shapes = [Y_shape, Y_h_shape]
+
+ if rnn_type == 'LSTM':
+ Y_c_shape = [1, batch_size, hidden_size]
+ outputs.append("Y_c")
+ graph_outputs.append(
+ helper.make_tensor_value_info("Y_c", TensorProto.FLOAT,
+ list(Y_c_shape)))
+ output_shapes.append(Y_c_shape)
+
+ rnn_node = helper.make_node(
+ rnn_type, inputs=input_names, outputs=outputs, hidden_size=hidden_size)
+ if activations is not None:
+ activations_attr = helper.make_attribute('activations', activations)
+ rnn_node.attribute.append(activations_attr)
+ if alphas is not None:
+ alphas_attr = helper.make_attribute('activation_alpha', alphas)
+ rnn_node.attribute.append(alphas_attr)
+ if betas is not None:
+ betas_attr = helper.make_attribute('activation_beta', betas)
+ rnn_node.attribute.append(betas_attr)
+ if linear_before_reset and rnn_type == 'GRU':
+ lbr_attr = helper.make_attribute('linear_before_reset', 1)
+ rnn_node.attribute.append(lbr_attr)
+
+ graph = helper.make_graph([rnn_node],
+ "rnn_test",
inputs=input_tensors,
- outputs=[
- helper.make_tensor_value_info("Y",
TensorProto.FLOAT,
- list(Y_shape)),
- helper.make_tensor_value_info("Y_h",
TensorProto.FLOAT,
-
list(Y_h_shape)),
- helper.make_tensor_value_info("Y_c",
TensorProto.FLOAT,
-
list(Y_c_shape))
- ])
+ outputs=graph_outputs)
- model = helper.make_model(graph, producer_name='lstm_test')
+ model = helper.make_model(graph, producer_name='rnn_test')
for target, ctx in ctx_list():
onnx_out = get_onnxruntime_output(model, input_values, 'float32')
@@ -2674,37 +2694,75 @@ def verify_lstm(seq_length,
model,
input_values,
target,
- ctx, [Y_shape, Y_h_shape, Y_c_shape],
- output_dtype=['float32', 'float32', 'float32'])
+ ctx,
+ output_shapes,
+ output_dtype=['float32'] * len(output_shapes))
for o_out, t_out in zip(onnx_out, tvm_out):
tvm.testing.assert_allclose(o_out, t_out, rtol=5e-3, atol=5e-3)
def test_lstm():
# No bias.
- verify_lstm(seq_length=2, batch_size=1, input_size=16, hidden_size=32,
use_bias=False)
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=False,
+ rnn_type='LSTM')
# large batch.
- verify_lstm(seq_length=4, batch_size=8, input_size=16, hidden_size=32,
use_bias=True)
+ verify_rnn(
+ seq_length=4,
+ batch_size=8,
+ input_size=16,
+ hidden_size=32,
+ use_bias=True,
+ rnn_type='LSTM')
# Non power of two.
- verify_lstm(seq_length=3, batch_size=3, input_size=16, hidden_size=40,
use_bias=True)
+ verify_rnn(
+ seq_length=3,
+ batch_size=3,
+ input_size=16,
+ hidden_size=40,
+ use_bias=True,
+ rnn_type='LSTM')
# Long sequence.
- verify_lstm(seq_length=8, batch_size=1, input_size=16, hidden_size=32,
use_bias=True)
+ verify_rnn(
+ seq_length=8,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=True,
+ rnn_type='LSTM')
# Large hidden.
- verify_lstm(seq_length=2, batch_size=1, input_size=16, hidden_size=128,
use_bias=True)
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=128,
+ use_bias=True,
+ rnn_type='LSTM')
# Large input.
- verify_lstm(seq_length=2, batch_size=1, input_size=64, hidden_size=32,
use_bias=True)
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=64,
+ hidden_size=32,
+ use_bias=True,
+ rnn_type='LSTM')
# Different activation testing.
# Default value hardsigmoid.
- verify_lstm(
+ verify_rnn(
seq_length=2,
batch_size=1,
input_size=16,
hidden_size=32,
use_bias=False,
- activations=['HardSigmoid', 'Tanh', 'Tanh'])
+ activations=['HardSigmoid', 'Tanh', 'Tanh'],
+ rnn_type='LSTM')
# Multiple parameterized activations.
- verify_lstm(
+ verify_rnn(
seq_length=2,
batch_size=1,
input_size=16,
@@ -2712,9 +2770,10 @@ def test_lstm():
use_bias=False,
activations=['HardSigmoid', 'LeakyRelu', 'Tanh'],
alphas=[2.0, 0.5],
- betas=[.3])
+ betas=[.3],
+ rnn_type='LSTM')
# All parameterized with new Affine activation.
- verify_lstm(
+ verify_rnn(
seq_length=2,
batch_size=1,
input_size=16,
@@ -2722,24 +2781,123 @@ def test_lstm():
use_bias=False,
activations=['HardSigmoid', 'LeakyRelu', 'Affine'],
alphas=[2.0, 0.5, 0.8],
- betas=[.3, 0.1])
+ betas=[.3, 0.1],
+ rnn_type='LSTM')
# Testing with initial state and peepholes
- verify_lstm(
+ verify_rnn(
seq_length=2,
batch_size=1,
input_size=16,
hidden_size=32,
use_bias=True,
- use_initial_state=True)
- verify_lstm(
+ use_initial_state=True,
+ rnn_type='LSTM')
+
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=True,
+ use_initial_state=True,
+ use_peep=True,
+ rnn_type='LSTM')
+
+
+def test_gru():
+ # No bias.
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=False,
+ rnn_type='GRU')
+ # large batch.
+ verify_rnn(
+ seq_length=4,
+ batch_size=8,
+ input_size=16,
+ hidden_size=32,
+ use_bias=True,
+ rnn_type='GRU',
+ linear_before_reset=True)
+ # Non power of two.
+ verify_rnn(
+ seq_length=3,
+ batch_size=3,
+ input_size=16,
+ hidden_size=40,
+ use_bias=True,
+ rnn_type='GRU')
+ # Long sequence.
+ verify_rnn(
+ seq_length=8,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=True,
+ rnn_type='GRU')
+ # Large hidden.
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=128,
+ use_bias=True,
+ rnn_type='GRU')
+ # Large input.
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=64,
+ hidden_size=32,
+ use_bias=True,
+ rnn_type='GRU')
+
+ # Different activation testing.
+ # Default value hardsigmoid.
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=False,
+ activations=['HardSigmoid', 'Softsign'],
+ rnn_type='GRU')
+ # Multiple parameterized activations.
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=False,
+ activations=['HardSigmoid', 'LeakyRelu'],
+ alphas=[2.0, 0.5],
+ betas=[.3],
+ rnn_type='GRU')
+ # All parameterized with new Affine activation.
+ verify_rnn(
+ seq_length=2,
+ batch_size=1,
+ input_size=16,
+ hidden_size=32,
+ use_bias=False,
+ activations=['HardSigmoid', 'Affine'],
+ alphas=[2.0, 0.8],
+ betas=[.3, 0.1],
+ rnn_type='GRU')
+
+ # Testing with initial state
+ verify_rnn(
seq_length=2,
batch_size=1,
input_size=16,
hidden_size=32,
use_bias=True,
use_initial_state=True,
- use_peep=True)
+ rnn_type='GRU')
def test_resize():
@@ -2992,6 +3150,7 @@ if __name__ == '__main__':
test_pooling()
test_lppool()
test_lstm()
+ test_gru()
test_resize()
test_nonzero()
test_topk()
