This is an automated email from the ASF dual-hosted git repository. haibin pushed a commit to branch numpy in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/numpy by this push: new db37dd9 [numpy] Fix numpy import in python2 (#14537) db37dd9 is described below commit db37dd9a93ab64f3247b9ca785b347d2239c8a70 Author: reminisce <wujun....@gmail.com> AuthorDate: Wed Mar 27 13:49:59 2019 -0700 [numpy] Fix numpy import in python2 (#14537) * Fix several test failures * Fix subgraph op infer shape * Fix sparse slice * Fix deconv infer shape * Fix numpy import compatibility problem in python2 --- python/mxnet/ndarray/_internal.py | 2 -- python/mxnet/ndarray/contrib.py | 1 + python/mxnet/ndarray/register.py | 7 ++--- python/mxnet/symbol/_internal.py | 2 -- python/mxnet/symbol/register.py | 7 ++--- src/common/utils.h | 5 ++++ src/operator/leaky_relu-inl.h | 4 +-- src/operator/nn/deconvolution-inl.h | 12 +++++---- src/operator/nn/deconvolution.cc | 50 ++++++++++++++++++++++++++--------- src/operator/tensor/matrix_op-inl.h | 2 +- tests/python/unittest/test_ndarray.py | 6 ++++- 11 files changed, 66 insertions(+), 32 deletions(-) diff --git a/python/mxnet/ndarray/_internal.py b/python/mxnet/ndarray/_internal.py index 5f3ce97..8045d9b 100644 --- a/python/mxnet/ndarray/_internal.py +++ b/python/mxnet/ndarray/_internal.py @@ -20,8 +20,6 @@ import os as _os import sys as _sys -import numpy as np - try: if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: from .._ctypes.ndarray import NDArrayBase, CachedOp diff --git a/python/mxnet/ndarray/contrib.py b/python/mxnet/ndarray/contrib.py index 74c355d..1718a2c 100644 --- a/python/mxnet/ndarray/contrib.py +++ b/python/mxnet/ndarray/contrib.py @@ -18,6 +18,7 @@ # coding: utf-8 # pylint: disable=wildcard-import, unused-wildcard-import,redefined-outer-name """Contrib NDArray API of MXNet.""" +from __future__ import absolute_import import math import numpy as np from ..context import current_context diff --git a/python/mxnet/ndarray/register.py b/python/mxnet/ndarray/register.py index 05d7f17..1ccf228 100644 --- a/python/mxnet/ndarray/register.py +++ b/python/mxnet/ndarray/register.py @@ -16,9 +16,10 @@ # under the License. """Register backend ops in mxnet.ndarray namespace""" +from __future__ import absolute_import import os as _os import ctypes -import numpy as np # pylint: disable=unused-import +import numpy as _np # pylint: disable=unused-import from ._internal import NDArrayBase, _imperative_invoke # pylint: disable=unused-import from ..ndarray_doc import _build_doc @@ -103,7 +104,7 @@ def %s(*%s, **kwargs):"""%(func_name, arr_name)) if dtype_name is not None: code.append(""" if '%s' in kwargs: - kwargs['%s'] = np.dtype(kwargs['%s']).name"""%( + kwargs['%s'] = _np.dtype(kwargs['%s']).name"""%( dtype_name, dtype_name, dtype_name)) code.append(""" _ = kwargs.pop('name', None) @@ -136,7 +137,7 @@ def %s(%s):"""%(func_name, ', '.join(signature))) code.append(""" if %s is not _Null: keys.append('%s') - vals.append(np.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) + vals.append(_np.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) if not signature_only: code.append(""" diff --git a/python/mxnet/symbol/_internal.py b/python/mxnet/symbol/_internal.py index 53fc684..7e9787e 100644 --- a/python/mxnet/symbol/_internal.py +++ b/python/mxnet/symbol/_internal.py @@ -22,8 +22,6 @@ import sys as _sys import os as _os -import numpy as np - try: if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: from .._ctypes.symbol import SymbolBase, _set_symbol_class diff --git a/python/mxnet/symbol/register.py b/python/mxnet/symbol/register.py index 15c8e5e..ac59f8b 100644 --- a/python/mxnet/symbol/register.py +++ b/python/mxnet/symbol/register.py @@ -17,9 +17,10 @@ # pylint: disable=unused-import """Register backend ops in mxnet.symbol namespace.""" +from __future__ import absolute_import import os as _os import ctypes -import numpy as np +import numpy as _np from . import _internal from ._internal import SymbolBase, _symbol_creator @@ -109,7 +110,7 @@ def %s(*%s, **kwargs):"""%(func_name, arr_name)) if dtype_name is not None: code.append(""" if '%s' in kwargs: - kwargs['%s'] = np.dtype(kwargs['%s']).name"""%( + kwargs['%s'] = _np.dtype(kwargs['%s']).name"""%( dtype_name, dtype_name, dtype_name)) code.append(""" attr = kwargs.pop('attr', None) @@ -175,7 +176,7 @@ def %s(%s):"""%(func_name, ', '.join(signature))) code.append(""" if %s is not _Null: _keys.append('%s') - _vals.append(np.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) + _vals.append(_np.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) code.append(""" if not hasattr(NameManager._current, "value"): diff --git a/src/common/utils.h b/src/common/utils.h index 4843d7e..4fb398d 100644 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -746,6 +746,11 @@ inline void ParallelCopy(DType* dst, const DType* src, index_t size) { * 4. -1 dim size means the dimension's size is unknown. * so that operator's infer shape function can work in backend. * \param shape to be converted. + * Note: It is possible that the shape to be converted is already + * numpy compatible. For example, when a subgraph operator's infer + * shape function is called from the infer shape pass of the whole + * graph, its input/output shapes have been converted to numpy + * compatible shapes. */ inline void ConvertToNumpyShape(mxnet::TShape* shape) { if (shape->ndim() == 0) { // legacy shape ndim = 0 means unknown diff --git a/src/operator/leaky_relu-inl.h b/src/operator/leaky_relu-inl.h index 22f5229..5518352 100644 --- a/src/operator/leaky_relu-inl.h +++ b/src/operator/leaky_relu-inl.h @@ -338,10 +338,10 @@ class LeakyReLUProp : public OperatorProperty { CHECK_EQ(in_shape->size(), 1U) << "Input:[data]"; } const mxnet::TShape &dshape = in_shape->at(leakyrelu::kData); - if (dshape.ndim() == 0) return false; + if (!mxnet::ndim_is_known(dshape)) return false; if (param_.act_type == leakyrelu::kPReLU) { const mxnet::TShape &gshape = in_shape->at(leakyrelu::kGamma); - if (gshape.ndim() == 0) { + if (!mxnet::ndim_is_known(gshape)) { in_shape->at(leakyrelu::kGamma) = mxnet::TShape(Shape1(dshape[1])); } if (dshape == gshape) { diff --git a/src/operator/nn/deconvolution-inl.h b/src/operator/nn/deconvolution-inl.h index b28e478..5f3137f 100644 --- a/src/operator/nn/deconvolution-inl.h +++ b/src/operator/nn/deconvolution-inl.h @@ -134,11 +134,13 @@ struct DeconvolutionParam : public dmlc::Parameter<DeconvolutionParam> { for (size_t i = 0; i < ndim; i++) { // input.ndim() can be larger than ndim, in case that the complete input // shape was passed and not only the ndim last ones - o_pad[i] = stride[i] * (input[(input_ndim - ndim) + i] - 1) + DilatedKernelSize(i); - CHECK_GE(o_pad[i], target_shape[i]) << "too big target shape"; - o_pad[i] -= target_shape[i]; - o_adj[i] = o_pad[i] % 2; - o_pad[i] = (o_pad[i] + 1) / 2; + if (mxnet::dim_size_is_known(input, input_ndim - ndim + i)) { + o_pad[i] = stride[i] * (input[(input_ndim - ndim) + i] - 1) + DilatedKernelSize(i); + CHECK_GE(o_pad[i], target_shape[i]) << "too big target shape"; + o_pad[i] -= target_shape[i]; + o_adj[i] = o_pad[i] % 2; + o_pad[i] = (o_pad[i] + 1) / 2; + } } } else { for (size_t i = 0; i < ndim; i++) { diff --git a/src/operator/nn/deconvolution.cc b/src/operator/nn/deconvolution.cc index d8c91f7..09b255d 100644 --- a/src/operator/nn/deconvolution.cc +++ b/src/operator/nn/deconvolution.cc @@ -54,7 +54,7 @@ static bool DeconvolutionShape(const nnvm::NodeAttrs& attrs, } out_shape->resize(1, mxnet::TShape()); const mxnet::TShape &dshape = (*in_shape)[deconv::kData]; - if (!shape_is_known(dshape)) return false; + if (!mxnet::ndim_is_known(dshape)) return false; if (param_.kernel.ndim() == 1) { // 1d conv @@ -90,8 +90,12 @@ static bool DeconvolutionShape(const nnvm::NodeAttrs& attrs, Shape<3> oshape; oshape[0] = dshape_ncw[0]; oshape[1] = param_.num_filter; - oshape[2] = param_.stride[0] * (dshape_ncw[2] - 1) + - dilated_ksize_x - 2 * o_pad[0] + o_adj[0]; + if (mxnet::dim_size_is_known(dshape_ncw[2])) { + oshape[2] = param_.stride[0] * (dshape_ncw[2] - 1) + + dilated_ksize_x - 2 * o_pad[0] + o_adj[0]; + } else { + oshape[2] = -1; + } if (param_.target_shape.ndim() > 0) { if (param_.target_shape[0] > 0) { @@ -141,10 +145,18 @@ static bool DeconvolutionShape(const nnvm::NodeAttrs& attrs, Shape<4> oshape; oshape[0] = dshape_nchw[0]; oshape[1] = param_.num_filter; - oshape[2] = param_.stride[0] * (dshape_nchw[2] - 1) + - dilated_ksize_y - 2 * o_pad[0] + o_adj[0]; - oshape[3] = param_.stride[1] * (dshape_nchw[3] - 1) + - dilated_ksize_x - 2 * o_pad[1] + o_adj[1]; + if (mxnet::dim_size_is_known(dshape_nchw[2])) { + oshape[2] = param_.stride[0] * (dshape_nchw[2] - 1) + + dilated_ksize_y - 2 * o_pad[0] + o_adj[0]; + } else { + oshape[2] = -1; + } + if (mxnet::dim_size_is_known(dshape_nchw[3])) { + oshape[3] = param_.stride[1] * (dshape_nchw[3] - 1) + + dilated_ksize_x - 2 * o_pad[1] + o_adj[1]; + } else { + oshape[3] = -1; + } if (param_.target_shape.ndim() > 1) { if (param_.target_shape[0] > 0) { @@ -203,12 +215,24 @@ static bool DeconvolutionShape(const nnvm::NodeAttrs& attrs, Shape<5> oshape; oshape[0] = dshape_ncdhw[0]; oshape[1] = param_.num_filter; - oshape[2] = param_.stride[0] * (dshape_ncdhw[2] - 1) + - dilated_ksize_d - 2 * o_pad[0] + o_adj[0]; - oshape[3] = param_.stride[1] * (dshape_ncdhw[3] - 1) + - dilated_ksize_y - 2 * o_pad[1] + o_adj[1]; - oshape[4] = param_.stride[2] * (dshape_ncdhw[4] - 1) + - dilated_ksize_x - 2 * o_pad[2] + o_adj[2]; + if (mxnet::dim_size_is_known(dshape_ncdhw[2])) { + oshape[2] = param_.stride[0] * (dshape_ncdhw[2] - 1) + + dilated_ksize_d - 2 * o_pad[0] + o_adj[0]; + } else { + oshape[2] = -1; + } + if (mxnet::dim_size_is_known(dshape_ncdhw[3])) { + oshape[3] = param_.stride[1] * (dshape_ncdhw[3] - 1) + + dilated_ksize_y - 2 * o_pad[1] + o_adj[1]; + } else { + oshape[3] = -1; + } + if (mxnet::dim_size_is_known(dshape_ncdhw[4])) { + oshape[4] = param_.stride[2] * (dshape_ncdhw[4] - 1) + + dilated_ksize_x - 2 * o_pad[2] + o_adj[2]; + } else { + oshape[4] = -1; + } if (param_.target_shape.ndim() > 2) { if (param_.target_shape[0] > 0) { diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index f79af7c..65f3b2f 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -594,7 +594,7 @@ void SliceCsrImpl(const SliceParam ¶m, const OpContext& ctx, mxnet::TShape begin(N, -1), end(N, -1); for (int i = 0; i < N; ++i) { int s = 0; - if (param.begin[i]) { + if (i < param.begin.ndim() && param.begin[i]) { s = *param.begin[i]; if (s < 0) s += ishape[i]; } diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index 3a17a1e..c71209f 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -120,7 +120,11 @@ def test_ndarray_setitem(): # numpy assignment for empty axis for trivial_shape in [(), (1,), (1, 1), (1, 1, 1)]: - x = mx.nd.zeros(trivial_shape) + if trivial_shape == tuple(): + with mx.numpy.enable_np_comp(): + x = mx.nd.zeros(trivial_shape) + else: + x = mx.nd.zeros(trivial_shape) x[:] = np.ones(trivial_shape) x_np = np.ones(trivial_shape, dtype=x.dtype) assert x.shape == trivial_shape