This is an automated email from the ASF dual-hosted git repository.
haoj pushed a commit to branch numpy
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit 27d5c69600c1ab90021bcc1e72a6c2ac51bed1cf
Author: reminisce <wujun....@gmail.com>
AuthorDate: Tue Jun 18 11:34:46 2019 -0700
[numpy] Fix d2l chapter 5 (#15264)
* Fix parameter initializer
* Add np.save and np.load
* Fix read-write
* Fix lint
---
python/mxnet/gluon/block.py | 11 ++-
python/mxnet/gluon/parameter.py | 44 ++++++----
python/mxnet/initializer.py | 14 +++-
python/mxnet/ndarray/utils.py | 7 ++
python/mxnet/numpy/__init__.py | 1 +
python/mxnet/numpy/multiarray.py | 3 +-
python/mxnet/numpy/utils.py | 122 ++++++++++++++++++++++++++++
tests/python/unittest/test_numpy_ndarray.py | 46 ++++++++++-
8 files changed, 224 insertions(+), 24 deletions(-)
diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py
index 588d12c..7866cfb 100644
--- a/python/mxnet/gluon/block.py
+++ b/python/mxnet/gluon/block.py
@@ -37,6 +37,7 @@ from .utils import _indent, _brief_print_list, HookHandle
from .utils import _check_same_symbol_type, _check_all_np_ndarrays
from .. import numpy_extension as _mx_npx
from .. import numpy as _mx_np
+from .. util import is_np_array
class _BlockScope(object):
@@ -335,7 +336,10 @@ class Block(object):
"""
params = self._collect_params_with_prefix()
arg_dict = {key : val._reduce() for key, val in params.items()}
- ndarray.save(filename, arg_dict)
+ if is_np_array():
+ _mx_np.save(filename, arg_dict)
+ else:
+ ndarray.save(filename, arg_dict)
def save_params(self, filename):
"""[Deprecated] Please use save_parameters. Note that if you want load
@@ -384,7 +388,10 @@ class Block(object):
`Saving and Loading Gluon Models \
<https://mxnet.incubator.apache.org/tutorials/gluon/save_load_params.html>`_
"""
- loaded = ndarray.load(filename)
+ if is_np_array():
+ loaded = _mx_np.load(filename)
+ else:
+ loaded = ndarray.load(filename)
params = self._collect_params_with_prefix()
if not loaded and not params:
return
diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py
index 6d8e5c0..89a3c33 100644
--- a/python/mxnet/gluon/parameter.py
+++ b/python/mxnet/gluon/parameter.py
@@ -18,6 +18,8 @@
# coding: utf-8
# pylint: disable=unnecessary-pass, too-many-lines
"""Neural network parameter."""
+from __future__ import absolute_import
+
__all__ = ['DeferredInitializationError', 'Parameter', 'Constant',
'ParameterDict', 'tensor_types']
@@ -32,6 +34,7 @@ from ..context import Context, cpu
from .. import autograd
from .utils import _indent, _brief_print_list, shape_is_known
from ..util import is_np_shape, is_np_array
+from .. import numpy as _mx_np # pylint: disable=reimported
# pylint: disable= invalid-name
tensor_types = (symbol.Symbol, ndarray.NDArray)
@@ -190,9 +193,9 @@ class Parameter(object):
return
assert len(self._shape) == len(new_shape) and \
- all(j in (0, i) for i, j in zip(new_shape, self._shape)), \
+ all(j in (-1, 0, i) for i, j in zip(new_shape, self._shape)), \
"Expected shape %s is incompatible with given shape %s."%(
- str(new_shape), str(self._shape))
+ str(new_shape), str(self._shape)) # -1 means unknown dim size
in np_shape mode
self._shape = new_shape
@@ -271,12 +274,14 @@ class Parameter(object):
if cast_dtype:
assert dtype_source in ['current', 'saved']
if self.shape:
+ unknown_dim_size = -1 if is_np_shape() else 0
for self_dim, data_dim in zip(self.shape, data.shape):
- assert self_dim in (0, data_dim), \
+ assert self_dim in (unknown_dim_size, data_dim), \
"Failed loading Parameter '%s' from saved params: " \
"shape incompatible expected %s vs saved %s"%(
self.name, str(self.shape), str(data.shape))
- self.shape = tuple(i if i != 0 else j for i, j in zip(self.shape,
data.shape))
+ self.shape = tuple(i if i != unknown_dim_size else j
+ for i, j in zip(self.shape, data.shape))
if self.dtype:
if cast_dtype and np.dtype(self.dtype).type != data.dtype:
if dtype_source == 'current':
@@ -326,13 +331,18 @@ class Parameter(object):
with autograd.pause():
if data is None:
- data = ndarray.zeros(shape=self.shape, dtype=self.dtype,
- ctx=context.cpu(), stype=self._stype)
+ kwargs = {'shape': self.shape, 'dtype': self.dtype, 'ctx':
context.cpu()}
+ if is_np_array():
+ if self._stype != 'default':
+ raise ValueError("mxnet.numpy.zeros does not support
stype = {}"
+ .format(self._stype))
+ zeros_fn = _mx_np.zeros
+ else:
+ kwargs['stype'] = self._stype
+ zeros_fn = ndarray.zeros
+ data = zeros_fn(**kwargs)
initializer.create(default_init)(
initializer.InitDesc(self.name, {'__init__': init}), data)
- # TODO(junwu): use np random operators when available
- if is_np_array():
- data = data.as_np_ndarray() # convert to np.ndarray
self._init_impl(data, ctx)
@@ -355,11 +365,15 @@ class Parameter(object):
self._grad = None
return
- self._grad = [ndarray.zeros(shape=i.shape, dtype=i.dtype,
ctx=i.context,
- stype=self._grad_stype) for i in
self._data]
- # TODO(junwu): use np.zeros
if is_np_array():
- self._grad = [arr.as_np_ndarray() for arr in self._grad]
+ if self._grad_stype != 'default':
+ raise ValueError("mxnet.numpy.zeros does not support stype =
{}"
+ .format(self._grad_stype))
+ self._grad = [_mx_np.zeros(shape=i.shape, dtype=i.dtype,
ctx=i.context)
+ for i in self._data]
+ else:
+ self._grad = [ndarray.zeros(shape=i.shape, dtype=i.dtype,
ctx=i.context,
+ stype=self._grad_stype) for i in
self._data]
autograd.mark_variables(self._check_and_get(self._data, list),
self._grad, self.grad_req)
@@ -773,12 +787,12 @@ class ParameterDict(object):
inferred_shape = []
matched = True
for dim1, dim2 in zip(v, existing):
- if dim1 != dim2 and dim1 * dim2 != 0:
+ if dim1 != dim2 and dim1 > 0 and dim2 > 0:
matched = False
break
elif dim1 == dim2:
inferred_shape.append(dim1)
- elif dim1 == 0:
+ elif dim1 in (0, -1): # -1 means unknown dim size
in np_shape mode
inferred_shape.append(dim2)
else:
inferred_shape.append(dim1)
diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py
index d028247..c5eef63 100755
--- a/python/mxnet/initializer.py
+++ b/python/mxnet/initializer.py
@@ -29,6 +29,8 @@ from .ndarray import NDArray, load
from . import random
from . import registry
from . import ndarray
+from . util import is_np_array
+from . import numpy as _mx_np # pylint: disable=reimported
# inherit str for backward compatibility
class InitDesc(str):
@@ -501,7 +503,8 @@ class Uniform(Initializer):
self.scale = scale
def _init_weight(self, _, arr):
- random.uniform(-self.scale, self.scale, out=arr)
+ uniform_fn = _mx_np.random.uniform if is_np_array() else random.uniform
+ uniform_fn(-self.scale, self.scale, out=arr)
@register
class Normal(Initializer):
@@ -534,7 +537,8 @@ class Normal(Initializer):
self.sigma = sigma
def _init_weight(self, _, arr):
- random.normal(0, self.sigma, out=arr)
+ normal_fn = _mx_np.random.normal if is_np_array() else random.normal
+ normal_fn(0, self.sigma, out=arr)
@register
class Orthogonal(Initializer):
@@ -633,9 +637,11 @@ class Xavier(Initializer):
raise ValueError("Incorrect factor type")
scale = np.sqrt(self.magnitude / factor)
if self.rnd_type == "uniform":
- random.uniform(-scale, scale, out=arr)
+ uniform_fn = _mx_np.random.uniform if is_np_array() else
random.uniform
+ uniform_fn(-scale, scale, out=arr)
elif self.rnd_type == "gaussian":
- random.normal(0, scale, out=arr)
+ normal_fn = _mx_np.random.normal if is_np_array() else
random.normal
+ normal_fn(0, scale, out=arr)
else:
raise ValueError("Unknown random type")
diff --git a/python/mxnet/ndarray/utils.py b/python/mxnet/ndarray/utils.py
index ff93d0b..730f217 100644
--- a/python/mxnet/ndarray/utils.py
+++ b/python/mxnet/ndarray/utils.py
@@ -248,6 +248,7 @@ def save(fname, data):
>>> mx.nd.load('my_dict')
{'y': <NDArray 1x4 @cpu(0)>, 'x': <NDArray 2x3 @cpu(0)>}
"""
+ from ..numpy import ndarray as np_ndarray
if isinstance(data, NDArray):
data = [data]
handles = c_array(NDArrayHandle, [])
@@ -257,11 +258,17 @@ def save(fname, data):
if any(not isinstance(k, string_types) for k in str_keys) or \
any(not isinstance(v, NDArray) for v in nd_vals):
raise TypeError('save only accept dict str->NDArray or list of
NDArray')
+ if any(isinstance(v, np_ndarray) for v in nd_vals):
+ raise TypeError('cannot save mxnet.numpy.ndarray using
mxnet.ndarray.save;'
+ ' use mxnet.numpy.save instead.')
keys = c_str_array(str_keys)
handles = c_handle_array(nd_vals)
elif isinstance(data, list):
if any(not isinstance(v, NDArray) for v in data):
raise TypeError('save only accept dict str->NDArray or list of
NDArray')
+ if any(isinstance(v, np_ndarray) for v in data):
+ raise TypeError('cannot save mxnet.numpy.ndarray using
mxnet.ndarray.save;'
+ ' use mxnet.numpy.save instead.')
keys = None
handles = c_handle_array(data)
else:
diff --git a/python/mxnet/numpy/__init__.py b/python/mxnet/numpy/__init__.py
index e1c9d90..266c2fa 100644
--- a/python/mxnet/numpy/__init__.py
+++ b/python/mxnet/numpy/__init__.py
@@ -24,5 +24,6 @@ from .multiarray import * # pylint: disable=wildcard-import
from . import _op
from . import _register
from ._op import * # pylint: disable=wildcard-import
+from .utils import * # pylint: disable=wildcard-import
__all__ = []
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 3c981d1..52a2cf4 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -1285,8 +1285,7 @@ def array(object, dtype=None, ctx=None):
try:
object = _np.array(object, dtype=dtype)
except Exception as e:
- print(e)
- raise TypeError('source array must be an array like object')
+ raise TypeError('{}'.format(str(e)))
ret = empty(object.shape, dtype=dtype, ctx=ctx)
if len(object.shape) == 0:
ret[()] = object
diff --git a/python/mxnet/numpy/utils.py b/python/mxnet/numpy/utils.py
new file mode 100644
index 0000000..48a47a3
--- /dev/null
+++ b/python/mxnet/numpy/utils.py
@@ -0,0 +1,122 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Util functions for the numpy module."""
+
+
+from __future__ import absolute_import
+
+import ctypes
+from .. util import is_np_array, is_np_shape
+from .. base import _LIB, check_call, string_types, c_str_array
+from .. base import c_handle_array, c_str, mx_uint, NDArrayHandle, py_str
+from . import ndarray
+
+__all__ = ['save', 'load']
+
+
+def save(file, arr):
+ """Saves a list of `ndarray`s or a dict of `str`->`ndarray` to file.
+
+ Examples of filenames:
+
+ - ``/path/to/file``
+ - ``s3://my-bucket/path/to/file`` (if compiled with AWS S3 supports)
+ - ``hdfs://path/to/file`` (if compiled with HDFS supports)
+
+ Parameters
+ ----------
+ file : str
+ Filename to which the data is saved.
+ arr : `ndarray` or list of `ndarray`s or dict of `str` to `ndarray`
+ The data to be saved.
+
+ Notes
+ -----
+ This function can only be called within numpy semantics, i.e.,
`npx.is_np_shape()`
+ and `npx.is_np_array()` must both return true.
+ """
+ if not (is_np_shape() and is_np_array()):
+ raise ValueError('Cannot save `mxnet.numpy.ndarray` in legacy mode.
Please activate'
+ ' numpy semantics by calling `npx.set_np()` in the
global scope'
+ ' before calling this function.')
+ if isinstance(arr, ndarray):
+ arr = [arr]
+ if isinstance(arr, dict):
+ str_keys = arr.keys()
+ nd_vals = arr.values()
+ if any(not isinstance(k, string_types) for k in str_keys) or \
+ any(not isinstance(v, ndarray) for v in nd_vals):
+ raise TypeError('Only accepts dict str->ndarray or list of
ndarrays')
+ keys = c_str_array(str_keys)
+ handles = c_handle_array(nd_vals)
+ elif isinstance(arr, list):
+ if any(not isinstance(v, ndarray) for v in arr):
+ raise TypeError('Only accepts dict str->ndarray or list of
ndarrays')
+ keys = None
+ handles = c_handle_array(arr)
+ else:
+ raise ValueError("data needs to either be a ndarray, dict of (str,
ndarray) pairs "
+ "or a list of ndarrays.")
+ check_call(_LIB.MXNDArraySave(c_str(file),
+ mx_uint(len(handles)),
+ handles,
+ keys))
+
+
+def load(file):
+ """Loads an array from file.
+
+ See more details in ``save``.
+
+ Parameters
+ ----------
+ file : str
+ The filename.
+
+ Returns
+ -------
+ result : list of ndarrays or dict of str -> ndarray
+ Data stored in the file.
+
+ Notes
+ -----
+ This function can only be called within numpy semantics, i.e.,
`npx.is_np_shape()`
+ and `npx.is_np_array()` must both return true.
+ """
+ if not (is_np_shape() and is_np_array()):
+ raise ValueError('Cannot load `mxnet.numpy.ndarray` in legacy mode.
Please activate'
+ ' numpy semantics by calling `npx.set_np()` in the
global scope'
+ ' before calling this function.')
+ if not isinstance(file, string_types):
+ raise TypeError('file required to be a string')
+ out_size = mx_uint()
+ out_name_size = mx_uint()
+ handles = ctypes.POINTER(NDArrayHandle)()
+ names = ctypes.POINTER(ctypes.c_char_p)()
+ check_call(_LIB.MXNDArrayLoad(c_str(file),
+ ctypes.byref(out_size),
+ ctypes.byref(handles),
+ ctypes.byref(out_name_size),
+ ctypes.byref(names)))
+ if out_name_size.value == 0:
+ return [ndarray(NDArrayHandle(handles[i])) for i in
range(out_size.value)]
+ else:
+ assert out_name_size.value == out_size.value
+ return dict(
+ (py_str(names[i]), ndarray(NDArrayHandle(handles[i])))
+ for i in range(out_size.value))
diff --git a/tests/python/unittest/test_numpy_ndarray.py
b/tests/python/unittest/test_numpy_ndarray.py
index 74b3d4d..0d8eacf 100644
--- a/tests/python/unittest/test_numpy_ndarray.py
+++ b/tests/python/unittest/test_numpy_ndarray.py
@@ -18,12 +18,13 @@
# pylint: skip-file
from __future__ import absolute_import
from __future__ import division
+import os
import numpy as _np
import mxnet as mx
from mxnet import np, npx, autograd
from mxnet.gluon import HybridBlock
from mxnet.test_utils import same, assert_almost_equal, rand_shape_nd,
rand_ndarray, assert_exception
-from common import with_seed
+from common import with_seed, TemporaryDirectory
@with_seed()
@@ -625,6 +626,49 @@ def test_np_ndarray_indexing():
test_setitem_autograd(np_array, index)
+@with_seed()
+@npx.use_np
+def test_np_save_load_ndarrays():
+ shapes = [(2, 0, 1), (0,), (), (), (0, 4), (), (3, 0, 0, 0), (2, 1), (0,
5, 0), (4, 5, 6), (0, 0, 0)]
+ array_list = [_np.random.randint(0, 10, size=shape) for shape in shapes]
+ array_list = [np.array(arr, dtype=arr.dtype) for arr in array_list]
+ # test save/load single ndarray
+ for i, arr in enumerate(array_list):
+ with TemporaryDirectory() as work_dir:
+ fname = os.path.join(work_dir, 'dataset.npy')
+ np.save(fname, arr)
+ arr_loaded = np.load(fname)
+ assert isinstance(arr_loaded, list)
+ assert len(arr_loaded) == 1
+ assert _np.array_equal(arr_loaded[0].asnumpy(),
array_list[i].asnumpy())
+
+ # test save/load a list of ndarrays
+ with TemporaryDirectory() as work_dir:
+ fname = os.path.join(work_dir, 'dataset.npy')
+ np.save(fname, array_list)
+ array_list_loaded = mx.nd.load(fname)
+ assert isinstance(arr_loaded, list)
+ assert len(array_list) == len(array_list_loaded)
+ assert all(isinstance(arr, np.ndarray) for arr in arr_loaded)
+ for a1, a2 in zip(array_list, array_list_loaded):
+ assert _np.array_equal(a1.asnumpy(), a2.asnumpy())
+
+ # test save/load a dict of str->ndarray
+ arr_dict = {}
+ keys = [str(i) for i in range(len(array_list))]
+ for k, v in zip(keys, array_list):
+ arr_dict[k] = v
+ with TemporaryDirectory() as work_dir:
+ fname = os.path.join(work_dir, 'dataset.npy')
+ np.save(fname, arr_dict)
+ arr_dict_loaded = np.load(fname)
+ assert isinstance(arr_dict_loaded, dict)
+ assert len(arr_dict_loaded) == len(arr_dict)
+ for k, v in arr_dict_loaded.items():
+ assert k in arr_dict
+ assert _np.array_equal(v.asnumpy(), arr_dict[k].asnumpy())
+
+
if __name__ == '__main__':
import nose
nose.runmodule()
