zheng-da commented on a change in pull request #13418: [MXNET-1185] Support large array in several operators (part 1) URL: https://github.com/apache/incubator-mxnet/pull/13418#discussion_r236930080
########## File path: tests/nightly/test_large_array.py ########## @@ -15,20 +15,118 @@ # specific language governing permissions and limitations # under the License. -import unittest import mxnet as mx +import numpy as np from mxnet import gluon, nd +# dimension constants +MEDIUM_X = 10000 +LARGE_X = 100000000 +LARGE_Y = 50000000 +SMALL_Y = 50 +LARGE_SIZE = LARGE_X * SMALL_Y -class TestLargeArray(unittest.TestCase): - def test_ndarray2numpy(self): - m = gluon.nn.Embedding(14000, 128) - m.initialize() - ind = nd.zeros((700000, 128)) - x = m(ind) - x.shape - test = x.asnumpy() - assert (x.shape == test.shape) +def test_gluon_embedding(): + m = gluon.nn.Embedding(SMALL_Y, MEDIUM_X) + m.initialize() + a = nd.zeros((MEDIUM_X, SMALL_Y)) + b = m(a) + assert b.shape == (MEDIUM_X, SMALL_Y, MEDIUM_X) + assert b.asnumpy().size == LARGE_SIZE + +def test_ndarray_zeros(): + a = nd.zeros(shape=(LARGE_X, SMALL_Y)) + assert a[-1][0] == 0 + assert a.shape == (LARGE_X, SMALL_Y) + assert a.size == LARGE_SIZE + +def test_ndarray_ones(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + assert a[-1][0] == 1 + assert nd.sum(a).asnumpy() == LARGE_SIZE + +def test_ndarray_random_uniform(): + a = nd.random.uniform(shape=(LARGE_X, SMALL_Y)) + assert a[-1][0] != 0 + +def test_ndarray_empty(): + a = nd.empty((LARGE_X, SMALL_Y)) + assert a.shape == (LARGE_X, SMALL_Y) + +def test_elementwise(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + b = nd.ones(shape=(LARGE_X, SMALL_Y)) + res = a + b + assert np.sum(res[-1].asnumpy() == 2) == a.shape[1] + res = a + 1 + assert np.sum(res[-1].asnumpy() == 2) == a.shape[1] + res = nd.sqrt(a + 3) + assert np.sum(res[-1].asnumpy() == 2) == a.shape[1] + +def test_reduce(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + assert nd.sum(a).asnumpy() == a.shape[0] * a.shape[1] + +def test_dot(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + b = nd.ones(shape=(SMALL_Y, SMALL_Y)) + res = nd.dot(a, b) + assert np.sum(res[-1].asnumpy() == SMALL_Y) == b.shape[1] + +def test_FullyConnected(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + b = nd.ones(shape=(SMALL_Y, SMALL_Y)) + res = nd.FullyConnected(a, b, num_hidden=b.shape[1], no_bias=True) + assert np.sum(res[-1].asnumpy() == SMALL_Y) == b.shape[1] + +def test_broadcast(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + b = nd.arange(0, LARGE_X).reshape(LARGE_X, 1) + res = nd.broadcast_to(b, shape=(b.shape[0], SMALL_Y)) + assert np.sum(res[-1].asnumpy() == LARGE_X) == res.shape[1] + res = mx.nd.broadcast_like(b, a) + assert np.sum(res[-1].asnumpy() == LARGE_X) == a.shape[1] + +def test_clip(): + a = nd.arange(0, LARGE_Y).reshape(LARGE_Y, 1) + res = nd.clip(a, a_min=100, a_max=1000) + assert np.sum(res[-1].asnumpy() == 1000) == a.shape[1] + +def test_take(): + # Test take + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + idx = nd.arange(LARGE_X-1000, LARGE_X) + res = nd.take(a, idx) + assert np.sum(res[-1].asnumpy() == 1) == res.shape[1] + +def test_slice(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + res = nd.slice(a, begin=(LARGE_X-1000, 1), end=(LARGE_X, SMALL_Y)) + assert np.sum(res[-1].asnumpy() == 1) == res.shape[1] + +def test_slice_assign(): + a = nd.ones(shape=(LARGE_X, SMALL_Y)) + res = a.copy() Review comment: copy isn't necessary here. I used it because I only created md.nd.ones once. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
