[incubator-mxnet] branch master updated: Add worker_fn argument to multiworker function (#12177)

[zhreshold]

This is an automated email from the ASF dual-hosted git repository.

zhreshold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new cd6b1cb  Add worker_fn argument to multiworker function (#12177)
cd6b1cb is described below

commit cd6b1cb5a6b4651f2b6fb64575d884a6fa3ef545
Author: Shuai Zheng 
AuthorDate: Wed Aug 15 14:59:58 2018 +0800

Add worker_fn argument to multiworker function (#12177)

* add worker_fn argument to multiworker function

* fix pylin
---
 python/mxnet/gluon/data/dataloader.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/python/mxnet/gluon/data/dataloader.py 
b/python/mxnet/gluon/data/dataloader.py
index e0b6aec..412d313 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -183,7 +183,8 @@ def fetcher_loop(data_queue, data_buffer, pin_memory=False):
 
 class _MultiWorkerIter(object):
 """Interal multi-worker iterator for DataLoader."""
-def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, 
pin_memory=False):
+def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, 
pin_memory=False,
+ worker_fn=worker_loop):
 assert num_workers > 0, "_MultiWorkerIter is not for {} 
workers".format(num_workers)
 self._num_workers = num_workers
 self._dataset = dataset
@@ -200,7 +201,7 @@ class _MultiWorkerIter(object):
 workers = []
 for _ in range(self._num_workers):
 worker = multiprocessing.Process(
-target=worker_loop,
+target=worker_fn,
 args=(self._dataset, self._key_queue, self._data_queue, 
self._batchify_fn))
 worker.daemon = True
 worker.start()

[GitHub] zhreshold closed pull request #12177: Add worker_fn argument to multiworker function

[GitBox]

zhreshold closed pull request #12177: Add worker_fn argument to multiworker 
function
URL: https://github.com/apache/incubator-mxnet/pull/12177
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/python/mxnet/gluon/data/dataloader.py 
b/python/mxnet/gluon/data/dataloader.py
index e0b6aec294a..412d3134476 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -183,7 +183,8 @@ def fetcher_loop(data_queue, data_buffer, pin_memory=False):
 
 class _MultiWorkerIter(object):
 """Interal multi-worker iterator for DataLoader."""
-def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, 
pin_memory=False):
+def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, 
pin_memory=False,
+ worker_fn=worker_loop):
 assert num_workers > 0, "_MultiWorkerIter is not for {} 
workers".format(num_workers)
 self._num_workers = num_workers
 self._dataset = dataset
@@ -200,7 +201,7 @@ def __init__(self, num_workers, dataset, batchify_fn, 
batch_sampler, pin_memory=
 workers = []
 for _ in range(self._num_workers):
 worker = multiprocessing.Process(
-target=worker_loop,
+target=worker_fn,
 args=(self._dataset, self._key_queue, self._data_queue, 
self._batchify_fn))
 worker.daemon = True
 worker.start()


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu commented on a change in pull request #12178: fix flasky unittest for deformable psroi pooling

[GitBox]

marcoabreu commented on a change in pull request #12178: fix flasky unittest 
for deformable psroi pooling
URL: https://github.com/apache/incubator-mxnet/pull/12178#discussion_r210208035
 
 

 ##
 File path: tests/python/unittest/test_operator.py
 ##
 @@ -5010,15 +5067,11 @@ def test_deformable_psroipooling():

sample_per_part=4, group_size=num_group,

pooled_size=num_group, output_dim=num_classes,
trans_std=0.1, 
no_trans=False, name='test_op')
-if grad_nodes[0] == 'offset_data':
-# wider tolerance needed for coordinate differential
-rtol, atol = 1.0, 1e-2
-else:
-rtol, atol = 1e-2, 1e-3
+rtol, atol = 1e-2, 1e-3
 # By now we only have gpu implementation
 if default_context().device_type == 'gpu':
 check_numeric_gradient(op, [im_data, rois_data, 
offset_data], rtol=rtol, atol=atol,
-   grad_nodes=grad_nodes, 
ctx=mx.gpu(0))
+   grad_nodes=grad_nodes, 
ctx=mx.gpu(1))
 
 Review comment:
   Is this intentional?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu closed pull request #12169: Remove fixed seed for test_huber_loss test

[GitBox]

marcoabreu closed pull request #12169: Remove fixed seed for test_huber_loss 
test
URL: https://github.com/apache/incubator-mxnet/pull/12169
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/tests/python/unittest/test_loss.py 
b/tests/python/unittest/test_loss.py
index 2cdc2909ffc..24cc747a308 100644
--- a/tests/python/unittest/test_loss.py
+++ b/tests/python/unittest/test_loss.py
@@ -274,7 +274,7 @@ def test_saveload():
 eval_metric=mx.metric.Loss())
 assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
 
-@with_seed(1234)
+@with_seed()
 def test_huber_loss():
 N = 20
 data = mx.random.uniform(-1, 1, shape=(N, 10))


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu closed issue #11696: test_loss.test_huber_loss has fixed seed that can mask flakiness

[GitBox]

marcoabreu closed issue #11696: test_loss.test_huber_loss has fixed seed that 
can mask flakiness
URL: https://github.com/apache/incubator-mxnet/issues/11696
 
 
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated: Remove fixed seed for test_huber tests (#12169)

[marcoabreu]

This is an automated email from the ASF dual-hosted git repository.

marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new b675d69  Remove fixed seed for test_huber tests (#12169)
b675d69 is described below

commit b675d69fb2d13f1ecb2fddef68cf1f424fcf89b3
Author: vdantu <36211508+vda...@users.noreply.github.com>
AuthorDate: Wed Aug 15 01:56:12 2018 -0700

Remove fixed seed for test_huber tests (#12169)
---
 tests/python/unittest/test_loss.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/python/unittest/test_loss.py 
b/tests/python/unittest/test_loss.py
index 2cdc290..24cc747 100644
--- a/tests/python/unittest/test_loss.py
+++ b/tests/python/unittest/test_loss.py
@@ -274,7 +274,7 @@ def test_saveload():
 eval_metric=mx.metric.Loss())
 assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
 
-@with_seed(1234)
+@with_seed()
 def test_huber_loss():
 N = 20
 data = mx.random.uniform(-1, 1, shape=(N, 10))

[GitHub] marcoabreu commented on issue #11858: Update contribute.md (Fix links to subscribe for users and contributors)

[GitBox]

marcoabreu commented on issue #11858: Update contribute.md (Fix links to 
subscribe for users and contributors)
URL: https://github.com/apache/incubator-mxnet/pull/11858#issuecomment-413136673
 
 
   Exactly


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] Biserkov opened a new pull request #12180: Make the Ubuntu scripts executable after download.

[GitBox]

Biserkov opened a new pull request #12180: Make the Ubuntu scripts executable 
after download.
URL: https://github.com/apache/incubator-mxnet/pull/12180
 
 
   Just a documentation fix.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on issue #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

larroy commented on issue #12161: [WIP] A solution to prevent zombie containers 
locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#issuecomment-413150063
 
 
   @lebeg addressed your comments.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on a change in pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

larroy commented on a change in pull request #12161: [WIP] A solution to 
prevent zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#discussion_r21017
 
 

 ##
 File path: ci/build.py
 ##
 @@ -37,11 +41,54 @@
 import platform
 from copy import deepcopy
 from itertools import chain
-from subprocess import call, check_call
+from subprocess import call, check_call, check_output
 from typing import *
 from util import *
+try:
+pip_install('docker')
+import docker
+import docker.models
+import docker.errors
+import signal
+import atexit
+except ImportError as e:
+logging.error(e)
+logging.critical("Py docker is required to run: 
https://pypi.org/project/docker/ it can be installed with 'pip3 install 
docker'")
+sys.exit(1)
+
+
+class Cleanup:
+def __init__(self):
+self.containers = set()
+
+def add_container(self, container: docker.models.containers.Container):
+assert isinstance(container, docker.models.containers.Container)
+self.containers.add(container)
+
+def remove_container(self, container: docker.models.containers.Container):
+assert isinstance(container, docker.models.containers.Container)
+self.containers.remove(container)
+
+def _cleanup_containers(self):
+if self.containers:
 
 Review comment:
   ⚡


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on a change in pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

larroy commented on a change in pull request #12161: [WIP] A solution to 
prevent zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#discussion_r210222518
 
 

 ##
 File path: ci/build.py
 ##
 @@ -23,7 +23,11 @@
 """
 
 __author__ = 'Marco de Abreu, Kellen Sunderland, Anton Chernov, Pedro Larroy'
-__version__ = '0.1'
+__version__ = '0.8'
+
+def pip_install(pkg):
 
 Review comment:
   ⚡


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on a change in pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

larroy commented on a change in pull request #12161: [WIP] A solution to 
prevent zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#discussion_r210222498
 
 

 ##
 File path: ci/build.py
 ##
 @@ -37,11 +41,54 @@
 import platform
 from copy import deepcopy
 from itertools import chain
-from subprocess import call, check_call
+from subprocess import call, check_call, check_output
 from typing import *
 from util import *
+try:
+pip_install('docker')
+import docker
+import docker.models
+import docker.errors
+import signal
+import atexit
+except ImportError as e:
 
 Review comment:
   ⚡


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on a change in pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

larroy commented on a change in pull request #12161: [WIP] A solution to 
prevent zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#discussion_r210222868
 
 

 ##
 File path: ci/build.py
 ##
 @@ -180,63 +245,133 @@ def default_ccache_dir() -> str:
 return ccache_dir
 return os.path.join(tempfile.gettempdir(), "ci_ccache")
 
+def trim_container_id(cid):
+return cid[:12]
 
 def container_run(platform: str,
   docker_binary: str,
   docker_registry: str,
   shared_memory_size: str,
-  local_ccache_dir: str,
   command: List[str],
+  local_ccache_dir: str,
+  cleanup: Cleanup,
   dry_run: bool = False,
-  interactive: bool = False) -> str:
+  interactive: bool = False) -> int:
+#
+# Environment setup
+#
+environment = {
+'CCACHE_MAXSIZE': '20G',
+'CCACHE_TEMPDIR': '/tmp/ccache',  # temp dir should be local and not 
shared
+'CCACHE_DIR': '/work/ccache',  # this path is inside the container as 
/work/ccache is mounted
+'CCACHE_LOGFILE': '/tmp/ccache.log',  # a container-scoped log, useful 
for ccache verification.
+}
+# These variables are passed to the container to the process tree killer 
can find runaway process inside the container
+# https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
+# 
https://github.com/jenkinsci/jenkins/blob/578d6bacb33a5e99f149de504c80275796f0b231/core/src/main/java/hudson/model/Run.java#L2393
+#
+JENKINS_ENV_VARS = ['BUILD_NUMBER', 'BUILD_ID', 'BUILD_TAG']
+environment.update({k: os.environ[k] for k in JENKINS_ENV_VARS if k in 
os.environ})
+environment.update({k: os.environ[k] for k in ['CCACHE_MAXSIZE'] if k in 
os.environ})
+
 tag = get_docker_tag(platform=platform, registry=docker_registry)
 mx_root = get_mxnet_root()
 local_build_folder = buildir()
 # We need to create it first, otherwise it will be created by the docker 
daemon with root only permissions
 os.makedirs(local_build_folder, exist_ok=True)
 os.makedirs(local_ccache_dir, exist_ok=True)
-logging.info("Using ccache directory: %s", local_ccache_dir)
-runlist = [docker_binary, 'run', '--rm', '-t',
+local_ccache_dir_sz = check_output(["du", "-hs", 
local_ccache_dir]).decode('utf-8').split('\t')[0]
+logging.info("Using ccache directory: %s (size: %s)", local_ccache_dir, 
local_ccache_dir_sz)
+docker_client = docker.from_env()
+# Equivalent command
+docker_cmd_list = [docker_binary, 'run',
+   '--rm',
'--shm-size={}'.format(shared_memory_size),
'-v', "{}:/work/mxnet".format(mx_root),  # mount mxnet root
'-v', "{}:/work/build".format(local_build_folder),  # mount 
mxnet/build for storing build artifacts
'-v', "{}:/work/ccache".format(local_ccache_dir),
'-u', '{}:{}'.format(os.getuid(), os.getgid()),
-   '-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
+   '-e', 'CCACHE_MAXSIZE=15G',
'-e', 'CCACHE_TEMPDIR=/tmp/ccache',  # temp dir should be local 
and not shared
'-e', "CCACHE_DIR=/work/ccache",  # this path is inside the 
container as /work/ccache is mounted
'-e', "CCACHE_LOGFILE=/tmp/ccache.log",  # a container-scoped 
log, useful for ccache verification.
+   '-ti',
tag]
-runlist.extend(command)
-cmd = ' \\\n\t'.join(runlist)
-ret = 0
-if not dry_run and not interactive:
-logging.info("Running %s in container %s", command, tag)
-logging.info("Executing:\n%s\n", cmd)
-ret = call(runlist)
-
-docker_run_cmd = ' '.join(runlist)
-if not dry_run and interactive:
-into_cmd = deepcopy(runlist)
-# -ti can't be after the tag, as is interpreted as a command so hook 
it up after the -u argument
-idx = into_cmd.index('-u') + 2
-into_cmd[idx:idx] = ['-ti']
-cmd = ' \\\n\t'.join(into_cmd)
-logging.info("Executing:\n%s\n", cmd)
-docker_run_cmd = ' '.join(into_cmd)
-ret = call(into_cmd)
-
-if not dry_run and not interactive and ret != 0:
-logging.error("Running of command in container failed (%s):\n%s\n", 
ret, cmd)
-logging.error("You can get into the container by adding the -i option 
to this script")
-raise subprocess.CalledProcessError(ret, cmd)
-
-return docker_run_cmd
+docker_cmd_list.extend(command)
+docker_cmd = ' \\\n\t'.join(docker_cmd_list)
+logging.info("Running %s in container %s", command, tag)
+logging.info("Executing the equivalent of:\n%s\n", docker_cmd)
+ret = 0 # return code of the command inside docker
+if not dry_run:
+
+
+#
+#
+signal.pthread_sigma

[GitHub] lebeg commented on issue #12102: site-wide social include

[GitBox]

lebeg commented on issue #12102: site-wide social include
URL: https://github.com/apache/incubator-mxnet/pull/12102#issuecomment-413151195
 
 
   I tested on my Mac / Chrome / 34-inch (3440 x 1440) and the social icons 
don't look good. 
   ![screen shot 2018-08-15 at 11 46 
47](https://user-images.githubusercontent.com/1753787/44142902-0d29440a-a082-11e8-9666-7d51a5b60c4a.png)
   I my opinion the icons are distracting a bit. But the main problem is the 
complexity of their layout. Do you think it would make sense to try them on the 
left menu instead? The one that gets hidden on tight mobile sceens.
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg commented on a change in pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

lebeg commented on a change in pull request #12161: [WIP] A solution to prevent 
zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#discussion_r210226128
 
 

 ##
 File path: ci/build.py
 ##
 @@ -37,11 +41,54 @@
 import platform
 from copy import deepcopy
 from itertools import chain
-from subprocess import call, check_call
+from subprocess import call, check_call, check_output
 from typing import *
 from util import *
+try:
+pip_install('docker')
+import docker
+import docker.models
+import docker.errors
+import signal
+import atexit
+except ImportError as e:
+logging.error(e)
+logging.critical("Py docker is required to run: 
https://pypi.org/project/docker/ it can be installed with 'pip3 install 
docker'")
+sys.exit(1)
+
+
+class Cleanup:
+def __init__(self):
+self.containers = set()
+
+def add_container(self, container: docker.models.containers.Container):
+assert isinstance(container, docker.models.containers.Container)
+self.containers.add(container)
+
+def remove_container(self, container: docker.models.containers.Container):
+assert isinstance(container, docker.models.containers.Container)
+self.containers.remove(container)
+
+def _cleanup_containers(self):
+if self.containers:
+logging.warning("Cleaning up containers")
+docker_client = docker.from_env()
+try:
+stop_timeout = int(os.environ.get("DOCKER_STOP_TIMEOUT", 3))
+except Exception as e:
+stop_timeout = 3
+for container in self.containers:
+try:
+container.stop(timeout=stop_timeout)
+logging.warning("☠: %s", trim_container_id(container.id))
 
 Review comment:
   Yes, might be more appropriate


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg commented on a change in pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

lebeg commented on a change in pull request #12161: [WIP] A solution to prevent 
zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#discussion_r210226299
 
 

 ##
 File path: ci/dockerfiles/runtime_functions.sh
 ##
 @@ -305,6 +305,14 @@ build_ubuntu_cpu() {
 build_ubuntu_cpu_openblas
 }
 
+build_ubuntu_base_cpu() {
+echo "skip"
 
 Review comment:
   Ok, I see


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu closed pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

marcoabreu closed pull request #12161: [WIP] A solution to prevent zombie 
containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/ci/README.md b/ci/README.md
index 548e9cb9b04..7338d65aecc 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -1,5 +1,6 @@
 # Containerized build & test utilities
 
+
 This folder contains scripts and dockerfiles used to build and test MXNet using
 Docker containers
 
diff --git a/ci/build.py b/ci/build.py
index 0a1ad4cf575..0f849f45996 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -23,7 +23,7 @@
 """
 
 __author__ = 'Marco de Abreu, Kellen Sunderland, Anton Chernov, Pedro Larroy'
-__version__ = '0.1'
+__version__ = '0.8'
 
 import argparse
 import glob
@@ -37,11 +37,54 @@
 import platform
 from copy import deepcopy
 from itertools import chain
-from subprocess import call, check_call
+from subprocess import call, check_call, check_output
 from typing import *
 from util import *
-
-CCACHE_MAXSIZE = '500G'
+import docker
+import docker.models
+import docker.errors
+import signal
+import atexit
+import pprint as pp
+
+
+class Cleanup:
+def __init__(self):
+self.containers = set()
+self.docker_stop_timeout = 3
+
+def add_container(self, container: docker.models.containers.Container):
+assert isinstance(container, docker.models.containers.Container)
+self.containers.add(container)
+
+def remove_container(self, container: docker.models.containers.Container):
+assert isinstance(container, docker.models.containers.Container)
+self.containers.remove(container)
+
+def _cleanup_containers(self):
+if self.containers:
+logging.warning("Cleaning up containers")
+else:
+return
+docker_client = docker.from_env()
+try:
+stop_timeout = int(os.environ.get("DOCKER_STOP_TIMEOUT", 
self.docker_stop_timeout))
+except Exception as e:
+stop_timeout = 3
+for container in self.containers:
+try:
+container.stop(timeout=stop_timeout)
+logging.info("☠: stopped container %s", 
trim_container_id(container.id))
+container.remove()
+logging.info("🚽: removed container %s", 
trim_container_id(container.id))
+except Exception as e:
+logging.exception(e)
+#pass
+self.containers.clear()
+
+def __call__(self):
+"""Perform cleanup"""
+self._cleanup_containers()
 
 
 
@@ -77,14 +120,30 @@ def f_retry(*args, **kwargs):
 mtries -= 1
 mdelay *= backoff
 return f(*args, **kwargs)
+
 return f_retry  # true decorator
+
 return decorated_retry
 
+
 def under_ci() -> bool:
 """:return: True if we run in Jenkins."""
 return 'JOB_NAME' in os.environ
 
-def get_platforms(path: Optional[str] = "docker"):
+
+def git_cleanup() -> None:
+"""Clean repo and subrepos, update subrepos"""
+logging.info("cleaning up repository")
+with remember_cwd():
+os.chdir(get_mxnet_root())
+check_call(['git', 'clean', '-ffdx'])
+check_call(['git', 'submodule', 'foreach', '--recursive', 'git', 
'clean', '-ffdx'])
+check_call(['git', 'submodule', 'update', '--recursive', '--init'])
+
+def get_dockerfiles_path():
+return "dockerfiles"
+
+def get_platforms(path: Optional[str] = get_dockerfiles_path()):
 """Get a list of architectures given our dockerfiles"""
 dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
 dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
@@ -94,10 +153,11 @@ def get_platforms(path: Optional[str] = "docker"):
 
 
 def get_docker_tag(platform: str, registry: str) -> str:
+""":return: docker tag to be used for the container"""
 return "{0}/build.{1}".format(registry, platform)
 
 
-def get_dockerfile(platform: str, path="docker") -> str:
+def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
 return os.path.join(path, "Dockerfile.build.{0}".format(platform))
 
 
@@ -121,7 +181,7 @@ def build_docker(platform: str, docker_binary: str, 
registry: str, num_retries:
 # container match permissions of the local user. Same for the group.
 #
 # These variables are used in the docker files to create user and group 
with these ids.
-# see: docker/install/ubuntu_adduser.sh
+# see: dockerfiles/install/ubuntu_adduser.sh
 #
 # cache-from is needed so we use the cached images tagged from the remote 
via
 # docker pull see: docker_cache.load_docker_cache
@@ -134,7 +194,7 @@ def build_docker(platform: str, docker_binary:

[GitHub] marcoabreu commented on issue #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

marcoabreu commented on issue #12161: [WIP] A solution to prevent zombie 
containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#issuecomment-413163908
 
 
   See Antons comment about your change deleting the entire shared CCache, for 
example. I'm closing now since there are being new pushes to this PR.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu edited a comment on issue #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

marcoabreu edited a comment on issue #12161: [WIP] A solution to prevent zombie 
containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#issuecomment-413163908
 
 
   See Antons comment about your change deleting the entire shared CCache, for 
example. I'm closing now since there are being new pushes to this PR.
   
   I'd like to have a technical deep dive and an analysis around the tests on 
dev before we execute this on prod.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu edited a comment on issue #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

marcoabreu edited a comment on issue #12161: [WIP] A solution to prevent zombie 
containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#issuecomment-413163908
 
 
   See Antons comment about your change deleting the entire shared CCache, for 
example. I'm closing now since there are being new pushes to this PR.
   
   I'd like to have a technical deep dive, a closer look at the tests and 
assessment of possible side-effects before we execute this on prod. This could 
bring down our CI system and have user impact, so we should be very careful.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu edited a comment on issue #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

marcoabreu edited a comment on issue #12161: [WIP] A solution to prevent zombie 
containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#issuecomment-413163908
 
 
   See Antons comment about your change deleting the entire shared CCache, for 
example. I'm closing now since there are being new commits to this branch.
   
   I'd like to have a technical deep dive, a closer look at the tests and 
assessment of possible side-effects before we execute this on prod. This could 
bring down our CI system and have user impact, so we should be very careful.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg commented on a change in pull request #11148: [MXNET-679] Refactor handling BLAS libraries with cmake

[GitBox]

lebeg commented on a change in pull request #11148: [MXNET-679] Refactor 
handling BLAS libraries with cmake
URL: https://github.com/apache/incubator-mxnet/pull/11148#discussion_r210255985
 
 

 ##
 File path: src/operator/rnn_impl.h
 ##
 @@ -994,7 +998,6 @@ void GruForwardTraining(DType* ws,
   DType* bx_l = bx;
   DType* bh_l = bh;
   DType* y_tmp = x_ptr;
-  unsigned int seed_ = 17 + rand() % 4096;  // NOLINT(runtime/threadsafe_fn)
 
 Review comment:
   Well, the problem is not about the seed itself, but with usage of `rand_r` 
and the fact that it's multithreaded. As far as I can tell in dropout-inl.h#L85 
the seed is set outside of OpenMP threads and no random number gets generated 
with it afterwards, so it should be fine. 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on a change in pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI

[GitBox]

larroy commented on a change in pull request #12161: [WIP] A solution to 
prevent zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161#discussion_r210256630
 
 

 ##
 File path: ci/build.py
 ##
 @@ -180,63 +245,133 @@ def default_ccache_dir() -> str:
 return ccache_dir
 return os.path.join(tempfile.gettempdir(), "ci_ccache")
 
+def trim_container_id(cid):
+return cid[:12]
 
 def container_run(platform: str,
   docker_binary: str,
   docker_registry: str,
   shared_memory_size: str,
-  local_ccache_dir: str,
   command: List[str],
+  local_ccache_dir: str,
+  cleanup: Cleanup,
   dry_run: bool = False,
-  interactive: bool = False) -> str:
+  interactive: bool = False) -> int:
+#
+# Environment setup
+#
+environment = {
+'CCACHE_MAXSIZE': '20G',
 
 Review comment:
   done


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] anirudh2290 commented on a change in pull request #12174: [MXNET-806] Report error when shape mismatch in "where" operator

[GitBox]

anirudh2290 commented on a change in pull request #12174: [MXNET-806] Report 
error when shape mismatch in "where" operator
URL: https://github.com/apache/incubator-mxnet/pull/12174#discussion_r210270908
 
 

 ##
 File path: tests/python/unittest/test_operator.py
 ##
 @@ -4442,6 +4442,18 @@ def test_where_numeric_gradient(shape, same_shape):
 condition_np, x_np, y_np = 
get_forward_inputs_condition_vector(shape)
 check_numeric_gradient(where_sym, [condition_np, x_np, y_np], 
grad_nodes=['x', 'y'])
 
+def test_invalid_shape():
+condition = mx.sym.Variable('condition')
+x = mx.sym.Variable('x')
+y = mx.sym.Variable('y')
+where_sym = mx.sym.where(condition, x, y)
 
 Review comment:
   can you add similar tests for mx.nd.where


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg commented on issue #12160: Remove conflicting llvm OpenMP from cmake builds

[GitBox]

lebeg commented on issue #12160: Remove conflicting llvm OpenMP from cmake 
builds
URL: https://github.com/apache/incubator-mxnet/pull/12160#issuecomment-413201017
 
 
   @szha yes, but for AppleClang we are failing on this 
[line](https://github.com/apache/incubator-mxnet/pull/12160/files#diff-af3b638bc2a3e6c650974192a53c7291R372)
 already, so no difference made in this case.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] wcollins-ebsco commented on issue #8191: RNN BucketSentenceIter - Must load all data into memory?

[GitBox]

wcollins-ebsco commented on issue #8191: RNN BucketSentenceIter - Must load all 
data into memory?
URL: 
https://github.com/apache/incubator-mxnet/issues/8191#issuecomment-413201572
 
 
   Thank you for your follow up.
   I switched to using scikit-learn and tensorflow due to this and many other 
problems I ran into while trying to use mxnet.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] wcollins-ebsco closed issue #8191: RNN BucketSentenceIter - Must load all data into memory?

[GitBox]

wcollins-ebsco closed issue #8191: RNN BucketSentenceIter - Must load all data 
into memory?
URL: https://github.com/apache/incubator-mxnet/issues/8191
 
 
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] marcoabreu commented on issue #12180: Make the Ubuntu scripts executable after download.

[GitBox]

marcoabreu commented on issue #12180: Make the Ubuntu scripts executable after 
download.
URL: https://github.com/apache/incubator-mxnet/pull/12180#issuecomment-413210331
 
 
   Feel free to just change the chmod of the files directly


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg opened a new pull request #12181: Fixed backwards compatibility checker

[GitBox]

lebeg opened a new pull request #12181: Fixed backwards compatibility checker
URL: https://github.com/apache/incubator-mxnet/pull/12181
 
 
   ## Description ##
   
   The corresponding python, python2, pip, pip2 were messed up.
   
   ## Checklist ##
   ### Essentials ###
   - [x] Changes are complete
   
   ### Changes ###
   - [x] Provided a virtualenv for python scripts
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] Oh233 commented on a change in pull request #12178: fix flasky unittest for deformable psroi pooling

[GitBox]

Oh233 commented on a change in pull request #12178: fix flasky unittest for 
deformable psroi pooling
URL: https://github.com/apache/incubator-mxnet/pull/12178#discussion_r210292861
 
 

 ##
 File path: tests/python/unittest/test_operator.py
 ##
 @@ -5010,15 +5067,11 @@ def test_deformable_psroipooling():

sample_per_part=4, group_size=num_group,

pooled_size=num_group, output_dim=num_classes,
trans_std=0.1, 
no_trans=False, name='test_op')
-if grad_nodes[0] == 'offset_data':
-# wider tolerance needed for coordinate differential
-rtol, atol = 1.0, 1e-2
-else:
-rtol, atol = 1e-2, 1e-3
+rtol, atol = 1e-2, 1e-3
 # By now we only have gpu implementation
 if default_context().device_type == 'gpu':
 check_numeric_gradient(op, [im_data, rois_data, 
offset_data], rtol=rtol, atol=atol,
-   grad_nodes=grad_nodes, 
ctx=mx.gpu(0))
+   grad_nodes=grad_nodes, 
ctx=mx.gpu(1))
 
 Review comment:
   Sorry, this is not intentional. I fixed it already.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on a change in pull request #12181: Fixed backwards compatibility checker

[GitBox]

larroy commented on a change in pull request #12181: Fixed backwards 
compatibility checker
URL: https://github.com/apache/incubator-mxnet/pull/12181#discussion_r210293910
 
 

 ##
 File path: ci/docker/runtime_functions.sh
 ##
 @@ -925,13 +925,23 @@ nightly_test_javascript() {
 nightly_model_backwards_compat_test() {
 set -ex
 export PYTHONPATH=/work/mxnet/python/
+
+cd /work/venv
+virtualenv -p python3
+source /work/venv/bin/activate
 
 Review comment:
   If we do a virtualenv why not do pip install instead of exporting PYTHONPATH?
   
   Also, why not relative paths?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg commented on a change in pull request #12181: Fixed backwards compatibility checker

[GitBox]

lebeg commented on a change in pull request #12181: Fixed backwards 
compatibility checker
URL: https://github.com/apache/incubator-mxnet/pull/12181#discussion_r210300434
 
 

 ##
 File path: ci/docker/runtime_functions.sh
 ##
 @@ -925,13 +925,23 @@ nightly_test_javascript() {
 nightly_model_backwards_compat_test() {
 set -ex
 export PYTHONPATH=/work/mxnet/python/
+
+cd /work/venv
+virtualenv -p python3
+source /work/venv/bin/activate
 
 Review comment:
   python, python2, python3 are different now, as their corresponding pip, 
pip2, pip3
   Maybe it would be better to make 2 different virtual env's for python2 and 
python3. Will try that.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] pengzhao-intel commented on issue #12058: MKLDNN can be turned off with env var

[GitBox]

pengzhao-intel commented on issue #12058: MKLDNN can be turned off with env var
URL: https://github.com/apache/incubator-mxnet/pull/12058#issuecomment-413236915
 
 
   Looks fine now even the fallback log is appeared many times :) 
   The good thing is the user would not miss this important info. 
   
   ```
   [patric@mlt-skx084 image-classification]$ export MXNET_MKLDNN_ENABLED=0
   [patric@mlt-skx084 image-classification]$ python benchmark_score.py
   /home/patric/.local/lib/python2.7/site-packages/h5py/__init__.py:34: 
FutureWarning: Conversion of the second argument of issubdtype from `float` to 
`np.floating` is deprecated. In future, it will be treated as `np.float64 == 
np.dtype(float).type`.
 from ._conv import register_converters as _register_converters
   INFO:root:network: alexnet
   INFO:root:device: cpu(0)
   /home/patric/develop/mxnet-mkldnn-env/python/mxnet/module/base_module.py:66: 
UserWarning: Data provided by label_shapes don't match names specified by 
label_names ([] vs. ['softmax_label'])
 warnings.warn(msg)
   [23:26:40] src/executor/../operator/../common/utils.h:447:
   Storage type fallback detected:
   operator = Convolution
   input storage types = [default, default, default, ]
   output storage types = [default, ]
   params = {"num_filter" : 96, "stride" : (4, 4), "kernel" : (11, 11), }
   context.dev_mask = cpu
   The operator with default storage type will be dispatched for execution. 
You're seeing this warning message because the operator above is unable to 
process the given ndarrays with specified storage types, context and parameter. 
Temporary dense ndarrays are generated in order to execute the operator. This 
does not affect the correctness of the programme. You can set environment 
variable MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.
   [23:26:40] src/executor/../operator/../common/utils.h:447:
   Storage type fallback detected:
   operator = Activation
   input storage types = [default, ]
   output storage types = [default, ]
   params = {"act_type" : relu, }
   context.dev_mask = cpu
   The operator with default storage type will be dispatched for execution. 
You're seeing this warning message because the operator above is unable to 
process the given ndarrays with specified storage types, context and parameter. 
Temporary dense ndarrays are generated in order to execute the operator. This 
does not affect the correctness of the programme. You can set environment 
variable MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.
   [23:26:40] src/executor/../operator/../common/utils.h:447:
   Storage type fallback detected:
   operator = LRN
   input storage types = [default, ]
   output storage types = [default, default, ]
   params = {"knorm" : 2, "beta" : 0.75, "nsize" : 5, "alpha" : 0.0001, }
   context.dev_mask = cpu
   The operator with default storage type will be dispatched for execution. 
You're seeing this warning message because the operator above is unable to 
process the given ndarrays with specified storage types, context and parameter. 
Temporary dense ndarrays are generated in order to execute the operator. This 
does not affect the correctness of the programme. You can set environment 
variable MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.
   [23:26:40] src/executor/../operator/../common/utils.h:447:
   Storage type fallback detected:
   operator = Pooling
   input storage types = [default, ]
   output storage types = [default, default, ]
   params = {"stride" : (2, 2), "pool_type" : max, "kernel" : (3, 3), }
   context.dev_mask = cpu
   The operator with default storage type will be dispatched for execution. 
You're seeing this warning message because the operator above is unable to 
process the given ndarrays with specified storage types, context and parameter. 
Temporary dense ndarrays are generated in order to execute the operator. This 
does not affect the correctness of the programme. You can set environment 
variable MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.
   [23:26:40] src/executor/../operator/../common/utils.h:447:
   Storage type fallback detected:
   operator = Convolution
   input storage types = [default, default, default, ]
   output storage types = [default, ]
   params = {"num_filter" : 256, "pad" : (2, 2), "kernel" : (5, 5), }
   context.dev_mask = cpu
   The operator with default storage type will be dispatched for execution. 
You're seeing this warning message because the operator above is unable to 
process the given ndarrays with specified storage types, context and parameter. 
Temporary dense ndarrays are generated in order to execute the operator. This 
does not affect the correctness of the programme. You can set environment 
variable MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.
   [23:26:40] src/executor/../operator/../common/utils.h:447:
   Storage type fallback detected:
   operator = Convolution
   input storage type

[GitHub] lebeg commented on issue #11990: [MXNET-744] Docs build tools update

[GitBox]

lebeg commented on issue #11990: [MXNET-744] Docs build tools update
URL: https://github.com/apache/incubator-mxnet/pull/11990#issuecomment-413246375
 
 
   LGTM


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last batch handle for imageiter

[GitBox]

stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last 
batch handle for imageiter
URL: https://github.com/apache/incubator-mxnet/pull/12131#discussion_r210321441
 
 

 ##
 File path: python/mxnet/image/image.py
 ##
 @@ -1207,8 +1237,28 @@ def next(self):
 except StopIteration:
 if not i:
 raise StopIteration
+return i
 
-return io.DataBatch([batch_data], [batch_label], batch_size - i)
+def next(self):
+"""Returns the next batch of data."""
+batch_size = self.batch_size
+c, h, w = self.data_shape
 
 Review comment:
   Yes we are


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] chsin commented on issue #12176: MXPredReshape bug: need to reshape softmax_label

[GitBox]

chsin commented on issue #12176: MXPredReshape bug: need to reshape 
softmax_label
URL: https://github.com/apache/incubator-mxnet/pull/12176#issuecomment-413247433
 
 
   @chinakook, To predict using the C API, you use `MXPredCreate` to load the 
pretrained symbols and weights and this works fine for [all the mxnet 
pretrained models that use 
SoftmaxOutput](http://data.mxnet.io/models/imagenet/), but the problem is that 
`MXPredReshape` doesn't work even though the models that were loaded properly. 
This causes reshape to fail for all the pretrained mxnet models (almost all use 
SoftmaxOutput) even though predcreate is fine. This is not an acceptable 
situation. I don't quite understand your comment, but you seem to be suggesting 
that this inconsistency is acceptable coding practice and it is the user that 
needs to clean up this mess by writing a script?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg commented on issue #11916: [MXNET-744] Sphinx error reduction

[GitBox]

lebeg commented on issue #11916: [MXNET-744] Sphinx error reduction
URL: https://github.com/apache/incubator-mxnet/pull/11916#issuecomment-413247607
 
 
   Looks good


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy commented on a change in pull request #12167: Add test to check that binded is not set when exception thrown

[GitBox]

sandeep-krishnamurthy commented on a change in pull request #12167: Add test to 
check that binded is not set when exception thrown
URL: https://github.com/apache/incubator-mxnet/pull/12167#discussion_r210322310
 
 

 ##
 File path: tests/python/unittest/test_module.py
 ##
 @@ -44,6 +44,18 @@ def test_module_dtype():
   assert x.dtype == dtype
 
 
+def test_module_bind():
+sym = mx.sym.Variable('data')
+sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC')
+
+mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)])
+assertRaises(TypeError, mod.bind, data_shapes=[('data', 
mx.nd.array([10,10]))])
+mod.binded = False
+
+mod.bind(data_shapes=[('data', (10,10))])
+mod.binded = True
 
 Review comment:
   you intended assert mod.binded == True?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy commented on a change in pull request #12167: Add test to check that binded is not set when exception thrown

[GitBox]

sandeep-krishnamurthy commented on a change in pull request #12167: Add test to 
check that binded is not set when exception thrown
URL: https://github.com/apache/incubator-mxnet/pull/12167#discussion_r210322276
 
 

 ##
 File path: tests/python/unittest/test_module.py
 ##
 @@ -44,6 +44,18 @@ def test_module_dtype():
   assert x.dtype == dtype
 
 
+def test_module_bind():
+sym = mx.sym.Variable('data')
+sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC')
+
+mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)])
+assertRaises(TypeError, mod.bind, data_shapes=[('data', 
mx.nd.array([10,10]))])
+mod.binded = False
 
 Review comment:
   you intended assert mod.binded == False?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] vandanavk opened a new pull request #12182: Remove Epoch training metric log

[GitBox]

vandanavk opened a new pull request #12182: Remove Epoch training metric log
URL: https://github.com/apache/incubator-mxnet/pull/12182
 
 
   ## Description ##
   The log `self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val)` gives 
the user the impression that the metric printed is for the entire epoch, which 
is misleading (Ref https://github.com/apache/incubator-mxnet/pull/10437). This 
log was maintained so that scripts that look for this text, are not broken. But 
this is continuing to cause confusion for users. Therefore, removing this 
particular log.
   
   ## Checklist ##
   ### Essentials ###
   Please feel free to remove inapplicable items for your PR.
   - [ ] The PR title starts with [MXNET-$JIRA_ID], where $JIRA_ID refers to 
the relevant [JIRA issue](https://issues.apache.org/jira/projects/MXNET/issues) 
created (except PRs with tiny changes)
   - [ ] Changes are complete (i.e. I finished coding on this PR)
   - [ ] All changes have test coverage:
   - Unit tests are added for small changes to verify correctness (e.g. adding 
a new operator)
   - Nightly tests are added for complicated/long-running ones (e.g. changing 
distributed kvstore)
   - Build tests will be added for build configuration changes (e.g. adding a 
new build option with NCCL)
   - [ ] Code is well-documented: 
   - For user-facing API changes, API doc string has been updated. 
   - For new C++ functions in header files, their functionalities and arguments 
are documented. 
   - For new examples, README.md is added to explain the what the example does, 
the source of the dataset, expected performance on test set and reference to 
the original paper if applicable
   - Check the API doc at 
http://mxnet-ci-doc.s3-accelerate.dualstack.amazonaws.com/PR-$PR_ID/$BUILD_ID/index.html
   - [ ] To the my best knowledge, examples are either not affected by this 
change, or have been fixed to be compatible with this change
   
   ### Changes ###
   - [ ] Feature1, tests, (and when applicable, API doc)
   - [ ] Feature2, tests, (and when applicable, API doc)
   
   ## Comments ##
   - Risk of breaking scripts that look for this text.
   
   @indhub 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] vdantu commented on issue #12179: Spelling mistake in "mxnet/symbol/image.py" in which "gen_image" is written to be "gen_iamge"

[GitBox]

vdantu commented on issue #12179: Spelling mistake in "mxnet/symbol/image.py" 
in which "gen_image" is written to be "gen_iamge"
URL: 
https://github.com/apache/incubator-mxnet/issues/12179#issuecomment-413251902
 
 
   @PlumedSerpent  Thanks for pointing it out.. 
   @yajiedesign : 
   
https://github.com/apache/incubator-mxnet/blame/master/python/mxnet/symbol/image.py#L22
   Any reason for this? 
   
   @mxnet-label-bot [Python, Bug]


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy closed pull request #12164: Removed fixed seed for test_nadam.

[GitBox]

sandeep-krishnamurthy closed pull request #12164: Removed fixed seed for 
test_nadam.
URL: https://github.com/apache/incubator-mxnet/pull/12164
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/tests/python/unittest/test_optimizer.py 
b/tests/python/unittest/test_optimizer.py
index bdd71eef81a..449cdb42346 100644
--- a/tests/python/unittest/test_optimizer.py
+++ b/tests/python/unittest/test_optimizer.py
@@ -943,7 +943,7 @@ def test_ftrl():
 compare_optimizer(opt1(lazy_update=True, **kwarg), opt2(**kwarg), 
shape,
   np.float32, w_stype='row_sparse', 
g_stype='row_sparse')
 
-@with_seed(1234)
+@with_seed()
 def test_nadam():
 
 def get_net(num_hidden, flatten=True):
@@ -965,10 +965,10 @@ def get_net(num_hidden, flatten=True):
 loss = Loss(output, l)
 loss = mx.sym.make_loss(loss)
 mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
-mod.fit(data_iter, num_epoch=60, optimizer_params={'learning_rate': 
0.0005, 'wd': 0.0005},
+mod.fit(data_iter, num_epoch=60, optimizer_params={'learning_rate': 0.001, 
'wd': 0.0005},
 initializer=mx.init.Xavier(magnitude=2), 
eval_metric=mx.metric.Loss(),
 optimizer='nadam')
-assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1
+assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.11
 
 # AdaGrad
 class PyAdaGrad(mx.optimizer.Optimizer):


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated: Removed fixed seed and increased learning rate and tolerance for test_nadam (#12164)

[skm]

This is an automated email from the ASF dual-hosted git repository.

skm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new 5b9251b  Removed fixed seed and increased learning rate and tolerance 
for test_nadam (#12164)
5b9251b is described below

commit 5b9251baf0666d9c5d0cdc6b208699b70224cd04
Author: access2rohit 
AuthorDate: Wed Aug 15 09:23:32 2018 -0700

Removed fixed seed and increased learning rate and tolerance for test_nadam 
(#12164)
---
 tests/python/unittest/test_optimizer.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/python/unittest/test_optimizer.py 
b/tests/python/unittest/test_optimizer.py
index bdd71ee..449cdb4 100644
--- a/tests/python/unittest/test_optimizer.py
+++ b/tests/python/unittest/test_optimizer.py
@@ -943,7 +943,7 @@ def test_ftrl():
 compare_optimizer(opt1(lazy_update=True, **kwarg), opt2(**kwarg), 
shape,
   np.float32, w_stype='row_sparse', 
g_stype='row_sparse')
 
-@with_seed(1234)
+@with_seed()
 def test_nadam():
 
 def get_net(num_hidden, flatten=True):
@@ -965,10 +965,10 @@ def test_nadam():
 loss = Loss(output, l)
 loss = mx.sym.make_loss(loss)
 mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
-mod.fit(data_iter, num_epoch=60, optimizer_params={'learning_rate': 
0.0005, 'wd': 0.0005},
+mod.fit(data_iter, num_epoch=60, optimizer_params={'learning_rate': 0.001, 
'wd': 0.0005},
 initializer=mx.init.Xavier(magnitude=2), 
eval_metric=mx.metric.Loss(),
 optimizer='nadam')
-assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1
+assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.11
 
 # AdaGrad
 class PyAdaGrad(mx.optimizer.Optimizer):

[GitHub] sandeep-krishnamurthy closed pull request #12153: documentation changes. added full reference

[GitBox]

sandeep-krishnamurthy closed pull request #12153: documentation changes. added 
full reference
URL: https://github.com/apache/incubator-mxnet/pull/12153
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py
index ab7dadb17a5..b69d0c9af0d 100644
--- a/python/mxnet/optimizer.py
+++ b/python/mxnet/optimizer.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,7 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# coding: utf-8
 # pylint: disable=too-many-lines
 """Weight updating functions."""
 import logging
@@ -548,7 +548,7 @@ def update_multi_precision(self, index, weight, grad, 
state):
 
 @register
 class Signum(Optimizer):
-"""The Signum optimizer that takes the sign of gradient or momentum.
+r"""The Signum optimizer that takes the sign of gradient or momentum.
 
 The optimizer updates the weight by::
 
@@ -556,7 +556,11 @@ class Signum(Optimizer):
 state = momentum * state + (1-momentum)*rescaled_grad
 weight = (1 - lr * wd_lh) * weight - lr * sign(state)
 
-See the original paper at: 
https://jeremybernste.in/projects/amazon/signum.pdf
+Reference:
+Jeremy Bernstein, Yu-Xiang Wang, Kamyar Azizzadenesheli & Anima 
Anandkumar. (2018).
+signSGD: Compressed Optimisation for Non-Convex Problems. In ICML'18.
+
+See: https://arxiv.org/abs/1802.04434
 
 For details of the update algorithm see
 :class:`~mxnet.ndarray.signsgd_update` and 
:class:`~mxnet.ndarray.signum_update`.


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated: documentation changes. added full reference (#12153)

[skm]

This is an automated email from the ASF dual-hosted git repository.

skm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new 57313e3  documentation changes. added full reference (#12153)
57313e3 is described below

commit 57313e3bfd52c07c944916076a7ba0b9d9029c58
Author: Yu-Xiang Wang 
AuthorDate: Wed Aug 15 09:24:49 2018 -0700

documentation changes. added full reference (#12153)

* documentation changes. added full reference

* fixing lint

* fixing more lint

* jenkins

* adding the coding line utf-8
---
 python/mxnet/optimizer.py | 10 +++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py
index ab7dadb..b69d0c9 100644
--- a/python/mxnet/optimizer.py
+++ b/python/mxnet/optimizer.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,7 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# coding: utf-8
 # pylint: disable=too-many-lines
 """Weight updating functions."""
 import logging
@@ -548,7 +548,7 @@ class SGD(Optimizer):
 
 @register
 class Signum(Optimizer):
-"""The Signum optimizer that takes the sign of gradient or momentum.
+r"""The Signum optimizer that takes the sign of gradient or momentum.
 
 The optimizer updates the weight by::
 
@@ -556,7 +556,11 @@ class Signum(Optimizer):
 state = momentum * state + (1-momentum)*rescaled_grad
 weight = (1 - lr * wd_lh) * weight - lr * sign(state)
 
-See the original paper at: 
https://jeremybernste.in/projects/amazon/signum.pdf
+Reference:
+Jeremy Bernstein, Yu-Xiang Wang, Kamyar Azizzadenesheli & Anima 
Anandkumar. (2018).
+signSGD: Compressed Optimisation for Non-Convex Problems. In ICML'18.
+
+See: https://arxiv.org/abs/1802.04434
 
 For details of the update algorithm see
 :class:`~mxnet.ndarray.signsgd_update` and 
:class:`~mxnet.ndarray.signum_update`.

[GitHub] stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last batch handle for imageiter

[GitBox]

stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last 
batch handle for imageiter
URL: https://github.com/apache/incubator-mxnet/pull/12131#discussion_r210325874
 
 

 ##
 File path: python/mxnet/image/image.py
 ##
 @@ -1149,22 +1158,44 @@ def __init__(self, batch_size, data_shape, 
label_width=1,
 else:
 self.auglist = aug_list
 self.cur = 0
+self._is_allowed_reading = True
+self._cached_data = None
+# handle the last batch
+if self.seq and last_batch == 'discard':
+new_seq_n = len(self.seq) - len(self.seq) % batch_size
+self.seq = self.seq[:new_seq_n]
 
 Review comment:
   According to 
(NDArrayIter)[https://mxnet.incubator.apache.org/_modules/mxnet/io.html#NDArrayIter],
 it shuffled only during the initialization. But you are right. We should 
shuffle for each epoch.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last batch handle for imageiter

[GitBox]

stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last 
batch handle for imageiter
URL: https://github.com/apache/incubator-mxnet/pull/12131#discussion_r210325874
 
 

 ##
 File path: python/mxnet/image/image.py
 ##
 @@ -1149,22 +1158,44 @@ def __init__(self, batch_size, data_shape, 
label_width=1,
 else:
 self.auglist = aug_list
 self.cur = 0
+self._is_allowed_reading = True
+self._cached_data = None
+# handle the last batch
+if self.seq and last_batch == 'discard':
+new_seq_n = len(self.seq) - len(self.seq) % batch_size
+self.seq = self.seq[:new_seq_n]
 
 Review comment:
   According to 
[NDArrayIter](https://mxnet.incubator.apache.org/_modules/mxnet/io.html#NDArrayIter),
 it shuffled only during the initialization. But you are right. We should 
shuffle for each epoch.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] vdantu commented on issue #12159: C++ api executor->Forward(false) is much slower than MXPredForward?

[GitBox]

vdantu commented on issue #12159: C++ api executor->Forward(false) is much 
slower than MXPredForward?
URL: 
https://github.com/apache/incubator-mxnet/issues/12159#issuecomment-413252620
 
 
   @xiaojingxie : This is a good question. Could you also post this on 
discuss.mxnet.io ? The question will receive a larger audience. 
   
   @mxnet-label-bot [C++, C, Question]


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last batch handle for imageiter

[GitBox]

stu1130 commented on a change in pull request #12131: [MXNET-737][WIP] Add last 
batch handle for imageiter
URL: https://github.com/apache/incubator-mxnet/pull/12131#discussion_r210325874
 
 

 ##
 File path: python/mxnet/image/image.py
 ##
 @@ -1149,22 +1158,44 @@ def __init__(self, batch_size, data_shape, 
label_width=1,
 else:
 self.auglist = aug_list
 self.cur = 0
+self._is_allowed_reading = True
+self._cached_data = None
+# handle the last batch
+if self.seq and last_batch == 'discard':
+new_seq_n = len(self.seq) - len(self.seq) % batch_size
+self.seq = self.seq[:new_seq_n]
 
 Review comment:
   According to 
[NDArrayIter](https://mxnet.incubator.apache.org/_modules/mxnet/io.html#NDArrayIter),
 it shuffle only during the initialization. But you are right. We should 
shuffle for each epoch.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] vandanavk commented on issue #8270: 10 Undefined names in Python code

[GitBox]

vandanavk commented on issue #8270: 10 Undefined names in Python code
URL: 
https://github.com/apache/incubator-mxnet/issues/8270#issuecomment-413254384
 
 
   Thanks @KellenSunderland 
   
   @cclauss I don't see the errors
   ```
   ./tests/python/unittest/test_engine_import.py:33:13: F821 undefined name 
'reload'
   reload(mxnet)
   ^
   ./tests/nightly/model_backwards_compatibility_check/common.py:216:12: F821 
undefined name 'cmp'
   return cmp(normalize(version1), normalize(version2))
   ```
   I've tried flake8 and pylint, Python2 and Python3.
   
   Although I do see 
   ```
   * Module mxnet.ndarray
   python/mxnet/ndarray/__init__.py(34): [E0602 ] Undefined variable 'ndarray'
   * Module mxnet.symbol
   python/mxnet/symbol/__init__.py(31): [E0602 ] Undefined variable 'symbol'
   ```
   After these are fixed, I'll add "undefined-variable" to pylintrc in the CI 
build. This will check future commits to python/mxnet.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy closed pull request #12027: [MXNET-768] Partially enable flaky test for norm operator

[GitBox]

sandeep-krishnamurthy closed pull request #12027: [MXNET-768] Partially enable 
flaky test for norm operator
URL: https://github.com/apache/incubator-mxnet/pull/12027
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/tests/python/unittest/test_operator.py 
b/tests/python/unittest/test_operator.py
index 90e85d123d5..53a726e7867 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -3107,7 +3107,7 @@ def l2norm(input_data, axis=0, keepdims=True):
 for order in [1, 2]:
 for dtype in [np.float16, np.float32, np.float64]:
 in_data = np.random.uniform(-1, 1, in_shape).astype(dtype)
-in_data[abs(in_data) < epsilon] = epsilon
+in_data[abs(in_data) < epsilon] = 2 * epsilon
 for i in range(in_data_dim):
 norm_sym = mx.symbol.norm(data=data, ord=order, axis=i, 
keepdims=True)
 npy_out = l1norm(in_data, i) if order is 1 else 
l2norm(in_data, i)
@@ -3121,20 +3121,22 @@ def l2norm(input_data, axis=0, keepdims=True):
 atol=1e-2 if dtype is np.float16 else 
1e-5, ctx=ctx)
 # Disable numeric gradient 
https://github.com/apache/incubator-mxnet/issues/11509
 # # check gradient
-# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-2, atol=1e-3)
-# if i < in_data_dim-1:
-# norm_sym = mx.symbol.norm(data=data, ord=order, axis=(i, 
i+1), keepdims=True)
-# npy_out = l1norm(in_data, (i, i+1)) if order is 1 else 
l2norm(in_data, (i, i+1))
-# npy_out_backward = np.sign(in_data) if order is 1 else 
in_data/npy_out
-# check_symbolic_forward(norm_sym, [in_data], [npy_out],
-#rtol=1e-2 if dtype is np.float16 
else 1e-5,
-#atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
-# check_symbolic_backward(norm_sym, [in_data], 
[np.ones(npy_out.shape)],
-# [npy_out_backward],
-# rtol=1e-2 if dtype is np.float16 
else 1e-5,
-# atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
-# # check gradient
-# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-2, atol=1e-3)
+# if dtype is not np.float16:
+# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-1, atol=1e-3)
+if i < in_data_dim-1:
+norm_sym = mx.symbol.norm(data=data, ord=order, axis=(i, 
i+1), keepdims=True)
+npy_out = l1norm(in_data, (i, i+1)) if order is 1 else 
l2norm(in_data, (i, i+1))
+npy_out_backward = np.sign(in_data) if order is 1 else 
in_data/npy_out
+check_symbolic_forward(norm_sym, [in_data], [npy_out],
+   rtol=1e-2 if dtype is np.float16 
else 1e-5,
+   atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
+check_symbolic_backward(norm_sym, [in_data], 
[np.ones(npy_out.shape)],
+[npy_out_backward],
+rtol=1e-2 if dtype is np.float16 
else 1e-5,
+atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
+# # check gradient
+# if dtype is not np.float16:
+# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-1, atol=1e-3)
 
 
 def test_layer_norm():


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated: Partially enable flaky test for norm operator (#12027)

[skm]

This is an automated email from the ASF dual-hosted git repository.

skm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new 391377a  Partially enable flaky test for norm operator (#12027)
391377a is described below

commit 391377aff635da0d79f5069ed7689c1ac3ea5c24
Author: Anirudh 
AuthorDate: Wed Aug 15 09:31:31 2018 -0700

Partially enable flaky test for norm operator (#12027)
---
 tests/python/unittest/test_operator.py | 32 +---
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/tests/python/unittest/test_operator.py 
b/tests/python/unittest/test_operator.py
index c395199..66e850f 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -3190,7 +3190,7 @@ def test_norm():
 for order in [1, 2]:
 for dtype in [np.float16, np.float32, np.float64]:
 in_data = np.random.uniform(-1, 1, in_shape).astype(dtype)
-in_data[abs(in_data) < epsilon] = epsilon
+in_data[abs(in_data) < epsilon] = 2 * epsilon
 for i in range(in_data_dim):
 norm_sym = mx.symbol.norm(data=data, ord=order, axis=i, 
keepdims=True)
 npy_out = l1norm(in_data, i) if order is 1 else 
l2norm(in_data, i)
@@ -3204,20 +3204,22 @@ def test_norm():
 atol=1e-2 if dtype is np.float16 else 
1e-5, ctx=ctx)
 # Disable numeric gradient 
https://github.com/apache/incubator-mxnet/issues/11509
 # # check gradient
-# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-2, atol=1e-3)
-# if i < in_data_dim-1:
-# norm_sym = mx.symbol.norm(data=data, ord=order, axis=(i, 
i+1), keepdims=True)
-# npy_out = l1norm(in_data, (i, i+1)) if order is 1 else 
l2norm(in_data, (i, i+1))
-# npy_out_backward = np.sign(in_data) if order is 1 else 
in_data/npy_out
-# check_symbolic_forward(norm_sym, [in_data], [npy_out],
-#rtol=1e-2 if dtype is np.float16 
else 1e-5,
-#atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
-# check_symbolic_backward(norm_sym, [in_data], 
[np.ones(npy_out.shape)],
-# [npy_out_backward],
-# rtol=1e-2 if dtype is np.float16 
else 1e-5,
-# atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
-# # check gradient
-# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-2, atol=1e-3)
+# if dtype is not np.float16:
+# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-1, atol=1e-3)
+if i < in_data_dim-1:
+norm_sym = mx.symbol.norm(data=data, ord=order, axis=(i, 
i+1), keepdims=True)
+npy_out = l1norm(in_data, (i, i+1)) if order is 1 else 
l2norm(in_data, (i, i+1))
+npy_out_backward = np.sign(in_data) if order is 1 else 
in_data/npy_out
+check_symbolic_forward(norm_sym, [in_data], [npy_out],
+   rtol=1e-2 if dtype is np.float16 
else 1e-5,
+   atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
+check_symbolic_backward(norm_sym, [in_data], 
[np.ones(npy_out.shape)],
+[npy_out_backward],
+rtol=1e-2 if dtype is np.float16 
else 1e-5,
+atol=1e-2 if dtype is np.float16 
else 1e-5, ctx=ctx)
+# # check gradient
+# if dtype is not np.float16:
+# check_numeric_gradient(norm_sym, [in_data], 
numeric_eps=epsilon, rtol=1e-1, atol=1e-3)
 
 
 def test_layer_norm():

[GitHub] sandeep-krishnamurthy commented on issue #12007: Add tutorial Gotchas using NumPy

[GitBox]

sandeep-krishnamurthy commented on issue #12007: Add tutorial Gotchas using 
NumPy
URL: https://github.com/apache/incubator-mxnet/pull/12007#issuecomment-413254616
 
 
   @aaronmarkham - ping


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy closed pull request #11918: Improve gather_nd and scatter_nd doc

[GitBox]

sandeep-krishnamurthy closed pull request #11918: Improve gather_nd and 
scatter_nd doc
URL: https://github.com/apache/incubator-mxnet/pull/11918
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/src/operator/tensor/indexing_op.cc 
b/src/operator/tensor/indexing_op.cc
index ef59145bb4a..b663ef0179d 100644
--- a/src/operator/tensor/indexing_op.cc
+++ b/src/operator/tensor/indexing_op.cc
@@ -570,6 +570,10 @@ Examples::
   indices = [[1, 1, 0], [0, 1, 0]]
   gather_nd(data, indices) = [2, 3, 0]
 
+  data = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
+  indices = [[0, 1], [1, 0]]
+  gather_nd(data, indices) = [[3, 4], [5, 6]]
+
 )code")
 .set_num_outputs(1)
 .set_num_inputs(2)
@@ -629,6 +633,21 @@ Examples::
   shape = (2, 2)
   scatter_nd(data, indices, shape) = [[0, 0], [2, 3]]
 
+  data = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
+  indices = [[0, 1], [1, 1]]
+  shape = (2, 2, 2, 2)
+  scatter_nd(data, indices, shape) = 0, 0],
+[0, 0]],
+
+   [[1, 2],
+[3, 4]]],
+
+  [[[0, 0],
+[0, 0]],
+
+   [[5, 6],
+[7, 8
+
 )code")
 .set_num_outputs(1)
 .set_num_inputs(2)


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated: add examples for slicing option (#11918)

[skm]

This is an automated email from the ASF dual-hosted git repository.

skm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new 8e60586  add examples for slicing option (#11918)
8e60586 is described below

commit 8e60586eba91de731b85ca8794aacaf01e568f4e
Author: Hao Jin 
AuthorDate: Wed Aug 15 09:35:52 2018 -0700

add examples for slicing option (#11918)
---
 src/operator/tensor/indexing_op.cc | 19 +++
 1 file changed, 19 insertions(+)

diff --git a/src/operator/tensor/indexing_op.cc 
b/src/operator/tensor/indexing_op.cc
index ef59145..b663ef0 100644
--- a/src/operator/tensor/indexing_op.cc
+++ b/src/operator/tensor/indexing_op.cc
@@ -570,6 +570,10 @@ Examples::
   indices = [[1, 1, 0], [0, 1, 0]]
   gather_nd(data, indices) = [2, 3, 0]
 
+  data = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
+  indices = [[0, 1], [1, 0]]
+  gather_nd(data, indices) = [[3, 4], [5, 6]]
+
 )code")
 .set_num_outputs(1)
 .set_num_inputs(2)
@@ -629,6 +633,21 @@ Examples::
   shape = (2, 2)
   scatter_nd(data, indices, shape) = [[0, 0], [2, 3]]
 
+  data = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
+  indices = [[0, 1], [1, 1]]
+  shape = (2, 2, 2, 2)
+  scatter_nd(data, indices, shape) = 0, 0],
+[0, 0]],
+
+   [[1, 2],
+[3, 4]]],
+
+  [[[0, 0],
+[0, 0]],
+
+   [[5, 6],
+[7, 8
+
 )code")
 .set_num_outputs(1)
 .set_num_inputs(2)

[GitHub] sandeep-krishnamurthy commented on issue #11867: we need to update the doc of scatter_nd and gather_nd

[GitBox]

sandeep-krishnamurthy commented on issue #11867: we need to update the doc of 
scatter_nd and gather_nd
URL: 
https://github.com/apache/incubator-mxnet/issues/11867#issuecomment-413255885
 
 
   Resolved by @haojin2 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy closed issue #11867: we need to update the doc of scatter_nd and gather_nd

[GitBox]

sandeep-krishnamurthy closed issue #11867: we need to update the doc of 
scatter_nd and gather_nd
URL: https://github.com/apache/incubator-mxnet/issues/11867
 
 
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy closed issue #11736: test_optimizer.test_nadam has fixed seed that can mask flakiness

[GitBox]

sandeep-krishnamurthy closed issue #11736: test_optimizer.test_nadam has fixed 
seed that can mask flakiness
URL: https://github.com/apache/incubator-mxnet/issues/11736
 
 
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] zheng-da opened a new issue #11867: we need to update the doc of scatter_nd and gather_nd

[GitBox]

zheng-da opened a new issue #11867: we need to update the doc of scatter_nd and 
gather_nd
URL: https://github.com/apache/incubator-mxnet/issues/11867
 
 
   
[scatter_nd](http://mxnet.incubator.apache.org/api/python/symbol/symbol.html#mxnet.symbol.scatter_nd)
 and 
[gather_nd](http://mxnet.incubator.apache.org/api/python/symbol/symbol.html#mxnet.symbol.gather_nd)
 in MXNet are similar to the ones in TensorFlow. But the current documentation 
is difficult to understand. We need to update the document.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy commented on issue #11867: we need to update the doc of scatter_nd and gather_nd

[GitBox]

sandeep-krishnamurthy commented on issue #11867: we need to update the doc of 
scatter_nd and gather_nd
URL: 
https://github.com/apache/incubator-mxnet/issues/11867#issuecomment-413257383
 
 
   Reopening as we have few more pending updates - @haojin2 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] piyushghai commented on a change in pull request #12181: Fixed backwards compatibility checker

[GitBox]

piyushghai commented on a change in pull request #12181: Fixed backwards 
compatibility checker
URL: https://github.com/apache/incubator-mxnet/pull/12181#discussion_r210331179
 
 

 ##
 File path: ci/docker/runtime_functions.sh
 ##
 @@ -925,13 +925,23 @@ nightly_test_javascript() {
 nightly_model_backwards_compat_test() {
 set -ex
 export PYTHONPATH=/work/mxnet/python/
+
+cd /work/venv
+virtualenv -p python3
+source /work/venv/bin/activate
 
 Review comment:
   Why do we want to go for a virtual env here ? 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated (8e60586 -> da2ce8f)

[nswamy]

This is an automated email from the ASF dual-hosted git repository.

nswamy pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git.


from 8e60586  add examples for slicing option (#11918)
 add da2ce8f  Module predict API can accept NDArray as input (#12166)

No new revisions were added by this update.

Summary of changes:
 python/mxnet/module/base_module.py   | 14 --
 tests/python/unittest/test_module.py | 14 ++
 2 files changed, 26 insertions(+), 2 deletions(-)

[GitHub] nswamy closed pull request #12166: Module predict API can accept NDArray as input

[GitBox]

nswamy closed pull request #12166: Module predict API can accept NDArray as 
input
URL: https://github.com/apache/incubator-mxnet/pull/12166
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/python/mxnet/module/base_module.py 
b/python/mxnet/module/base_module.py
index 654e41bf365..08ab8fa89e4 100644
--- a/python/mxnet/module/base_module.py
+++ b/python/mxnet/module/base_module.py
@@ -22,6 +22,7 @@
 import time
 import logging
 import warnings
+import numpy as np
 
 from .. import metric
 from .. import ndarray
@@ -29,7 +30,7 @@
 from ..context import cpu
 from ..model import BatchEndParam
 from ..initializer import Uniform
-from ..io import DataDesc
+from ..io import DataDesc, DataIter, DataBatch
 from ..base import _as_list
 
 
@@ -333,7 +334,7 @@ def predict(self, eval_data, num_batch=None, 
merge_batches=True, reset=True,
 
 Parameters
 --
-eval_data : DataIter
+eval_data : DataIter or NDArray or numpy array
 Evaluation data to run prediction on.
 num_batch : int
 Defaults to ``None``, indicates running all the batches in the 
data iterator.
@@ -363,6 +364,15 @@ def predict(self, eval_data, num_batch=None, 
merge_batches=True, reset=True,
 """
 assert self.binded and self.params_initialized
 
+if isinstance(eval_data, (ndarray.NDArray, np.ndarray)):
+if isinstance(eval_data, np.ndarray):
+eval_data = ndarray.array(eval_data)
+self.forward(DataBatch([eval_data]))
+return self.get_outputs()[0]
+
+if not isinstance(eval_data, DataIter):
+raise ValueError('eval_data must be of type NDArray or DataIter')
+
 if reset:
 eval_data.reset()
 
diff --git a/tests/python/unittest/test_module.py 
b/tests/python/unittest/test_module.py
index a21527a5a4a..5e60989489f 100644
--- a/tests/python/unittest/test_module.py
+++ b/tests/python/unittest/test_module.py
@@ -772,6 +772,8 @@ def test_forward_reshape():
  for_training=False, force_rebind=True)
 assert mod.predict(pred_dataiter).shape == tuple([10, num_class])
 
+@with_seed()
+def test_forward_types():
 #Test forward with other data batch API
 Batch = namedtuple('Batch', ['data'])
 data = mx.sym.Variable('data')
@@ -786,6 +788,18 @@ def test_forward_reshape():
 mod.forward(Batch(data2))
 assert mod.get_outputs()[0].shape == (3, 5)
 
+#Test forward with other NDArray and np.ndarray inputs
+data = mx.sym.Variable('data')
+out = data * 2
+mod = mx.mod.Module(symbol=out, label_names=None)
+mod.bind(data_shapes=[('data', (1, 10))])
+mod.init_params()
+data1 = mx.nd.ones((1, 10))
+assert mod.predict(data1).shape == (1, 10)
+data2 = np.ones((1, 10))
+assert mod.predict(data1).shape == (1, 10)
+
+
 
 if __name__ == '__main__':
 import nose


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] cclauss commented on issue #8270: 10 Undefined names in Python code

[GitBox]

cclauss commented on issue #8270: 10 Undefined names in Python code
URL: 
https://github.com/apache/incubator-mxnet/issues/8270#issuecomment-413258613
 
 
   * reload(): https://docs.python.org/3/whatsnew/3.0.html#builtins
   * Was moved to 
https://docs.python.org/3/library/importlib.html#importlib.reload
   * cmp(): https://docs.python.org/3/whatsnew/3.0.html#ordering-comparisons
   * http://python-future.org/compatible_idioms.html#cmp has a formula for 
building your own __cmp()__


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated: [MXNET-744] Docs build tools update (#11990)

[nswamy]

This is an automated email from the ASF dual-hosted git repository.

nswamy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new c220974  [MXNET-744] Docs build tools update (#11990)
c220974 is described below

commit c220974f3de322d88dafb66eadd3f26ba3e07232
Author: Aaron Markham 
AuthorDate: Wed Aug 15 09:49:01 2018 -0700

[MXNET-744] Docs build tools update (#11990)

[MXNET-744] Docs build tools update (#11990)
---
 ci/docker/runtime_functions.sh   |   11 +-
 docs/Jenkinsfile |2 +-
 docs/Makefile|2 +
 docs/build_version_doc/artifacts/mxnet.css   | 1479 ++
 docs/build_version_doc/build_all_version.sh  |  116 +-
 docs/build_version_doc/update_all_version.sh |   31 +-
 docs/conf.py |8 +-
 docs/mxdoc.py|   45 +-
 docs/settings.ini|   38 +
 9 files changed, 1673 insertions(+), 59 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 24f844f..e4aac8b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -837,11 +837,14 @@ build_docs() {
 pushd .
 cd /work/mxnet/docs/build_version_doc
 # Parameters are set in the Jenkins pipeline: restricted-website-build
-# $1 is the list of branches to build; $2 is the list of tags to display
+# $1: the list of branches/tags to build
+# $2: the list of tags to display
 # So you can build from the 1.2.0 branch, but display 1.2.1 on the site
-./build_all_version.sh $1 $2
-# $3 is the default version tag for the website; $4 is the base URL
-./update_all_version.sh $2 $3 $4
+# $3: the fork URL
+./build_all_version.sh $1 $2 $3
+# $4: the default version tag for the website
+# $5: the base URL
+./update_all_version.sh $2 $4 $5
 cd VersionedWeb
 tar -zcvf ../artifacts.tgz .
 popd
diff --git a/docs/Jenkinsfile b/docs/Jenkinsfile
index 0e1dce2..9c7b857 100644
--- a/docs/Jenkinsfile
+++ b/docs/Jenkinsfile
@@ -37,7 +37,7 @@ core_logic: {
   ws('workspace/docs') {
 utils.init_git()
 timeout(time: max_time, unit: 'MINUTES') {
-sh "ci/build.py -p ubuntu_cpu --docker-registry 
${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 
/work/runtime_functions.sh build_docs ${params.tags_to_build} 
${params.tag_list} ${params.tag_default} ${params.domain}"
+sh "ci/build.py -p ubuntu_cpu --docker-registry 
${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 
/work/runtime_functions.sh build_docs ${params.tags_to_build} 
${params.tag_list} ${params.fork} ${params.tag_default} ${params.domain}"
 archiveArtifacts 'docs/build_version_doc/artifacts.tgz'
 build 'restricted-website-publish'
 }
diff --git a/docs/Makefile b/docs/Makefile
index 4673d0e..90603b0 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -72,6 +72,8 @@ livehtml:
sphinx-autobuild --ignore "web-data/*" -b html $(ALLSPHINXOPTS) 
$(BUILDDIR)/html
 
 html:
+   export BUILD_VER=$(BUILD_VER)
+   @echo "Env var set for BUILD_VER: $(BUILD_VER)"
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
diff --git a/docs/build_version_doc/artifacts/mxnet.css 
b/docs/build_version_doc/artifacts/mxnet.css
new file mode 100644
index 000..cb77196
--- /dev/null
+++ b/docs/build_version_doc/artifacts/mxnet.css
@@ -0,0 +1,1479 @@
+/* AmazonEmber font ---*/
+@font-face {
+font-family: AmazonEmber;
+src: 
url(https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/font/AmazonEmber_Rg.ttf);
+font-weight: normal;
+}
+
+@font-face {
+font-family: AmazonEmber;
+src: 
url(https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/font/AmazonEmber_Bd.ttf);
+font-weight: bold;
+}
+
+@font-face {
+font-family: AmazonEmber;
+src: 
url(https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/font/AmazonEmber_Th.ttf);
+font-weight: 300;
+}
+
+
+/* basic html elements and classes */
+html, body {
+margin: 0;
+padding: 0;
+background-color: #ff;
+}
+
+body {
+display: block;
+visibility: hidden;
+background-size: cover;
+background-attachment: fixed;
+}
+
+body, div {
+font-family: AmazonEmber;
+font-size: 16px;
+color: #000;
+}
+
+p {
+font-family: AmazonEmber;
+color: #000;
+font-size: 16px;
+line-height: 1.5em;
+}
+
+#language-filter {
+float: right;
+margin-top: 5px;
+}
+
+li, dt a, dt span {
+font-size: 16px;
+line-height: 1.5em;
+}
+
+.footer {
+z-index: 10;
+position: relativ

[GitHub] nswamy closed pull request #11990: [MXNET-744] Docs build tools update

[GitBox]

nswamy closed pull request #11990: [MXNET-744] Docs build tools update
URL: https://github.com/apache/incubator-mxnet/pull/11990
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 1c861beb916..0d01d4fbde7 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -788,11 +788,14 @@ build_docs() {
 pushd .
 cd /work/mxnet/docs/build_version_doc
 # Parameters are set in the Jenkins pipeline: restricted-website-build
-# $1 is the list of branches to build; $2 is the list of tags to display
+# $1: the list of branches/tags to build
+# $2: the list of tags to display
 # So you can build from the 1.2.0 branch, but display 1.2.1 on the site
-./build_all_version.sh $1 $2
-# $3 is the default version tag for the website; $4 is the base URL
-./update_all_version.sh $2 $3 $4
+# $3: the fork URL
+./build_all_version.sh $1 $2 $3
+# $4: the default version tag for the website
+# $5: the base URL
+./update_all_version.sh $2 $4 $5
 cd VersionedWeb
 tar -zcvf ../artifacts.tgz .
 popd
diff --git a/docs/Jenkinsfile b/docs/Jenkinsfile
index ef0755faac7..cd0bcd058cd 100644
--- a/docs/Jenkinsfile
+++ b/docs/Jenkinsfile
@@ -51,7 +51,7 @@ try {
   ws('workspace/docs') {
 init_git()
 timeout(time: max_time, unit: 'MINUTES') {
-sh "ci/build.py -p ubuntu_cpu --docker-registry 
${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 
/work/runtime_functions.sh build_docs ${params.tags_to_build} 
${params.tag_list} ${params.tag_default} ${params.domain}"
+sh "ci/build.py -p ubuntu_cpu --docker-registry 
${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 
/work/runtime_functions.sh build_docs ${params.tags_to_build} 
${params.tag_list} ${params.fork} ${params.tag_default} ${params.domain}"
 archiveArtifacts 'docs/build_version_doc/artifacts.tgz'
 build 'restricted-website-publish'
 }
diff --git a/docs/Makefile b/docs/Makefile
index 4673d0ed1b5..90603b0875f 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -72,6 +72,8 @@ livehtml:
sphinx-autobuild --ignore "web-data/*" -b html $(ALLSPHINXOPTS) 
$(BUILDDIR)/html
 
 html:
+   export BUILD_VER=$(BUILD_VER)
+   @echo "Env var set for BUILD_VER: $(BUILD_VER)"
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
diff --git a/docs/build_version_doc/artifacts/mxnet.css 
b/docs/build_version_doc/artifacts/mxnet.css
new file mode 100644
index 000..cb771967b62
--- /dev/null
+++ b/docs/build_version_doc/artifacts/mxnet.css
@@ -0,0 +1,1479 @@
+/* AmazonEmber font ---*/
+@font-face {
+font-family: AmazonEmber;
+src: 
url(https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/font/AmazonEmber_Rg.ttf);
+font-weight: normal;
+}
+
+@font-face {
+font-family: AmazonEmber;
+src: 
url(https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/font/AmazonEmber_Bd.ttf);
+font-weight: bold;
+}
+
+@font-face {
+font-family: AmazonEmber;
+src: 
url(https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/font/AmazonEmber_Th.ttf);
+font-weight: 300;
+}
+
+
+/* basic html elements and classes */
+html, body {
+margin: 0;
+padding: 0;
+background-color: #ff;
+}
+
+body {
+display: block;
+visibility: hidden;
+background-size: cover;
+background-attachment: fixed;
+}
+
+body, div {
+font-family: AmazonEmber;
+font-size: 16px;
+color: #000;
+}
+
+p {
+font-family: AmazonEmber;
+color: #000;
+font-size: 16px;
+line-height: 1.5em;
+}
+
+#language-filter {
+float: right;
+margin-top: 5px;
+}
+
+li, dt a, dt span {
+font-size: 16px;
+line-height: 1.5em;
+}
+
+.footer {
+z-index: 10;
+position: relative;
+bottom: 0;
+width: 100%;
+}
+
+/* /\*Content paragraph space*\/ */
+/* div.content p { */
+/* margin-top: 20px; */
+/* } */
+
+/*-- Headings ---*/
+h1, h2, h3, h4 {
+font-family: AmazonEmber;
+margin-top: 18px;
+color: #444;
+}
+
+h1 {
+margin-top: 5px;
+margin-bottom: 25px;
+font-size: 28px;
+font-weight: bold;
+}
+
+h2 {
+font-size: 24px;
+font-weight: bold;
+margin-bottom: 15px;
+}
+
+h3 {
+font-size: 20px;
+font-weight: bold;
+margin-bottom: 10px;
+}
+
+h4 {
+font-size: 16px;
+font-weight: bold;
+margin-bottom: 10px;
+}
+
+img {
+border-style: hidden;
+border-width: 0px 0px 0px 0px;
+bo

[GitHub] ankkhedia commented on issue #12178: fix flasky unittest for deformable psroi pooling

[GitBox]

ankkhedia commented on issue #12178: fix flasky unittest for deformable psroi 
pooling
URL: https://github.com/apache/incubator-mxnet/pull/12178#issuecomment-413260221
 
 
   The changes looks good to me and have verified it locally for 2500 runs with 
random seeds.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] cclauss commented on issue #8270: 10 Undefined names in Python code

[GitBox]

cclauss commented on issue #8270: 10 Undefined names in Python code
URL: 
https://github.com/apache/incubator-mxnet/issues/8270#issuecomment-413260434
 
 
   __mxnet__ --> __mxnet.ndarray__


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] cclauss edited a comment on issue #8270: 10 Undefined names in Python code

[GitBox]

cclauss edited a comment on issue #8270: 10 Undefined names in Python code
URL: 
https://github.com/apache/incubator-mxnet/issues/8270#issuecomment-413260434
 
 
   __ndarray__ --> __mxnet.ndarray__


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] cclauss removed a comment on issue #8270: 10 Undefined names in Python code

[GitBox]

cclauss removed a comment on issue #8270: 10 Undefined names in Python code
URL: 
https://github.com/apache/incubator-mxnet/issues/8270#issuecomment-413260434
 
 
   __ndarray__ --> __mxnet.ndarray__


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sad- closed pull request #11858: Update contribute.md (Fix links to subscribe for users and contributors)

[GitBox]

sad- closed pull request #11858: Update contribute.md (Fix links to subscribe 
for users and contributors)
URL: https://github.com/apache/incubator-mxnet/pull/11858
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/docs/community/contribute.md b/docs/community/contribute.md
index f278425b2fc..74e695be3ba 100644
--- a/docs/community/contribute.md
+++ b/docs/community/contribute.md
@@ -45,10 +45,10 @@ Please join either or both of the MXNet mailing lists:
 
 **For MXNet Users, join the USER mailing list**:
 
-- [MXNet Apache USER mailing 
list](https://lists.apache.org/list.html?d...@mxnet.apache.org) 
(u...@mxnet.apache.org): To subscribe, send an email to mailto:user-subscr...@mxnet.apache.org";>user-subscr...@mxnet.apache.org
 
+- [MXNet Apache USER mailing 
list](https://lists.apache.org/list.html?u...@mxnet.apache.org) 
(u...@mxnet.apache.org): To subscribe, send an email to mailto:user-subscr...@mxnet.apache.org";>user-subscr...@mxnet.apache.org
 
 
 **For Contributors to MXNet, join the DEV mailing list**:
-- [MXNet Apache USER mailing 
list](https://lists.apache.org/list.html?d...@mxnet.apache.org) 
(u...@mxnet.apache.org): To subscribe, send an email to mailto:dev-subscr...@mxnet.apache.org";>user-subscr...@mxnet.apache.org
 
+- [MXNet Apache DEV mailing 
list](https://lists.apache.org/list.html?d...@mxnet.apache.org) 
(d...@mxnet.apache.org): To subscribe, send an email to mailto:dev-subscr...@mxnet.apache.org";>dev-subscr...@mxnet.apache.org 
 
 
 
 * [archive](https://lists.apache.org/list.html?d...@mxnet.apache.org) 


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sad- opened a new pull request #12183: Patch: Update contribute.md (Fix links to subscribe for users and contributors)

[GitBox]

sad- opened a new pull request #12183: Patch: Update contribute.md (Fix links 
to subscribe for users and contributors)
URL: https://github.com/apache/incubator-mxnet/pull/12183
 
 
   ## Description ##
   This change is to update the contribute.md page to use the right 
references/links for the subscribing to the different mailing lists.
   
   ## Checklist ##
   ### Essentials ###
   Please feel free to remove inapplicable items for your PR.
   - [ ] The PR title starts with [MXNET-$JIRA_ID], where $JIRA_ID refers to 
the relevant [JIRA issue](https://issues.apache.org/jira/projects/MXNET/issues) 
created (except PRs with tiny changes)
   - [ ] Changes are complete (i.e. I finished coding on this PR)
   - [ ] All changes have test coverage:
   - Unit tests are added for small changes to verify correctness (e.g. adding 
a new operator)
   - Nightly tests are added for complicated/long-running ones (e.g. changing 
distributed kvstore)
   - Build tests will be added for build configuration changes (e.g. adding a 
new build option with NCCL)
   - [ ] Code is well-documented: 
   - For user-facing API changes, API doc string has been updated. 
   - For new C++ functions in header files, their functionalities and arguments 
are documented. 
   - For new examples, README.md is added to explain the what the example does, 
the source of the dataset, expected performance on test set and reference to 
the original paper if applicable
   - Check the API doc at 
http://mxnet-ci-doc.s3-accelerate.dualstack.amazonaws.com/PR-$PR_ID/$BUILD_ID/index.html
   - [ ] To the my best knowledge, examples are either not affected by this 
change, or have been fixed to be compatible with this change
   
   ### Changes ###
   - [ ] Feature1, tests, (and when applicable, API doc)
   - [ ] Feature2, tests, (and when applicable, API doc)
   
   ## Comments ##
   - If this change is a backward incompatible change, why must this change be 
made.
   - Interesting edge cases to note here
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sad- commented on issue #11858: Update contribute.md (Fix links to subscribe for users and contributors)

[GitBox]

sad- commented on issue #11858: Update contribute.md (Fix links to subscribe 
for users and contributors)
URL: https://github.com/apache/incubator-mxnet/pull/11858#issuecomment-413261893
 
 
   https://github.com/apache/incubator-mxnet/pull/12183


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] cclauss commented on issue #8270: 10 Undefined names in Python code

[GitBox]

cclauss commented on issue #8270: 10 Undefined names in Python code
URL: 
https://github.com/apache/incubator-mxnet/issues/8270#issuecomment-413264002
 
 
   See PEP8 for reasons why star imports are for the birds...  
   
   On the `python/mxnet/ndarray/__init__.py` issue, you could try adding a line 
`from .ndarray import __all__ as ndarray___all__` and then below replace 
`ndarray.__all__` with `ndarray___all__` .
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] apeforest commented on a change in pull request #12174: [MXNET-806] Report error when shape mismatch in "where" operator

[GitBox]

apeforest commented on a change in pull request #12174: [MXNET-806] Report 
error when shape mismatch in "where" operator
URL: https://github.com/apache/incubator-mxnet/pull/12174#discussion_r210339535
 
 

 ##
 File path: tests/python/unittest/test_operator.py
 ##
 @@ -4442,6 +4442,18 @@ def test_where_numeric_gradient(shape, same_shape):
 condition_np, x_np, y_np = 
get_forward_inputs_condition_vector(shape)
 check_numeric_gradient(where_sym, [condition_np, x_np, y_np], 
grad_nodes=['x', 'y'])
 
+def test_invalid_shape():
+condition = mx.sym.Variable('condition')
+x = mx.sym.Variable('x')
+y = mx.sym.Variable('y')
+where_sym = mx.sym.where(condition, x, y)
 
 Review comment:
   @anirudh2290 Added test for mx.nd.where.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] yzhliu commented on a change in pull request #11844: [MXNET-689] add DataDesc type for the Scala Package

[GitBox]

yzhliu commented on a change in pull request #11844: [MXNET-689] add DataDesc 
type for the Scala Package
URL: https://github.com/apache/incubator-mxnet/pull/11844#discussion_r210342202
 
 

 ##
 File path: scala-package/core/src/main/scala/org/apache/mxnet/IO.scala
 ##
 @@ -352,9 +399,19 @@ object DataDesc {
* for each data-parallelism device.
*/
   def getBatchAxis(layout: Option[String]): Int = {
-layout.map(_.indexOf('N')).getOrElse(0)
+if (layout.isEmpty|| layout.get == Layout.UNDEFINED) {
+  logger.warn("Found Undefined Layout, will use default index 0")
 
 Review comment:
   will use default index 0 for batch axis


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] yzhliu commented on a change in pull request #11844: [MXNET-689] add DataDesc type for the Scala Package

[GitBox]

yzhliu commented on a change in pull request #11844: [MXNET-689] add DataDesc 
type for the Scala Package
URL: https://github.com/apache/incubator-mxnet/pull/11844#discussion_r210342716
 
 

 ##
 File path: 
scala-package/core/src/main/scala/org/apache/mxnet/io/NDArrayIter.scala
 ##
 @@ -39,35 +40,35 @@ import scala.collection.immutable.ListMap
  * the size of data does not match batch_size. Roll over is intended
  * for training and can cause problems if used for prediction.
  */
-class NDArrayIter(data: IndexedSeq[(String, NDArray)],
-  label: IndexedSeq[(String, NDArray)],
+class NDArrayIter(data: IndexedSeq[(DataDesc, NDArray)],
+  label: IndexedSeq[(DataDesc, NDArray)],
   private val dataBatchSize: Int, shuffle: Boolean,
   lastBatchHandle: String) extends DataIter {
 
   /**
-   * @param data Specify the data. Data names will be data_0, data_1, ..., etc.
-   * @param label Same as data, but is not fed to the model during testing.
-   *  Label names will be label_0, label_1, ..., etc.
-   * @param dataBatchSize Batch Size
-   * @param shuffle Whether to shuffle the data
-   * @param lastBatchHandle "pad", "discard" or "roll_over". How to handle the 
last batch
-   *
-   * This iterator will pad, discard or roll over the last batch if
-   * the size of data does not match batch_size. Roll over is intended
-   * for training and can cause problems if used for prediction.
-   */
+* @param data Specify the data. Data names will be data_0, data_1, ..., 
etc.
+* @param label Same as data, but is not fed to the model during testing.
+*  Label names will be label_0, label_1, ..., etc.
+* @param dataBatchSize Batch Size
+* @param shuffle Whether to shuffle the data
+* @param lastBatchHandle "pad", "discard" or "roll_over". How to handle 
the last batch
+*
+* This iterator will pad, discard or roll over the last batch if
+* the size of data does not match batch_size. Roll over is intended
+* for training and can cause problems if used for prediction.
+*/
 
 Review comment:
   avoid changes like these.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] azai91 commented on a change in pull request #12167: Add test to check that binded is not set when exception thrown

[GitBox]

azai91 commented on a change in pull request #12167: Add test to check that 
binded is not set when exception thrown
URL: https://github.com/apache/incubator-mxnet/pull/12167#discussion_r210344901
 
 

 ##
 File path: tests/python/unittest/test_module.py
 ##
 @@ -44,6 +44,18 @@ def test_module_dtype():
   assert x.dtype == dtype
 
 
+def test_module_bind():
+sym = mx.sym.Variable('data')
+sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC')
+
+mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)])
+assertRaises(TypeError, mod.bind, data_shapes=[('data', 
mx.nd.array([10,10]))])
+mod.binded = False
+
+mod.bind(data_shapes=[('data', (10,10))])
+mod.binded = True
 
 Review comment:
   wow...thanks lol


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy commented on issue #12178: fix flasky unittest for deformable psroi pooling

[GitBox]

sandeep-krishnamurthy commented on issue #12178: fix flasky unittest for 
deformable psroi pooling
URL: https://github.com/apache/incubator-mxnet/pull/12178#issuecomment-413282951
 
 
   @marcoabreu - Your concerns are addressed. This is ready to be merged.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] apeforest commented on issue #12168: Error in Operator implementation guide

[GitBox]

apeforest commented on issue #12168: Error in Operator implementation guide
URL: 
https://github.com/apache/incubator-mxnet/issues/12168#issuecomment-413287070
 
 
   @nswamy Please label [Doc] for this issue. Thanks


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] lebeg commented on a change in pull request #12181: Fixed backwards compatibility checker

[GitBox]

lebeg commented on a change in pull request #12181: Fixed backwards 
compatibility checker
URL: https://github.com/apache/incubator-mxnet/pull/12181#discussion_r210360092
 
 

 ##
 File path: ci/docker/runtime_functions.sh
 ##
 @@ -925,13 +925,23 @@ nightly_test_javascript() {
 nightly_model_backwards_compat_test() {
 set -ex
 export PYTHONPATH=/work/mxnet/python/
+
+cd /work/venv
+virtualenv -p python3
+source /work/venv/bin/activate
 
 Review comment:
   Nevermind. It was just an attempt to insure python and pip are in sync.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] zheng-da opened a new pull request #12184: Fix a bug in CachedOP.

[GitBox]

zheng-da opened a new pull request #12184: Fix a bug in CachedOP.
URL: https://github.com/apache/incubator-mxnet/pull/12184
 
 
   ## Description ##
   As reported in https://github.com/apache/incubator-mxnet/issues/12116, the 
modification in https://github.com/apache/incubator-mxnet/pull/11951 causes 
excessive memory consumption. The reason is that 
https://github.com/apache/incubator-mxnet/pull/11951 actually turns on 
recording even in the inference mode.
   
   ## Checklist ##
   ### Essentials ###
   Please feel free to remove inapplicable items for your PR.
   - [ ] The PR title starts with [MXNET-$JIRA_ID], where $JIRA_ID refers to 
the relevant [JIRA issue](https://issues.apache.org/jira/projects/MXNET/issues) 
created (except PRs with tiny changes)
   - [ ] Changes are complete (i.e. I finished coding on this PR)
   - [ ] All changes have test coverage:
   - Unit tests are added for small changes to verify correctness (e.g. adding 
a new operator)
   - Nightly tests are added for complicated/long-running ones (e.g. changing 
distributed kvstore)
   - Build tests will be added for build configuration changes (e.g. adding a 
new build option with NCCL)
   - [ ] Code is well-documented: 
   - For user-facing API changes, API doc string has been updated. 
   - For new C++ functions in header files, their functionalities and arguments 
are documented. 
   - For new examples, README.md is added to explain the what the example does, 
the source of the dataset, expected performance on test set and reference to 
the original paper if applicable
   - Check the API doc at 
http://mxnet-ci-doc.s3-accelerate.dualstack.amazonaws.com/PR-$PR_ID/$BUILD_ID/index.html
   - [ ] To the my best knowledge, examples are either not affected by this 
change, or have been fixed to be compatible with this change
   
   ### Changes ###
   - [ ] Feature1, tests, (and when applicable, API doc)
   - [ ] Feature2, tests, (and when applicable, API doc)
   
   ## Comments ##
   - If this change is a backward incompatible change, why must this change be 
made.
   - Interesting edge cases to note here
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] zheng-da commented on issue #12184: Fix a bug in CachedOP.

[GitBox]

zheng-da commented on issue #12184: Fix a bug in CachedOP.
URL: https://github.com/apache/incubator-mxnet/pull/12184#issuecomment-413291382
 
 
   @piiswrong 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] azai91 commented on issue #12080: Fix MKLDNNSum cpp test failure

[GitBox]

azai91 commented on issue #12080: Fix MKLDNNSum cpp test failure
URL: https://github.com/apache/incubator-mxnet/pull/12080#issuecomment-413292506
 
 
   looks good to me


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] safrooze commented on a change in pull request #12184: Fix a bug in CachedOP.

[GitBox]

safrooze commented on a change in pull request #12184: Fix a bug in CachedOP.
URL: https://github.com/apache/incubator-mxnet/pull/12184#discussion_r210371600
 
 

 ##
 File path: src/imperative/cached_op.cc
 ##
 @@ -823,9 +823,10 @@ OpStatePtr CachedOp::DynamicForward(
 
   // If we are already recording, we don't need RunGraph to record all
   // computation again.
+  if (recording && !inlining_) recording = false;
   RunGraph(false, idx, arrays, 0, idx.num_nodes(), std::move(array_reqs),
std::move(ref_count), &states, dispatch_modes,
-   !recording || inlining_);
+   recording);
 
 Review comment:
   I recommend putting the recording logic in this line and not changing the 
semantics of `recording` variable in case in the future someone decides to use 
`recording` below `RunGraph` call.
   ```
 RunGraph(false, idx, arrays, 0, idx.num_nodes(), std::move(array_reqs),
  std::move(ref_count), &states, dispatch_modes,
  recording && inlining_);
   ```


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] safrooze commented on a change in pull request #12184: Fix a bug in CachedOP.

[GitBox]

safrooze commented on a change in pull request #12184: Fix a bug in CachedOP.
URL: https://github.com/apache/incubator-mxnet/pull/12184#discussion_r210373938
 
 

 ##
 File path: src/imperative/cached_op.cc
 ##
 @@ -823,9 +823,10 @@ OpStatePtr CachedOp::DynamicForward(
 
   // If we are already recording, we don't need RunGraph to record all
   // computation again.
 
 Review comment:
   I recommend clarifying in the comment that "When CachedOp is inlining, 
RunGraph is in charge of recording the graph, otherwise we are in charge of 
recording and RunGraph shall not be recording".


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] zheng-da commented on a change in pull request #12184: Fix a bug in CachedOP.

[GitBox]

zheng-da commented on a change in pull request #12184: Fix a bug in CachedOP.
URL: https://github.com/apache/incubator-mxnet/pull/12184#discussion_r210377856
 
 

 ##
 File path: src/imperative/cached_op.cc
 ##
 @@ -823,9 +823,10 @@ OpStatePtr CachedOp::DynamicForward(
 
   // If we are already recording, we don't need RunGraph to record all
   // computation again.
+  if (recording && !inlining_) recording = false;
   RunGraph(false, idx, arrays, 0, idx.num_nodes(), std::move(array_reqs),
std::move(ref_count), &states, dispatch_modes,
-   !recording || inlining_);
+   recording);
 
 Review comment:
   i'm trying to follow the logic of the original code. A little logic error 
can cause unexpected problems.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] hcho3 closed pull request #12173: Revert "update dmlc-core (#12129)"

[GitBox]

hcho3 closed pull request #12173: Revert "update dmlc-core (#12129)"
URL: https://github.com/apache/incubator-mxnet/pull/12173
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core
index 958c22b32c1..649be18a8c5 16
--- a/3rdparty/dmlc-core
+++ b/3rdparty/dmlc-core
@@ -1 +1 @@
-Subproject commit 958c22b32c116ec967a9247d09eddb9c21ea6d4f
+Subproject commit 649be18a8c55c48517861d67158a45dec54992ee
diff --git a/tests/python/unittest/test_io.py b/tests/python/unittest/test_io.py
index ddb06f9052b..4dfa69cc105 100644
--- a/tests/python/unittest/test_io.py
+++ b/tests/python/unittest/test_io.py
@@ -309,7 +309,6 @@ def test_DataBatch():
 'DataBatch: data shapes: \[\(2L?, 3L?\), \(7L?, 8L?\)\] label shapes: 
\[\(4L?, 5L?\)\]', str(batch)))
 
 
-@unittest.skip("Broken test: 
https://github.com/apache/incubator-mxnet/issues/12139";)
 def test_CSVIter():
 def check_CSVIter_synthetic(dtype='float32'):
 cwd = os.getcwd()


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] hcho3 commented on issue #12173: Revert "update dmlc-core (#12129)"

[GitBox]

hcho3 commented on issue #12173: Revert "update dmlc-core (#12129)"
URL: https://github.com/apache/incubator-mxnet/pull/12173#issuecomment-413305377
 
 
   Closing this now. I'll submit a new pull request where individual commits 
from dmlc-core are cherry-picked.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] larroy commented on issue #12112: flaky test: test_contrib_control_flow.test_while_loop_for_foreach

[GitBox]

larroy commented on issue #12112: flaky test: 
test_contrib_control_flow.test_while_loop_for_foreach
URL: 
https://github.com/apache/incubator-mxnet/issues/12112#issuecomment-413310541
 
 
   This test is still causing problems:
   
   
http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1421/pipeline


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas using NumPy

[GitBox]

aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas 
using NumPy
URL: https://github.com/apache/incubator-mxnet/pull/12007#discussion_r208977926
 
 

 ##
 File path: docs/tutorials/gluon/gotchas_numpy_in_mxnet.md
 ##
 @@ -0,0 +1,174 @@
+
+# Gotchas using NumPy in Apache MXNet
+
+The goal of this tutorial is to explain some common misconceptions about using 
[NumPy](http://www.numpy.org/) arrays in Apache MXNet. We are going to explain 
why you need to minimize or completely remove usage of NumPy from your Apache 
MXNet code. We also going to show how to minimize NumPy performance impact, 
when you have to use NumPy.
+
+## Asynchronous and non-blocking nature of Apache MXNet
+
+Instead of using NumPy arrays Apache MXNet offers its own array implementation 
named 
[NDArray](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html). 
`NDArray API` was intentionally designed to be similar to `NumPy`, but there 
are differences.
+
+One key difference is in the way calculations are executed. Every `NDArray` 
manipulation in Apache MXNet is done in asynchronous, non-blocking way. That 
means, that when we write code like `c = a * b`, where both `a` and `b` are 
`NDArrays`, the function got pushed to the [Execution 
Engine](https://mxnet.incubator.apache.org/architecture/overview.html#execution-engine),
 which starts the calculation. The function immediately returns back, and the  
user thread can continue execution, despite the fact that the calculation may 
not have been completed yet. 
 
 Review comment:
   nit: got --> was or is


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas using NumPy

[GitBox]

aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas 
using NumPy
URL: https://github.com/apache/incubator-mxnet/pull/12007#discussion_r210387178
 
 

 ##
 File path: docs/tutorials/gluon/gotchas_numpy_in_mxnet.md
 ##
 @@ -0,0 +1,174 @@
+
+# Gotchas using NumPy in Apache MXNet
+
+The goal of this tutorial is to explain some common misconceptions about using 
[NumPy](http://www.numpy.org/) arrays in Apache MXNet. We are going to explain 
why you need to minimize or completely remove usage of NumPy from your Apache 
MXNet code. We also going to show how to minimize NumPy performance impact, 
when you have to use NumPy.
+
+## Asynchronous and non-blocking nature of Apache MXNet
+
+Instead of using NumPy arrays Apache MXNet offers its own array implementation 
named 
[NDArray](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html). 
`NDArray API` was intentionally designed to be similar to `NumPy`, but there 
are differences.
+
+One key difference is in the way calculations are executed. Every `NDArray` 
manipulation in Apache MXNet is done in asynchronous, non-blocking way. That 
means, that when we write code like `c = a * b`, where both `a` and `b` are 
`NDArrays`, the function got pushed to the [Execution 
Engine](https://mxnet.incubator.apache.org/architecture/overview.html#execution-engine),
 which starts the calculation. The function immediately returns back, and the  
user thread can continue execution, despite the fact that the calculation may 
not have been completed yet. 
+
+`Execution Engine` builds computation graph which may reorder or combine some 
calculations, but it honors dependency order: if there are other manipulation 
with `c` done later in the code, the `Execution Engine` will start doing them 
once the result of `c` is available. We don't need to write callbacks to start 
execution of subsequent code - the `Execution Engine` is going to do it for us. 
+
+To get the result of the computation we only need to access the resulting 
variable, and the flow of the code will be blocked until the computation 
results are assigned to the resulting variable. This behavior allows to 
increase code performance while still supporting imperative programming mode. 
+
+Refer to [this 
tutorial](https://mxnet.incubator.apache.org/tutorials/basic/ndarray.html), if 
you are new to Apache MXNet and would like to learn more how to manipulate 
NDArrays.
 
 Review comment:
   I prefer to link with specific information to boost the performance of 
indexing and SEO.  So... Refer to the [intro tutorial to NDArray](#) if you are 
new to MXNet... 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas using NumPy

[GitBox]

aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas 
using NumPy
URL: https://github.com/apache/incubator-mxnet/pull/12007#discussion_r210388467
 
 

 ##
 File path: docs/tutorials/gluon/gotchas_numpy_in_mxnet.md
 ##
 @@ -0,0 +1,174 @@
+
+# Gotchas using NumPy in Apache MXNet
+
+The goal of this tutorial is to explain some common misconceptions about using 
[NumPy](http://www.numpy.org/) arrays in Apache MXNet. We are going to explain 
why you need to minimize or completely remove usage of NumPy from your Apache 
MXNet code. We also going to show how to minimize NumPy performance impact, 
when you have to use NumPy.
+
+## Asynchronous and non-blocking nature of Apache MXNet
+
+Instead of using NumPy arrays Apache MXNet offers its own array implementation 
named 
[NDArray](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html). 
`NDArray API` was intentionally designed to be similar to `NumPy`, but there 
are differences.
+
+One key difference is in the way calculations are executed. Every `NDArray` 
manipulation in Apache MXNet is done in asynchronous, non-blocking way. That 
means, that when we write code like `c = a * b`, where both `a` and `b` are 
`NDArrays`, the function got pushed to the [Execution 
Engine](https://mxnet.incubator.apache.org/architecture/overview.html#execution-engine),
 which starts the calculation. The function immediately returns back, and the  
user thread can continue execution, despite the fact that the calculation may 
not have been completed yet. 
+
+`Execution Engine` builds computation graph which may reorder or combine some 
calculations, but it honors dependency order: if there are other manipulation 
with `c` done later in the code, the `Execution Engine` will start doing them 
once the result of `c` is available. We don't need to write callbacks to start 
execution of subsequent code - the `Execution Engine` is going to do it for us. 
+
+To get the result of the computation we only need to access the resulting 
variable, and the flow of the code will be blocked until the computation 
results are assigned to the resulting variable. This behavior allows to 
increase code performance while still supporting imperative programming mode. 
+
+Refer to [this 
tutorial](https://mxnet.incubator.apache.org/tutorials/basic/ndarray.html), if 
you are new to Apache MXNet and would like to learn more how to manipulate 
NDArrays.
+
+## Converting NDArray to NumPy Array blocks calculation
+
+Many people are familiar with NumPy and flexible doing tensor manipulations 
using it. `NDArray API` offers  a convinient [.asnumpy() 
method](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.asnumpy)
 to cast `nd.array` to `np.array`. However, by doing this cast and using 
`np.array` for calculation, we cannot use all the goodness of `Execution 
Engine`. All manipulations done on `np.array` are blocking. Moreover, the cast 
to `np.array` itself is a blocking operation (same as 
[.asscalar()](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.asscalar),
 
[.wait_to_read()](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.wait_to_read)
 and 
[.waitall()](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.waitall)).
 
+
+That means that if we have a long computation graph and, at some point, we 
want to cast the result to `np.array`, it may feel like the casting takes a lot 
of time. But what really takes this time is `Execution Engine`, which finishes 
all the async calculations we have pushed into it to get the final result, 
which then will be converted to `np.array`.
+
+Because of the blocking nature of [.asnumpy() 
method](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.asnumpy),
 using it reduces the execution performance, especially if the calculations are 
done on GPU: Apache MXNet has to copy data from GPU to CPU to return 
`np.array`. 
+
+The best solution is to **make manipulations directly on NDArrays by methods 
provided in [NDArray 
API](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html)**.
+
+## NumPy operators vs. NDArray operators
+
+Despite the fact that [NDArray 
API](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html) was 
specifically designed to be similar to `NumPy`, sometimes it is not easy to 
replace existing `NumPy` computations. The main reason is that not all 
operators, that are available in `NumPy`, are available in `NDArray API`. [This 
regularly updated page](https://github.com/apache/incubator-mxnet/issues/3199) 
contains the list of `NDArray API`  operators in progress, where:
 
 Review comment:
   Why isn't that info moved over to Confluence?


This is an automated message from the Apache Git Service.
To respond to the message, p

[GitHub] aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas using NumPy

[GitBox]

aaronmarkham commented on a change in pull request #12007: Add tutorial Gotchas 
using NumPy
URL: https://github.com/apache/incubator-mxnet/pull/12007#discussion_r210389424
 
 

 ##
 File path: docs/tutorials/gluon/gotchas_numpy_in_mxnet.md
 ##
 @@ -0,0 +1,174 @@
+
+# Gotchas using NumPy in Apache MXNet
+
+The goal of this tutorial is to explain some common misconceptions about using 
[NumPy](http://www.numpy.org/) arrays in Apache MXNet. We are going to explain 
why you need to minimize or completely remove usage of NumPy from your Apache 
MXNet code. We also going to show how to minimize NumPy performance impact, 
when you have to use NumPy.
+
+## Asynchronous and non-blocking nature of Apache MXNet
+
+Instead of using NumPy arrays Apache MXNet offers its own array implementation 
named 
[NDArray](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html). 
`NDArray API` was intentionally designed to be similar to `NumPy`, but there 
are differences.
+
+One key difference is in the way calculations are executed. Every `NDArray` 
manipulation in Apache MXNet is done in asynchronous, non-blocking way. That 
means, that when we write code like `c = a * b`, where both `a` and `b` are 
`NDArrays`, the function got pushed to the [Execution 
Engine](https://mxnet.incubator.apache.org/architecture/overview.html#execution-engine),
 which starts the calculation. The function immediately returns back, and the  
user thread can continue execution, despite the fact that the calculation may 
not have been completed yet. 
+
+`Execution Engine` builds computation graph which may reorder or combine some 
calculations, but it honors dependency order: if there are other manipulation 
with `c` done later in the code, the `Execution Engine` will start doing them 
once the result of `c` is available. We don't need to write callbacks to start 
execution of subsequent code - the `Execution Engine` is going to do it for us. 
+
+To get the result of the computation we only need to access the resulting 
variable, and the flow of the code will be blocked until the computation 
results are assigned to the resulting variable. This behavior allows to 
increase code performance while still supporting imperative programming mode. 
+
+Refer to [this 
tutorial](https://mxnet.incubator.apache.org/tutorials/basic/ndarray.html), if 
you are new to Apache MXNet and would like to learn more how to manipulate 
NDArrays.
+
+## Converting NDArray to NumPy Array blocks calculation
+
+Many people are familiar with NumPy and flexible doing tensor manipulations 
using it. `NDArray API` offers  a convinient [.asnumpy() 
method](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.asnumpy)
 to cast `nd.array` to `np.array`. However, by doing this cast and using 
`np.array` for calculation, we cannot use all the goodness of `Execution 
Engine`. All manipulations done on `np.array` are blocking. Moreover, the cast 
to `np.array` itself is a blocking operation (same as 
[.asscalar()](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.asscalar),
 
[.wait_to_read()](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.wait_to_read)
 and 
[.waitall()](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.waitall)).
 
+
+That means that if we have a long computation graph and, at some point, we 
want to cast the result to `np.array`, it may feel like the casting takes a lot 
of time. But what really takes this time is `Execution Engine`, which finishes 
all the async calculations we have pushed into it to get the final result, 
which then will be converted to `np.array`.
+
+Because of the blocking nature of [.asnumpy() 
method](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html#mxnet.ndarray.NDArray.asnumpy),
 using it reduces the execution performance, especially if the calculations are 
done on GPU: Apache MXNet has to copy data from GPU to CPU to return 
`np.array`. 
+
+The best solution is to **make manipulations directly on NDArrays by methods 
provided in [NDArray 
API](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html)**.
+
+## NumPy operators vs. NDArray operators
+
+Despite the fact that [NDArray 
API](https://mxnet.incubator.apache.org/api/python/ndarray/ndarray.html) was 
specifically designed to be similar to `NumPy`, sometimes it is not easy to 
replace existing `NumPy` computations. The main reason is that not all 
operators, that are available in `NumPy`, are available in `NDArray API`. [This 
regularly updated page](https://github.com/apache/incubator-mxnet/issues/3199) 
contains the list of `NDArray API`  operators in progress, where:
+
+* v : Already done and it is consistent to NumPy
+* p : Partially done. The part that should be fixed is in the comments
+* x : Does not exist. Needs to be added into Apache MXNet
+* = : Does not exist, and will not b

[GitHub] safrooze commented on a change in pull request #12184: Fix a bug in CachedOP.

[GitBox]

safrooze commented on a change in pull request #12184: Fix a bug in CachedOP.
URL: https://github.com/apache/incubator-mxnet/pull/12184#discussion_r210392084
 
 

 ##
 File path: src/imperative/cached_op.cc
 ##
 @@ -823,9 +823,10 @@ OpStatePtr CachedOp::DynamicForward(
 
   // If we are already recording, we don't need RunGraph to record all
   // computation again.
+  if (recording && !inlining_) recording = false;
   RunGraph(false, idx, arrays, 0, idx.num_nodes(), std::move(array_reqs),
std::move(ref_count), &states, dispatch_modes,
-   !recording || inlining_);
+   recording);
 
 Review comment:
   I find it easier to understand the logic: RunGraph must record only when 
autograd is recording and CachedOp is inlining. An easier to understand logic 
can prevent future confusion. Up to you to change or not.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] safrooze commented on issue #12184: Fix a bug in CachedOP.

[GitBox]

safrooze commented on issue #12184: Fix a bug in CachedOP.
URL: https://github.com/apache/incubator-mxnet/pull/12184#issuecomment-413320292
 
 
   Do we have any unit-test framework to check the growth of the bufferpool 
that we can use to write a unit-test for this fix?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy closed pull request #12137: [MXNET-696] Fix undefined name errors

[GitBox]

sandeep-krishnamurthy closed pull request #12137: [MXNET-696] Fix undefined 
name errors
URL: https://github.com/apache/incubator-mxnet/pull/12137
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/example/deep-embedded-clustering/model.py 
b/example/deep-embedded-clustering/model.py
index 777634e3cf8..9b6185c9fd1 100644
--- a/example/deep-embedded-clustering/model.py
+++ b/example/deep-embedded-clustering/model.py
@@ -22,7 +22,7 @@
 import numpy as np
 try:
 import cPickle as pickle
-except ModuleNotFoundError:
+except ImportError:
 import pickle
 
 
@@ -75,4 +75,4 @@ def load(self, fname):
 self.args[key][:] = v
 
 def setup(self, *args, **kwargs):
-raise NotImplementedError("must override this")
\ No newline at end of file
+raise NotImplementedError("must override this")
diff --git a/example/neural-style/end_to_end/model_vgg19.py 
b/example/neural-style/end_to_end/model_vgg19.py
index 0d369ae08f5..1bc38766beb 100644
--- a/example/neural-style/end_to_end/model_vgg19.py
+++ b/example/neural-style/end_to_end/model_vgg19.py
@@ -90,6 +90,7 @@ def get_executor_with_style(style, content, input_size, ctx):
 arg_dict=arg_dict)
 
 def get_executor_content(content, input_size, ctx):
+out = mx.sym.Group([content])
 arg_shapes, output_shapes, aux_shapes = content.infer_shape(data=(1, 3, 
input_size[0], input_size[1]))
 arg_names = out.list_arguments()
 arg_dict = dict(zip(arg_names, [mx.nd.zeros(shape, ctx=ctx) for shape in 
arg_shapes]))
diff --git a/example/reinforcement-learning/a3c/a3c.py 
b/example/reinforcement-learning/a3c/a3c.py
index f74ce77b652..c100f61304d 100644
--- a/example/reinforcement-learning/a3c/a3c.py
+++ b/example/reinforcement-learning/a3c/a3c.py
@@ -203,7 +203,7 @@ def test():
 mx.gpu(int(i)) for i in args.gpus.split(',')]
 
 # module
-dataiter = robo_data.RobosimsDataIter('scenes', args.batch_size, 
args.input_length, web_viz=True)
+dataiter = rl_data.GymDataIter('scenes', args.batch_size, 
args.input_length, web_viz=True)
 print(dataiter.provide_data)
 net = sym.get_symbol_thor(dataiter.act_dim)
 module = mx.mod.Module(net, data_names=[d[0] for d in 
dataiter.provide_data], label_names=('policy_label', 'value_label'), 
context=devs)
diff --git a/example/sparse/factorization_machine/metric.py 
b/example/sparse/factorization_machine/metric.py
index 05ef04a0c48..a8c52c781c0 100644
--- a/example/sparse/factorization_machine/metric.py
+++ b/example/sparse/factorization_machine/metric.py
@@ -107,7 +107,9 @@ def update(self, labels, preds):
 label_sum = label_weight.sum()
 if label_sum == 0 or label_sum == label_weight.size:
 raise Exception("AUC with one class is undefined")
-
+
+label_one_num = np.count_nonzero(label_weight)
+label_zero_num = len(label_weight) - label_one_num
 total_area = label_zero_num * label_one_num
 height = 0
 width = 0


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] sandeep-krishnamurthy closed pull request #12152: [MXNET-696] Fix profiler executer when memonger is used

[GitBox]

sandeep-krishnamurthy closed pull request #12152: [MXNET-696] Fix profiler 
executer when memonger is used
URL: https://github.com/apache/incubator-mxnet/pull/12152
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/example/profiler/README.md b/example/profiler/README.md
index 7d3c42b629d..1b9279ccf22 100644
--- a/example/profiler/README.md
+++ b/example/profiler/README.md
@@ -5,8 +5,12 @@ Please refer to [this 
link](http://mxnet.incubator.apache.org/faq/perf.html?high
 for visualizing profiling results and make sure that you have installed a 
version of MXNet compiled
 with `USE_PROFILER=1`.
 
-- profiler_executor.py. To run this example, simply type `python 
profiler_executor.py` in terminal.
-It will generate a json file named `profile_executor_5iter.json`.
+- profiler_executor.py. To run this example,
+- clone mxnet-memonger (git clone 
https://github.com/dmlc/mxnet-memonger.git).
+- Add mxnet-memonger folder to PYTHONPATH.
+export PYTHONPATH=$PYTHONPATH:/path/to/mxnet-memonger
+- type python profiler_executor.py in terminal.
+It will generate a json file named `profile_executor_5iter.json`.
 
 - profiler_imageiter.py. You first need to create a file named `test.rec`,
 which is an image dataset file before running this example.
@@ -20,4 +24,4 @@ that you have installed a GPU enabled version of MXNet before 
running this examp
 `python profiler_matmul.py` and it will generate `profile_matmul_20iter.json`.
 
 - profiler_ndarray.py. This examples profiles a series of `NDArray` 
operations. Simply type
-`python profiler_ndarray.py` in terminal and it will generate 
`profile_ndarray.json`.
\ No newline at end of file
+`python profiler_ndarray.py` in terminal and it will generate 
`profile_ndarray.json`.
diff --git a/example/profiler/profiler_executor.py 
b/example/profiler/profiler_executor.py
index 8ab417a9744..91532535bd0 100644
--- a/example/profiler/profiler_executor.py
+++ b/example/profiler/profiler_executor.py
@@ -21,6 +21,7 @@
 import time
 import numpy as np
 from mxnet import profiler
+import memonger
 
 
 def parse_args():
@@ -86,7 +87,8 @@ def get_symbol():
 
 def get_module(ctx, sym, provide_data, provide_label, batch_size=None, 
is_train=True, use_memonger=False):
 if use_memonger:
-sym = search_plan(sym, data=data_shapes)
+name, data_shapes = provide_data[0]
+sym = memonger.search_plan(sym, data=data_shapes)
 mod = mx.mod.Module(symbol=sym,
 data_names=[name for name, _ in provide_data],
 label_names=[name for name, _ in provide_label],


 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[incubator-mxnet] branch master updated: Fix profiler executer when memonger is used (#12152)

[skm]

This is an automated email from the ASF dual-hosted git repository.

skm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new 8f61777  Fix profiler executer when memonger is used (#12152)
8f61777 is described below

commit 8f61777ab241868195e188269f42d855d97a692b
Author: Vandana Kannan 
AuthorDate: Wed Aug 15 13:24:16 2018 -0700

Fix profiler executer when memonger is used (#12152)
---
 example/profiler/README.md| 10 +++---
 example/profiler/profiler_executor.py |  4 +++-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/example/profiler/README.md b/example/profiler/README.md
index 7d3c42b..1b9279c 100644
--- a/example/profiler/README.md
+++ b/example/profiler/README.md
@@ -5,8 +5,12 @@ Please refer to [this 
link](http://mxnet.incubator.apache.org/faq/perf.html?high
 for visualizing profiling results and make sure that you have installed a 
version of MXNet compiled
 with `USE_PROFILER=1`.
 
-- profiler_executor.py. To run this example, simply type `python 
profiler_executor.py` in terminal.
-It will generate a json file named `profile_executor_5iter.json`.
+- profiler_executor.py. To run this example,
+- clone mxnet-memonger (git clone 
https://github.com/dmlc/mxnet-memonger.git).
+- Add mxnet-memonger folder to PYTHONPATH.
+export PYTHONPATH=$PYTHONPATH:/path/to/mxnet-memonger
+- type python profiler_executor.py in terminal.
+It will generate a json file named `profile_executor_5iter.json`.
 
 - profiler_imageiter.py. You first need to create a file named `test.rec`,
 which is an image dataset file before running this example.
@@ -20,4 +24,4 @@ that you have installed a GPU enabled version of MXNet before 
running this examp
 `python profiler_matmul.py` and it will generate `profile_matmul_20iter.json`.
 
 - profiler_ndarray.py. This examples profiles a series of `NDArray` 
operations. Simply type
-`python profiler_ndarray.py` in terminal and it will generate 
`profile_ndarray.json`.
\ No newline at end of file
+`python profiler_ndarray.py` in terminal and it will generate 
`profile_ndarray.json`.
diff --git a/example/profiler/profiler_executor.py 
b/example/profiler/profiler_executor.py
index 8ab417a..9153253 100644
--- a/example/profiler/profiler_executor.py
+++ b/example/profiler/profiler_executor.py
@@ -21,6 +21,7 @@ import os
 import time
 import numpy as np
 from mxnet import profiler
+import memonger
 
 
 def parse_args():
@@ -86,7 +87,8 @@ def get_symbol():
 
 def get_module(ctx, sym, provide_data, provide_label, batch_size=None, 
is_train=True, use_memonger=False):
 if use_memonger:
-sym = search_plan(sym, data=data_shapes)
+name, data_shapes = provide_data[0]
+sym = memonger.search_plan(sym, data=data_shapes)
 mod = mx.mod.Module(symbol=sym,
 data_names=[name for name, _ in provide_data],
 label_names=[name for name, _ in provide_label],

[incubator-mxnet] branch master updated: [MXNET-696] Fix undefined name errors (#12137)

[skm]

This is an automated email from the ASF dual-hosted git repository.

skm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
 new 6b01a4f  [MXNET-696] Fix undefined name errors (#12137)
6b01a4f is described below

commit 6b01a4f0185930cff578a3a0b68354c5a1cb50cd
Author: Vandana Kannan 
AuthorDate: Wed Aug 15 13:23:42 2018 -0700

[MXNET-696] Fix undefined name errors (#12137)

* Fix undefined name error in neural style example

* Fix import exception error

* Fix undefined name in AUCMetric

* Fix undefined name in a3c example
---
 example/deep-embedded-clustering/model.py  | 4 ++--
 example/neural-style/end_to_end/model_vgg19.py | 1 +
 example/reinforcement-learning/a3c/a3c.py  | 2 +-
 example/sparse/factorization_machine/metric.py | 4 +++-
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/example/deep-embedded-clustering/model.py 
b/example/deep-embedded-clustering/model.py
index 777634e..9b6185c 100644
--- a/example/deep-embedded-clustering/model.py
+++ b/example/deep-embedded-clustering/model.py
@@ -22,7 +22,7 @@ import mxnet as mx
 import numpy as np
 try:
 import cPickle as pickle
-except ModuleNotFoundError:
+except ImportError:
 import pickle
 
 
@@ -75,4 +75,4 @@ class MXModel(object):
 self.args[key][:] = v
 
 def setup(self, *args, **kwargs):
-raise NotImplementedError("must override this")
\ No newline at end of file
+raise NotImplementedError("must override this")
diff --git a/example/neural-style/end_to_end/model_vgg19.py 
b/example/neural-style/end_to_end/model_vgg19.py
index 0d369ae..1bc3876 100644
--- a/example/neural-style/end_to_end/model_vgg19.py
+++ b/example/neural-style/end_to_end/model_vgg19.py
@@ -90,6 +90,7 @@ def get_executor_with_style(style, content, input_size, ctx):
 arg_dict=arg_dict)
 
 def get_executor_content(content, input_size, ctx):
+out = mx.sym.Group([content])
 arg_shapes, output_shapes, aux_shapes = content.infer_shape(data=(1, 3, 
input_size[0], input_size[1]))
 arg_names = out.list_arguments()
 arg_dict = dict(zip(arg_names, [mx.nd.zeros(shape, ctx=ctx) for shape in 
arg_shapes]))
diff --git a/example/reinforcement-learning/a3c/a3c.py 
b/example/reinforcement-learning/a3c/a3c.py
index f74ce77..c100f61 100644
--- a/example/reinforcement-learning/a3c/a3c.py
+++ b/example/reinforcement-learning/a3c/a3c.py
@@ -203,7 +203,7 @@ def test():
 mx.gpu(int(i)) for i in args.gpus.split(',')]
 
 # module
-dataiter = robo_data.RobosimsDataIter('scenes', args.batch_size, 
args.input_length, web_viz=True)
+dataiter = rl_data.GymDataIter('scenes', args.batch_size, 
args.input_length, web_viz=True)
 print(dataiter.provide_data)
 net = sym.get_symbol_thor(dataiter.act_dim)
 module = mx.mod.Module(net, data_names=[d[0] for d in 
dataiter.provide_data], label_names=('policy_label', 'value_label'), 
context=devs)
diff --git a/example/sparse/factorization_machine/metric.py 
b/example/sparse/factorization_machine/metric.py
index 05ef04a..a8c52c7 100644
--- a/example/sparse/factorization_machine/metric.py
+++ b/example/sparse/factorization_machine/metric.py
@@ -107,7 +107,9 @@ class AUCMetric(mx.metric.EvalMetric):
 label_sum = label_weight.sum()
 if label_sum == 0 or label_sum == label_weight.size:
 raise Exception("AUC with one class is undefined")
-
+
+label_one_num = np.count_nonzero(label_weight)
+label_zero_num = len(label_weight) - label_one_num
 total_area = label_zero_num * label_one_num
 height = 0
 width = 0