[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196652237
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,185 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_compilation_warning',
false)
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_pip', false)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_docker', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_source', false)
+}
+ }
+},
+'InstallationGuide: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-gpu') {
+ init_git()
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_virtualenv', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_pip', true)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_docker', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_source', true)
+}
+ }
+},
+'PipTest: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-pipTest') {
+ init_git()
+}
+ }
+},
+'Amalgamation-atlas: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-amalgamation1') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation
USE_BLAS=atlas', false)
+}
+
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196651974 ## File path: ci/docker/Dockerfile.build.ubuntu_nightly_gpu ## @@ -0,0 +1,64 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for CPU Review comment: Nit: CPU -> GPU This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196651940 ## File path: ci/docker/Dockerfile.build.ubuntu_base_cpu ## @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. Review comment: Nit: No cuda in this container This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196537084 ## File path: docs/install/index.md ## @@ -84,7 +84,7 @@ $ wget https://bootstrap.pypa.io/get-pip.py && sudo python get-pip.py **Step 2** Install MXNet with OpenBLAS acceleration. ```bash -$ pip install mxnet +$ sudo pip install mxnet Review comment: Ye the nose-discussion is a different one, just wanted it for reference. But I agree - it should not be necessary to run pip as sudo since we don't expect our users to do it either. Fully agree. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301937 ## File path: docs/install/index.md ## @@ -84,7 +84,7 @@ $ wget https://bootstrap.pypa.io/get-pip.py && sudo python get-pip.py **Step 2** Install MXNet with OpenBLAS acceleration. ```bash -$ pip install mxnet +$ sudo pip install mxnet Review comment: @szha @KellenSunderland does this fall under the same sudo-discussion as https://github.com/apache/incubator-mxnet/pull/11317 ? @aaronmarkham @kpmurali is it okay to change the installation instructions to force sudo? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301028 ## File path: ci/docker/Dockerfile.build.ubuntu_base_cpu ## @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +RUN apt-get update && apt-get -y install sudo + +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY install/ubuntu_runas_sudo.sh /work/ +RUN /work/ubuntu_runas_sudo.sh + +ARG USER_ID=os.getuid() Review comment: Please move it up and change it to the same style as in https://github.com/apache/incubator-mxnet/blob/master/ci/docker/Dockerfile.build.ubuntu_cpu#L48 Otherwise, the argument will not be picked up. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301110 ## File path: ci/docker/Dockerfile.build.ubuntu_base_gpu ## @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM nvidia/cuda:9.1-cudnn7-devel + +WORKDIR /work/deps + +RUN apt-get update && apt-get -y install sudo + +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY install/ubuntu_runas_sudo.sh /work/ +RUN /work/ubuntu_runas_sudo.sh + +ARG USER_ID=os.getuid() Review comment: Same here This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301183 ## File path: ci/docker/Dockerfile.build.ubuntu_emscripten ## @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- Review comment: Ping This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301937 ## File path: docs/install/index.md ## @@ -84,7 +84,7 @@ $ wget https://bootstrap.pypa.io/get-pip.py && sudo python get-pip.py **Step 2** Install MXNet with OpenBLAS acceleration. ```bash -$ pip install mxnet +$ sudo pip install mxnet Review comment: @szha @KellenSunderland does this fall under the same sudo-discussion as https://github.com/apache/incubator-mxnet/pull/11317 ? @aaronmarkham is it okay to change the installation instructions to force sudo? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196302161
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,194 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_gpu', 'nightly_test_compilation_warning',
false)
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_pip', false)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_docker', false)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_cpu_source', false)
Review comment:
ping
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301604
##
File path: ci/docker/runtime_functions.sh
##
@@ -774,6 +774,58 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+./tests/nightly/apache_rat_license_check/license_check.sh
+}
+
+#Checks MXNet for Compilation Warnings
+nightly_test_compilation_warning() {
+set -ex
+export PYTHONPATH=./python/
+./tests/nightly/compilation_warnings/compilation_warnings.sh
+}
+
+#Checks the MXNet Installation Guide - currently checks pip, build from source
and virtual env on cpu and gpu
+nightly_test_installation() {
+set -ex
+source ./tests/jenkins/run_test_installation_docs.sh docs/install/index.md
1 1686; ${1}
Review comment:
Could you document these arguments? Right now, I don't understand what they
would do
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301293 ## File path: ci/docker/Dockerfile.build.ubuntu_nightly_gpu ## @@ -0,0 +1,63 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for CPU + +FROM nvidia/cuda:9.1-cudnn7-devel + +WORKDIR /work/deps + +COPY install/ubuntu_core.sh /work/ +RUN /work/ubuntu_core.sh +COPY install/ubuntu_ccache.sh /work/ +RUN /work/ubuntu_ccache.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh +COPY install/ubuntu_r.sh /work/ +RUN /work/ubuntu_r.sh +COPY install/ubuntu_perl.sh /work/ +RUN /work/ubuntu_perl.sh +COPY install/ubuntu_clang.sh /work/ +RUN /work/ubuntu_clang.sh +COPY install/ubuntu_mklml.sh /work/ +RUN /work/ubuntu_mklml.sh +COPY install/ubuntu_tvm.sh /work/ +RUN /work/ubuntu_tvm.sh +COPY install/ubuntu_llvm.sh /work/ +RUN /work/ubuntu_llvm.sh +COPY install/ubuntu_caffe.sh /work/ +RUN /work/ubuntu_caffe.sh +COPY install/ubuntu_onnx.sh /work/ +RUN /work/ubuntu_onnx.sh +COPY install/ubuntu_docs.sh /work/ +RUN /work/ubuntu_docs.sh +COPY install/ubuntu_tutorials.sh /work/ +RUN /work/ubuntu_tutorials.sh +COPY install/ubuntu_nightly_tests.sh /work/ +RUN /work/ubuntu_nightly_tests.sh +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +ARG USER_ID=0 Review comment: Same here, has to be moved up This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196301239 ## File path: ci/docker/Dockerfile.build.ubuntu_nightly_cpu ## @@ -0,0 +1,57 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_core.sh /work/ +RUN /work/ubuntu_core.sh +COPY install/ubuntu_ccache.sh /work/ +RUN /work/ubuntu_ccache.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh +COPY install/ubuntu_r.sh /work/ +RUN /work/ubuntu_r.sh +COPY install/ubuntu_perl.sh /work/ +RUN /work/ubuntu_perl.sh +COPY install/ubuntu_clang.sh /work/ +RUN /work/ubuntu_clang.sh +COPY install/ubuntu_mklml.sh /work/ +RUN /work/ubuntu_mklml.sh +COPY install/ubuntu_caffe.sh /work/ +RUN /work/ubuntu_caffe.sh +COPY install/ubuntu_onnx.sh /work/ +RUN /work/ubuntu_onnx.sh +COPY install/ubuntu_docs.sh /work/ +RUN /work/ubuntu_docs.sh +COPY install/ubuntu_nightly_tests.sh /work/ +RUN /work/ubuntu_nightly_tests.sh +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +ARG USER_ID=0 Review comment: Same here - has to be moved up This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196300756
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,194 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_gpu', 'nightly_test_compilation_warning',
false)
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_pip', false)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_docker', false)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_cpu_source', false)
+}
+ }
+},
+'InstallationGuide: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-gpu') {
+ init_git()
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_virtualenv', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_pip', true)
+ ///Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_docker', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_source', true)
+}
+ }
+},
+'PipTest: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-pipTest') {
+ init_git()
+}
+ }
+},
+'Amalgamation-atlas: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-amalgamation1') {
+ init_git()
+ //sh "ci/build.py --platform ubuntu_nightly_cpu
/work/runtime_functions.sh
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r196300576 ## File path: ci/docker/Dockerfile.build.ubuntu_base_cpu ## @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM ubuntu:16.04 + +ARG USER_ID=os.getuid() Review comment: Right, but this value is set by our bootstrapping script here: https://github.com/apache/incubator-mxnet/blob/master/ci/build.py#L79 You should not override it manually This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195571306 ## File path: ci/docker/Dockerfile.build.ubuntu_emscripten ## @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet JS amalgamation test. Builds on top of the ubuntu_cpu dockerfile. + +FROM ubuntu:16.04 + +ARG USER_ID=0 Review comment: move This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195542314
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,194 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_gpu', 'nightly_test_compilation_warning',
false)
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
Review comment:
Why does the installation guide on CPU require a GPU instances?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541860 ## File path: docs/install/index.md ## @@ -84,7 +84,7 @@ $ wget https://bootstrap.pypa.io/get-pip.py && sudo python get-pip.py **Step 2** Install MXNet with OpenBLAS acceleration. ```bash -$ pip install mxnet +$ sudo pip install mxnet Review comment: I don't know if we need -H This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541505
##
File path: ci/docker/runtime_functions.sh
##
@@ -678,6 +678,65 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+#This Test fails without changing permissions
+chmod -R 0755 tests/nightly/
+./tests/nightly/apache_rat_license_check/license_check.sh
+}
+
+#Checks MXNet for Compilation Warnings
+nightly_test_compilation_warning() {
+set -ex
+export PYTHONPATH=./python/
+#This test fails without changing permissions
+chmod -R 0755 tests/nightly/
Review comment:
Same here
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195543406 ## File path: ci/docker/Dockerfile.build.ubuntu_base_cpu ## @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM ubuntu:16.04 + +ARG USER_ID=os.getuid() Review comment: (see other occasions for reference) This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541213 ## File path: ci/docker/install/ubuntu_nightly_tests.sh ## @@ -0,0 +1,30 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#Install steps for the nightly tests + +set -ex +apt-get update + +#Install for Compilation warning Nightly Test +add-apt-repository ppa:ubuntu-toolchain-r/test +apt-get -y install time Review comment: run apt-get update AFTER adding the repository or it will have no effect This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541558
##
File path: ci/docker/runtime_functions.sh
##
@@ -678,6 +678,65 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+#This Test fails without changing permissions
+chmod -R 0755 tests/nightly/
+./tests/nightly/apache_rat_license_check/license_check.sh
+}
+
+#Checks MXNet for Compilation Warnings
+nightly_test_compilation_warning() {
+set -ex
+export PYTHONPATH=./python/
+#This test fails without changing permissions
+chmod -R 0755 tests/nightly/
+./tests/nightly/compilation_warnings/compilation_warnings.sh
+}
+
+#Checks the MXNet Installation Guide - currently checks pip, build from source
and virtual env on cpu and gpu
+nightly_test_installation() {
+set -ex
+#chmod -R 0755 ./tests/jenkins
Review comment:
clean up
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541584
##
File path: ci/docker/runtime_functions.sh
##
@@ -678,6 +678,65 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+#This Test fails without changing permissions
+chmod -R 0755 tests/nightly/
+./tests/nightly/apache_rat_license_check/license_check.sh
+}
+
+#Checks MXNet for Compilation Warnings
+nightly_test_compilation_warning() {
+set -ex
+export PYTHONPATH=./python/
+#This test fails without changing permissions
+chmod -R 0755 tests/nightly/
+./tests/nightly/compilation_warnings/compilation_warnings.sh
+}
+
+#Checks the MXNet Installation Guide - currently checks pip, build from source
and virtual env on cpu and gpu
+nightly_test_installation() {
+set -ex
+#chmod -R 0755 ./tests/jenkins
+source ./tests/jenkins/run_test_installation_docs.sh docs/install/index.md
1 1686; ${1}
+}
+
+#Runs a simple MNIST training example
+nightly_test_image_classification() {
+#chmod -R 0755 tests/nightly/
Review comment:
clean up
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195542980
##
File path: tests/nightly/JenkinsfileForBinaries
##
@@ -0,0 +1,121 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/tvm/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('Build') {
+parallel 'GPU: CUDA9.1+cuDNN7': {
+ node('mxnetlinux-cpu') {
+ws('workspace/build-gpu') {
+ init_git()
+ //sh "ci/build.py --platform ubuntu_build_cuda
/work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
+ docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7',
false)
+ pack_lib('gpu', mx_lib)
+}
+ }
+}
+ }
+
+ stage('NightlyTests'){
+parallel 'ImageClassification: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-ImageClassificationTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ docker_run('ubuntu_nightly_gpu',
'nightly_test_image_classification', true)
+}
+ }
+},
+'KVStore_SingleNode: GPU': {
+ node('mxnetlinux-gpu-p3') {
+ws('workspace/nt-KVStoreTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ //https://github.com/apache/incubator-mxnet/issues/11289
+ //docker_run('ubuntu_nightly_gpu',
'nightly_test_KVStore_singleNode', true)
Review comment:
No-op
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195542381
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,194 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_gpu', 'nightly_test_compilation_warning',
false)
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_pip', false)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_docker', false)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_cpu_source', false)
Review comment:
Did you mean ubuntu_base_cpu instead of GPU?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195542580
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,194 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_gpu', 'nightly_test_compilation_warning',
false)
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_pip', false)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_docker', false)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_cpu_source', false)
+}
+ }
+},
+'InstallationGuide: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-gpu') {
+ init_git()
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_virtualenv', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_pip', true)
+ ///Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_docker', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_source', true)
+}
+ }
+},
+'PipTest: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-pipTest') {
+ init_git()
+}
+ }
+},
+'Amalgamation-atlas: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-amalgamation1') {
+ init_git()
+ //sh "ci/build.py --platform ubuntu_nightly_cpu
/work/runtime_functions.sh
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195542441
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,194 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_gpu', 'nightly_test_compilation_warning',
false)
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_virtualenv', false)
+ docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_pip', false)
+ //Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_cpu', 'nightly_test_installation
ubuntu_python_cpu_docker', false)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_cpu_source', false)
+}
+ }
+},
+'InstallationGuide: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-gpu') {
+ init_git()
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_virtualenv', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_pip', true)
+ ///Docker Install Test is currently disabled and tracked here:
https://github.com/apache/incubator-mxnet/issues/11288
+ //docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_docker', true)
+ docker_run('ubuntu_base_gpu', 'nightly_test_installation
ubuntu_python_gpu_source', true)
+}
+ }
+},
+'PipTest: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-pipTest') {
+ init_git()
Review comment:
No-op
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195540764 ## File path: ci/docker/Dockerfile.build.ubuntu_nightly_cpu ## @@ -0,0 +1,57 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_core.sh /work/ +RUN /work/ubuntu_core.sh +COPY install/ubuntu_ccache.sh /work/ +RUN /work/ubuntu_ccache.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh +COPY install/ubuntu_r.sh /work/ +RUN /work/ubuntu_r.sh +COPY install/ubuntu_perl.sh /work/ +RUN /work/ubuntu_perl.sh +COPY install/ubuntu_clang.sh /work/ +RUN /work/ubuntu_clang.sh +COPY install/ubuntu_mklml.sh /work/ +RUN /work/ubuntu_mklml.sh +COPY install/ubuntu_caffe.sh /work/ +RUN /work/ubuntu_caffe.sh +COPY install/ubuntu_onnx.sh /work/ +RUN /work/ubuntu_onnx.sh +COPY install/ubuntu_docs.sh /work/ +RUN /work/ubuntu_docs.sh + +ARG USER_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh +COPY install/ubuntu_nightly_tests.sh /work/ Review comment: adduser.sh has to be the last script to be executed. Please move this behind ubuntu_docs. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541605
##
File path: ci/docker/runtime_functions.sh
##
@@ -678,6 +678,65 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+#This Test fails without changing permissions
+chmod -R 0755 tests/nightly/
+./tests/nightly/apache_rat_license_check/license_check.sh
+}
+
+#Checks MXNet for Compilation Warnings
+nightly_test_compilation_warning() {
+set -ex
+export PYTHONPATH=./python/
+#This test fails without changing permissions
+chmod -R 0755 tests/nightly/
+./tests/nightly/compilation_warnings/compilation_warnings.sh
+}
+
+#Checks the MXNet Installation Guide - currently checks pip, build from source
and virtual env on cpu and gpu
+nightly_test_installation() {
+set -ex
+#chmod -R 0755 ./tests/jenkins
+source ./tests/jenkins/run_test_installation_docs.sh docs/install/index.md
1 1686; ${1}
+}
+
+#Runs a simple MNIST training example
+nightly_test_image_classification() {
+#chmod -R 0755 tests/nightly/
+./tests/nightly/test_image_classification.sh
+}
+
+#Single Node KVStore Test
+nightly_test_KVStore_singleNode() {
+#chmod -R 0755 tests/nightly/
Review comment:
clean up
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541083 ## File path: ci/docker/install/ubuntu_emscripten.sh ## @@ -0,0 +1,41 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +#The script has been copied as is from a dockerfile that existed on previous MXNet versions (0.11) +#Written By: Ly + +set -ex + +apt-get install nodejs unzip + +git clone https://github.com/kripken/emscripten.git +git clone https://github.com/kripken/emscripten-fastcomp +cd emscripten-fastcomp +git clone https://github.com/kripken/emscripten-fastcomp-clang tools/clang +mkdir build && cd build + +cmake .. -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86;JSBackend" \ +-DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DCLANG_INCLUDE_EXAMPLES=OFF \ +-DCLANG_INCLUDE_TESTS=OFF && make -j$(nproc) + +chmod -R 777 /work/deps/emscripten-fastcomp/ +export LLVM=/work/deps/emscripten-fastcomp/build/bin Review comment: This will be a NO-OP. Environment variables have to be set on a Dockerfile level This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195540544 ## File path: ci/docker/Dockerfile.build.ubuntu_emscripten ## @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- Review comment: I know thumbs up means done, but you didn't change anything here ._. Please reduce the installed dependencies here. You might also want to add install/ubuntu_emscripten.sh to the regular dockerfiles we have - or make a second pair specifically for nightly and add it there. There are three options, pick whichever you prefer :) This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195542133
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,194 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}
%USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM%
/work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ docker_run('ubuntu_nightly_cpu', 'nightly_test_rat_check', false)
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ docker_run('ubuntu_nightly_gpu', 'nightly_test_compilation_warning',
false)
Review comment:
Why do compilation warnings require a GPU instances?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195540899 ## File path: ci/docker/install/ubuntu_emscripten.sh ## @@ -0,0 +1,41 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +#The script has been copied as is from a dockerfile that existed on previous MXNet versions (0.11) +#Written By: Ly + +set -ex + +apt-get install nodejs unzip + +git clone https://github.com/kripken/emscripten.git Review comment: Please pin the version (for all repositories) This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195541462
##
File path: ci/docker/runtime_functions.sh
##
@@ -678,6 +678,65 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+#This Test fails without changing permissions
+chmod -R 0755 tests/nightly/
Review comment:
Please remove this and instead make chmod 0755 on
tests/nightly/apache_rat_license_check/license_check.sh your repository and
then commit them via git. Git preserves the file-permissions - they have just
been set wrong.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195540103 ## File path: ci/docker/Dockerfile.build.ubuntu_base_gpu ## @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM nvidia/cuda:9.1-cudnn7-devel + +ARG USER_ID=os.getuid() Review comment: See comment from above This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195540799 ## File path: ci/docker/Dockerfile.build.ubuntu_nightly_gpu ## @@ -0,0 +1,63 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for CPU + +FROM nvidia/cuda:9.1-cudnn7-devel + +WORKDIR /work/deps + +COPY install/ubuntu_core.sh /work/ +RUN /work/ubuntu_core.sh +COPY install/ubuntu_ccache.sh /work/ +RUN /work/ubuntu_ccache.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh +COPY install/ubuntu_r.sh /work/ +RUN /work/ubuntu_r.sh +COPY install/ubuntu_perl.sh /work/ +RUN /work/ubuntu_perl.sh +COPY install/ubuntu_clang.sh /work/ +RUN /work/ubuntu_clang.sh +COPY install/ubuntu_mklml.sh /work/ +RUN /work/ubuntu_mklml.sh +COPY install/ubuntu_tvm.sh /work/ +RUN /work/ubuntu_tvm.sh +COPY install/ubuntu_llvm.sh /work/ +RUN /work/ubuntu_llvm.sh +COPY install/ubuntu_caffe.sh /work/ +RUN /work/ubuntu_caffe.sh +COPY install/ubuntu_onnx.sh /work/ +RUN /work/ubuntu_onnx.sh +COPY install/ubuntu_docs.sh /work/ +RUN /work/ubuntu_docs.sh +COPY install/ubuntu_tutorials.sh /work/ +RUN /work/ubuntu_tutorials.sh + +ARG USER_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh +COPY install/ubuntu_nightly_tests.sh /work/ Review comment: same as above This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r195540060 ## File path: ci/docker/Dockerfile.build.ubuntu_base_cpu ## @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet Installation Tests on Ubuntu 16.04 +# This should run in an empty docker with ubuntu and cuda. + +FROM ubuntu:16.04 + +ARG USER_ID=os.getuid() Review comment: Please set to 0 (see other Dockerfiles for reference) and move it before COPY install/ubuntu_adduser.sh /work/ This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r194853090 ## File path: tests/nightly/broken_link_checker_test/broken_link_checker.sh ## @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#Author: Amol Lele + +#software-properties-common, curl are installed in the docker container 'ubuntu_cpu' +# install git-core + +#rm -rf incubator-mxnet-site && rm -rf website-link-checker + +git config --global user.email \"$apache_usern...@gl.com\" && git config --global user.name \"$APACHE_USERNAME\" + +echo "clone the repo and checkout the correct branch" +git clone https://$APACHE_USERNAME:$apache_passw...@github.com/leleamol/incubator-mxnet-site.git Review comment: Agree, we're currently reworking that part This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r194852976
##
File path: tests/nightly/broken_link_checker_test/JenkinsfileForBLC
##
@@ -0,0 +1,72 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for the broken link checker test.
+
+err = null
+
+def init_git() {
Review comment:
Yeah, that'd be great. That way we can also centralize the whole glue around
build.py etc
https://jenkins.io/blog/2017/10/02/pipeline-templates-with-shared-libraries/
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r194852647
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,180 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_nightly_cpu
/work/runtime_functions.sh nightly_test_rat_check"
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_nightly_gpu
/work/runtime_functions.sh nightly_test_compilation_warning"
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_base_cpu
/work/runtime_functions.sh nightly_test_installation
ubuntu_python_cpu_virtualenv"
+ sh "ci/build.py --platform ubuntu_base_cpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_cpu_pip"
+ //Docker installation test is commented out since this would lead to
docker within a docker
+ //sh "ci/build.py --platform ubuntu_base_cpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_cpu_docker"
+ sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_cpu_source"
+}
+ }
+},
+'InstallationGuide: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-gpu') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation
ubuntu_python_gpu_virtualenv"
+ sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_gpu_pip"
+ //Docker installation test is commented out since this would lead to
docker within a docker
Review comment:
There are instructions available here:
http://jpetazzo.github.io/2015/09/03/do-not-use-docker-in-docker-for-ci/
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r194852794
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,180 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_nightly_cpu
/work/runtime_functions.sh nightly_test_rat_check"
+}
+ }
+},
+'CompilationWarnings: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-compilationTest') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_nightly_gpu
/work/runtime_functions.sh nightly_test_compilation_warning"
+}
+ }
+},
+'InstallationGuide: CPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-cpu') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_base_cpu
/work/runtime_functions.sh nightly_test_installation
ubuntu_python_cpu_virtualenv"
+ sh "ci/build.py --platform ubuntu_base_cpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_cpu_pip"
+ //Docker installation test is commented out since this would lead to
docker within a docker
+ //sh "ci/build.py --platform ubuntu_base_cpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_cpu_docker"
+ sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_cpu_source"
+}
+ }
+},
+'InstallationGuide: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-Installation-gpu') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation
ubuntu_python_gpu_virtualenv"
+ sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_gpu_pip"
+ //Docker installation test is commented out since this would lead to
docker within a docker
+ //sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_gpu_docker"
+ sh "ci/build.py --platform ubuntu_base_gpu
/work/runtime_functions.sh nightly_test_installation ubuntu_python_gpu_source"
+}
+ }
+},
+'PipTest: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-pipTest') {
+ init_git()
+ //sh "ci/build.py --platform ubuntu_nightly_gpu
/work/runtime_functions.sh nightly_test_pip_test"
+}
+ }
+},
+'Amalgamation-atlas: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-amalgamation1') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_nightly_cpu
/work/runtime_functions.sh nightly_test_amalgamation USE_BLAS=atlas"
+}
+ }
+},
+'Amalgamation-atlas-min: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-amalgamation2') {
+
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r194852291
##
File path: tests/nightly/JenkinsfileForBinaries
##
@@ -0,0 +1,111 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+
+try {
+ stage('Build') {
+parallel 'GPU: CUDA9.1+cuDNN7': {
+ node('mxnetlinux-cpu') {
+ws('workspace/build-gpu') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_build_cuda
/work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
+ pack_lib('gpu', mx_lib)
+}
+ }
+}
+ }
+
+ stage('NightlyTests'){
+parallel 'ImageClassification: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-ImageClassificationTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh nightly_test_image_classification"
+}
+ }
+},
+'KVStore_SingleNode: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-KVStoreTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ //Note: This test is commented for now since it needs a p2/p3
instance
Review comment:
mxnetlinux-gpu-p3
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r194587572
##
File path: tests/jenkins/run_test_installation_docs.sh
##
@@ -299,28 +317,53 @@ LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux
Python GPU Installation Instr
set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO}
${LINUX_PYTHON_GPU_END_LINENO}
-# mxnet/base-cuda9 is a simple Docker Image with
'nvidia/cuda:9.0-cudnn7-devel' and 'apt-get install sudo'.
+ubuntu_python_gpu_virtualenv()
+{
+#$WORDTOREMOVE
+echo
+echo "### Testing Virtualenv ###"
+echo "${virtualenv_commands}"
+echo
+#virtualenv_commands=${virtualenv_commands//$WORDTOREMOVE/}
Review comment:
It's commented out, so we probably don't need it anymore, right?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r194587514
##
File path: ci/docker/runtime_functions.sh
##
@@ -591,6 +591,65 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+#This Test fails without changing permissions
Review comment:
This means that the file has not been committed with the correct file
permissions. Try chmod 0755
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193911055 ## File path: ci/docker/Dockerfile.build.ubuntu_emscripten ## @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- Review comment: It's for Amalgamation (JavaScript generation) and we have to keep it. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193911055 ## File path: ci/docker/Dockerfile.build.ubuntu_emscripten ## @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- Review comment: It's for Amalgamation and we have to keep it. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193044553 ## File path: tests/nightly/apache_rat_license_check/license_check.sh ## @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +#mvn and svn are installed in the docker container via the nghtly test script + +echo "download RAT" +svn co http://svn.apache.org/repos/asf/creadur/rat/trunk/ #>/dev/null + +echo "cd into directory" +cd trunk + +echo "mvn install" +mvn -Dmaven.test.skip=true install #>/dev/null + +echo "build success, cd into target" +cd apache-rat/target Review comment: This should be part of some installation and not part of every run This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193044127
##
File path: tests/nightly/JenkinsfileForBinaries
##
@@ -0,0 +1,111 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+
+try {
+ stage('Build') {
+parallel 'GPU: CUDA9.1+cuDNN7': {
+ node('mxnetlinux-cpu') {
+ws('workspace/build-gpu') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_build_cuda
/work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
+ pack_lib('gpu', mx_lib)
+}
+ }
+}
+ }
+
+ stage('NightlyTests'){
+parallel 'ImageClassification: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-ImageClassificationTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh nightly_test_image_classification"
+}
+ }
+},
+'KVStore_SingleNode: GPU': {
+ node('mxnetlinux-gpu') {
+ws('workspace/nt-KVStoreTest') {
+ init_git()
+ unpack_lib('gpu', mx_lib)
+ //Note: This test is commented for now since it needs a p2/p3
instance
Review comment:
We have p3 instances available
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193043881
##
File path: tests/nightly/Jenkinsfile
##
@@ -0,0 +1,180 @@
+// -*- mode: groovy -*-
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+//This is a Jenkinsfile for nightly tests. The format and some functions have
been picked up from the top-level Jenkinsfile
+
+err = null
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/nnvm/lib/libnnvm.a'
+
+// pack libraries for later use
+def pack_lib(name, libs=mx_lib) {
+ sh """
+echo "Packing ${libs} into ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+ stash includes: libs, name: name
+}
+
+// unpack libraries saved before
+def unpack_lib(name, libs=mx_lib) {
+ unstash name
+ sh """
+echo "Unpacked ${libs} from ${name}"
+echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+"""
+}
+
+def init_git() {
+ deleteDir()
+ retry(5) {
+try {
+ timeout(time: 15, unit: 'MINUTES') {
+checkout scm
+sh 'git submodule update --init --recursive'
+sh 'git clean -d -f'
+ }
+} catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+}
+ }
+}
+
+
+try {
+ stage('NightlyTests'){
+parallel 'RATCheck: CPU': {
+ node('mxnetlinux-cpu') {
+ws('workspace/nt-RATTest') {
+ init_git()
+ sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
nightly_test_rat_check"
Review comment:
Please update the Jenkinsfile to incorporate the latest build.py changes.
See the original Jenkinsfile for reference
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193043648
##
File path: tests/jenkins/run_test_installation_docs.sh
##
@@ -299,28 +317,53 @@ LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux
Python GPU Installation Instr
set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO}
${LINUX_PYTHON_GPU_END_LINENO}
-# mxnet/base-cuda9 is a simple Docker Image with
'nvidia/cuda:9.0-cudnn7-devel' and 'apt-get install sudo'.
+ubuntu_python_gpu_virtualenv()
+{
+#$WORDTOREMOVE
+echo
+echo "### Testing Virtualenv ###"
+echo "${virtualenv_commands}"
+echo
+#virtualenv_commands=${virtualenv_commands//$WORDTOREMOVE/}
Review comment:
Remove
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193043157 ## File path: ci/docker/Dockerfile.build.ubuntu_emscripten ## @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- Review comment: Are all these dependencies really required for emscripten? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2
new pipelines to the Official CI and run nightly tests.
URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193042844
##
File path: ci/docker/runtime_functions.sh
##
@@ -591,6 +591,65 @@ build_docs() {
popd
}
+
+# Functions that run the nightly Tests:
+
+#Runs Apache RAT Check on MXNet Source for License Headers
+nightly_test_rat_check() {
+set -ex
+#This Test fails without changing permissions
Review comment:
With what error?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193042600 ## File path: ci/docker/install/ubuntu_nightly_tests.sh ## @@ -0,0 +1,30 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#Install steps for the nightly tests + +set -ex +apt-get update + +#Install for Compilation warning Nightly Test +add-apt-repository ppa:ubuntu-toolchain-r/test +apt-get -y install time g++-5 Review comment: We should have g++-5 already installed, right? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193042398 ## File path: ci/docker/install/ubuntu_emscripten.sh ## @@ -0,0 +1,41 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +#The script has been copied as is from a dockerfile that existed on previous MXNet versions (0.11) +#Written By: Ly + +set -ex + +apt-get install nodejs unzip + +git clone https://github.com/kripken/emscripten.git +git clone https://github.com/kripken/emscripten-fastcomp +cd emscripten-fastcomp +git clone https://github.com/kripken/emscripten-fastcomp-clang tools/clang +mkdir build && cd build + +cmake .. -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86;JSBackend" \ +-DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DCLANG_INCLUDE_EXAMPLES=OFF \ +-DCLANG_INCLUDE_TESTS=OFF && make Review comment: don't forget make -j or it won't be parallel This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests.
marcoabreu commented on a change in pull request #10827: [MXNET-405][WIP] Add 2 new pipelines to the Official CI and run nightly tests. URL: https://github.com/apache/incubator-mxnet/pull/10827#discussion_r193041704 ## File path: ci/docker/Dockerfile.build.ubuntu_emscripten ## @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run the MXNet JS amalgamation test. Builds on top of the ubuntu_cpu dockerfile. + +FROM ubuntu:16.04 + +ARG USER_ID=0 + +WORKDIR /work/deps + +COPY install/ubuntu_core.sh /work/ +RUN /work/ubuntu_core.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh +COPY install/ubuntu_r.sh /work/ +RUN /work/ubuntu_r.sh +COPY install/ubuntu_perl.sh /work/ +RUN /work/ubuntu_perl.sh +COPY install/ubuntu_clang.sh /work/ +RUN /work/ubuntu_clang.sh +COPY install/ubuntu_mklml.sh /work/ +RUN /work/ubuntu_mklml.sh +COPY install/ubuntu_caffe.sh /work/ +RUN /work/ubuntu_caffe.sh +COPY install/ubuntu_onnx.sh /work/ +RUN /work/ubuntu_onnx.sh +COPY install/ubuntu_docs.sh /work/ +RUN /work/ubuntu_docs.sh +COPY install/ubuntu_adduser.sh /work/ Review comment: Please keep this one at the end This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
