This is an automated email from the ASF dual-hosted git repository. ioleksandr pushed a commit to branch DLAB-523 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit d27782dd503223810edd458edc565760e544d50c Author: Oleksandr_Isniuk <i.oleksa...@gmail.com> AuthorDate: Fri Apr 5 18:23:34 2019 +0300 [GCP]: Fix Data Engine creation fails on DeepLearning -remove old and unsupported library Caffe -change versionc of cuda for Data Engine on deeplearning --- .../src/dataengine/scripts/configure_dataengine.py | 8 +++- .../scripts/configure_deep_learning_node.py | 2 - .../src/general/lib/os/debian/notebook_lib.py | 39 ------------------ .../src/general/lib/os/redhat/notebook_lib.py | 46 ---------------------- 4 files changed, 6 insertions(+), 89 deletions(-) diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py index 9a4fb29..e5aacb8 100644 --- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py +++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py @@ -73,6 +73,12 @@ files_dir = '/root/files/' local_spark_path = '/opt/spark/' jars_dir = '/opt/jars/' r_libs = ['R6', 'pbdZMQ', 'RCurl', 'devtools', 'reshape2', 'caTools', 'rJava', 'ggplot2'] +if os.environ['application'] == 'deeplearning': + tensorflow_version = '1.4.0' + cuda_version = '8.0' + cuda_file_name = 'cuda_8.0.44_linux-run' + cudnn_version = '6.0' + cudnn_file_name = 'cudnn-8.0-linux-x64-v6.0.tgz' def start_spark(os_user, master_ip, node): @@ -165,8 +171,6 @@ if __name__ == "__main__": # INSTALL DEEP LEARNING FRAMEWORKS if os.environ['application'] == 'deeplearning': - print("Installing Caffe") - install_caffe(args.os_user, args.region, caffe_version) print("Installing Caffe2") install_caffe2(args.os_user, caffe2_version, cmake_version) print("Installing Torch") diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py index ec188ed..3903bbb 100644 --- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py +++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py @@ -121,8 +121,6 @@ if __name__ == "__main__": install_theano(args.os_user, theano_version) print("Installing Keras") install_keras(args.os_user, keras_version) - print("Installing Caffe") - install_caffe(args.os_user, args.region, caffe_version) print("Installing Caffe2") install_caffe2(args.os_user, caffe2_version, cmake_version) print("Installing Torch") diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py index ecb5f02..91da82d 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py @@ -405,45 +405,6 @@ def get_available_os_pkgs(): sys.exit(1) -def install_caffe(os_user, region, caffe_version): - if not exists('/home/{}/.ensure_dir/caffe_ensured'.format(os_user)): - env.shell = "/bin/bash -l -c -i" - sudo('apt-get install -y python-dev') - sudo('apt-get install -y python3-dev') - sudo('apt-get install -y libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler') - sudo('apt-get install -y --no-install-recommends libboost-all-dev') - sudo('apt-get install -y libatlas-base-dev libopenblas-dev') - sudo('apt-get install -y libgflags-dev libgoogle-glog-dev liblmdb-dev') - with cd('/usr/lib/x86_64-linux-gnu/'): - sudo('ln -s libhdf5_serial_hl.so.10.0.2 libhdf5_hl.so') - sudo('ln -s libhdf5_serial.so.10.1.0 libhdf5.so') - sudo('git clone https://github.com/BVLC/caffe.git') - with cd('/home/{}/caffe/'.format(os_user)): - sudo('git checkout {}'.format(caffe_version)) - sudo('pip2 install -r python/requirements.txt --no-cache-dir') - sudo('pip3 install -r python/requirements.txt --no-cache-dir') - sudo('echo "CUDA_DIR := /usr/local/cuda" > Makefile.config') - cuda_arch = sudo("/opt/cuda-8.0/extras/demo_suite/deviceQuery | grep 'CUDA Capability' | tr -d ' ' | cut -f2 -d ':'") - sudo('echo "CUDA_ARCH := -gencode arch=compute_{0},code=sm_{0}" >> Makefile.config'.format(cuda_arch.replace('.', ''))) - sudo('echo "PYTHON_INCLUDE := /usr/include/python2.7 /usr/local/lib/python2.7/dist-packages/numpy/core/include" >> Makefile.config') - sudo('echo "BLAS := open" >> Makefile.config') - sudo('echo "BLAS_INCLUDE := /usr/include/openblas" >> Makefile.config') - #sudo('echo "OPENCV_VERSION := 3" >> Makefile.config') - sudo('echo "LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_serial_hl hdf5_serial" >> Makefile.config') - sudo('echo "PYTHON_LIB := /usr/lib" >> Makefile.config') - sudo('echo "INCLUDE_DIRS := \\\$(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial/ /usr /usr/lib /usr/include/python2.7 /usr/local/lib/python2.7/dist-packages/numpy/core/include" >> Makefile.config') - sudo('echo "LIBRARY_DIRS := \\\$(PYTHON_LIB) /usr/local/lib /usr/lib /usr /usr/lib" >> Makefile.config') - sudo('echo "BUILD_DIR := build" >> Makefile.config') - sudo('echo "DISTRIBUTE_DIR := distribute" >> Makefile.config') - sudo('echo "TEST_GPUID := 0" >> Makefile.config') - sudo('echo "Q ?= @" >> Makefile.config') - sudo('make all -j$(nproc)') - sudo('make test -j$(nproc)') - run('make runtest') - sudo('make pycaffe') - sudo('touch /home/' + os_user + '/.ensure_dir/caffe_ensured') - - def install_caffe2(os_user, caffe2_version, cmake_version): if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)): env.shell = "/bin/bash -l -c -i" diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py index d03710d..a103d26 100644 --- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py @@ -402,52 +402,6 @@ def install_opencv(os_user): sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured') -def install_caffe(os_user, region, caffe_version): - if not exists('/home/{}/.ensure_dir/caffe_ensured'.format(os_user)): - env.shell = "/bin/bash -l -c -i" - install_opencv(os_user) - with cd('/etc/yum.repos.d/'): - if region == 'cn-north-1': - mirror = 'mirror.lzu.edu.cn' - else: - mirror = 'mirror.centos.org' - sudo('echo "[centosrepo]" > centos.repo') - sudo('echo "name=Centos 7 Repository" >> centos.repo') - sudo('echo "baseurl=http://{}/centos/7/os/x86_64/" >> centos.repo'.format(mirror)) - sudo('echo "enabled=1" >> centos.repo') - sudo('echo "gpgcheck=1" >> centos.repo') - sudo('echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> centos.repo'.format(mirror)) - sudo('yum update-minimal --security -y') - sudo('yum install -y --nogpgcheck protobuf-devel leveldb-devel snappy-devel boost-devel hdf5-devel gcc gcc-c++') - sudo('yum install -y gflags-devel glog-devel lmdb-devel yum-utils && package-cleanup --cleandupes') - sudo('yum install -y openblas-devel gflags-devel glog-devel lmdb-devel') - sudo('git clone https://github.com/BVLC/caffe.git') - with cd('/home/{}/caffe/'.format(os_user)): - sudo('git checkout {}'.format(caffe_version)) - sudo('pip2 install -r python/requirements.txt --no-cache-dir') - sudo('pip3.5 install -r python/requirements.txt --no-cache-dir') - sudo('echo "CUDA_DIR := /usr/local/cuda" > Makefile.config') - cuda_arch = sudo("/opt/cuda-8.0/extras/demo_suite/deviceQuery | grep 'CUDA Capability' | tr -d ' ' | cut -f2 -d ':'") - sudo('echo "CUDA_ARCH := -gencode arch=compute_{0},code=sm_{0}" >> Makefile.config'.format(cuda_arch.replace('.', ''))) - sudo('echo "PYTHON_INCLUDE := /usr/include/python2.7 /usr/lib64/python2.7/site-packages/numpy/core/include" >> Makefile.config') - sudo('echo "BLAS := open" >> Makefile.config') - sudo('echo "BLAS_INCLUDE := /usr/include/openblas" >> Makefile.config') - sudo('echo "OPENCV_VERSION := 3" >> Makefile.config') - sudo('echo "LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_serial_hl hdf5_serial" >> Makefile.config') - sudo('echo "PYTHON_LIB := /usr/lib" >> Makefile.config') - sudo('echo "INCLUDE_DIRS := \\\$(PYTHON_INCLUDE) /usr/local/include /usr /usr/lib64 /usr/include/python2.7 /usr/lib64/python2.7/site-packages/numpy/core/include" >> Makefile.config') - sudo('echo "LIBRARY_DIRS := \\\$(PYTHON_LIB) /usr/local/lib /usr/lib /usr /usr/lib64" >> Makefile.config') - sudo('echo "BUILD_DIR := build" >> Makefile.config') - sudo('echo "DISTRIBUTE_DIR := distribute" >> Makefile.config') - sudo('echo "TEST_GPUID := 0" >> Makefile.config') - sudo('echo "Q ?= @" >> Makefile.config') - sudo('make all -j$(nproc)') - sudo('make test -j$(nproc)') - run('make runtest') - sudo('make pycaffe') - sudo('touch /home/' + os_user + '/.ensure_dir/caffe_ensured') - - def install_caffe2(os_user, caffe2_version, cmake_version): if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)): env.shell = "/bin/bash -l -c -i" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org