This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2691 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 00a0e58cb4975fbffd9dc6185a857edc96836a4f Author: leonidfrolov <[email protected]> AuthorDate: Tue Mar 15 19:22:52 2022 +0200 [DATALAB-2691]: fixed python libs install for gcp dataproc on zeppelin --- .../src/general/lib/gcp/actions_lib.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py index 8d3543c..6bfc177 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py @@ -1355,15 +1355,16 @@ class GCPActions: subprocess.run('sudo -i virtualenv /opt/python/python{}'.format(python_version), shell=True, check=True) venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version) pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3]) - subprocess.run('{0} && sudo -i {1} install -U pip==9.0.3'.format(venv_command, pip_command), shell=True, check=True) - subprocess.run('{0} && sudo -i {1} install pyzmq==17.0.0'.format(venv_command, pip_command), shell=True, check=True) - subprocess.run('{0} && sudo -i {1} install ipython ipykernel --no-cache-dir'.format(venv_command, pip_command), shell=True, check=True) - subprocess.run('{0} && sudo -i {1} install boto boto3 NumPy=={2} SciPy Matplotlib pandas Sympy Pillow sklearn --no-cache-dir' - .format(venv_command, pip_command, numpy_version), shell=True, check=True) + subprocess.run('bash -c "{0} && sudo -i {1} install -U pip==9.0.3"'.format(venv_command, pip_command), shell=True, check=True) + subprocess.run('bash -c "{0} && sudo -i {1} install pyzmq==17.0.0"'.format(venv_command, pip_command), shell=True, check=True) + for lib in ['ipython', 'ipykernel', 'NumPy=={}'.format(numpy_version), 'boto', 'boto3', 'pybind11', + 'pythran', 'cython', 'SciPy', 'Matplotlib', 'pandas', 'Sympy', 'Pillow', 'sklearn']: + subprocess.run('bash -c "{0} && sudo -i {1} install {2} --no-cache-dir"' + .format(venv_command, pip_command, lib), shell=True, check=True) if application == 'deeplearning': - subprocess.run('{0} && sudo -i {1} install mxnet-cu80 opencv-python keras Theano --no-cache-dir'.format(venv_command, pip_command), shell=True, check=True) + subprocess.run('bash -c "{0} && sudo -i {1} install mxnet-cu80 opencv-python keras Theano --no-cache-dir"'.format(venv_command, pip_command), shell=True, check=True) python_without_dots = python_version.replace('.', '') - subprocess.run('{0} && sudo -i {1} install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{2}-cp{2}m-linux_x86_64.whl --no-cache-dir' + subprocess.run('bash -c "{0} && sudo -i {1} install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{2}-cp{2}m-linux_x86_64.whl --no-cache-dir"' .format(venv_command, pip_command, python_without_dots[:2]), shell=True, check=True) subprocess.run('sudo rm -rf /usr/bin/python{}-dp'.format(python_version[0:3]), shell=True, check=True) subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'.format(python_version, python_version[0:3]), shell=True, check=True) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
