This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2414 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6c694ce1044d11c44f34cc6329f1b6802b98210c Author: leonidfrolov <[email protected]> AuthorDate: Wed Jun 2 14:58:03 2021 +0300 [DATALAB-2398]: virtualenv usage for rstudio --- .../src/general/lib/os/debian/notebook_lib.py | 5 ++++- .../src/rstudio/scripts/configure_rstudio_node.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py index d6d7d63..4bcd2d9 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py @@ -122,7 +122,7 @@ def ensure_r(os_user, r_libs, region, r_mirror): sys.exit(1) -def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version): +def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version, python_venv_path=''): if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/rstudio_ensured'): try: manage_pkg('-y install', 'remote', 'r-base') @@ -150,6 +150,7 @@ def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version): datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Renviron'.format(os_user)) datalab.fab.conn.sudo('''echo 'SPARK_HOME="{0}"' >> /home/{1}/.Renviron'''.format(local_spark_path, os_user)) datalab.fab.conn.sudo('''echo 'JAVA_HOME="{0}"' >> /home/{1}/.Renviron'''.format(java_home, os_user)) + #datalab.fab.conn.sudo('''echo 'RETICULATE_PYTHON="{0}"' >> /home/{1}/.Renviron'''.format(python_venv_path, os_user)) datalab.fab.conn.sudo('touch /home/{}/.Rprofile'.format(os_user)) datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user)) datalab.fab.conn.sudo('''echo 'library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >> /home/{}/.Rprofile'''.format(os_user)) @@ -255,6 +256,8 @@ def ensure_python3_libraries(os_user): #manage_pkg('-y install', 'remote', 'python3-setuptools') manage_pkg('-y install', 'remote', 'python3-pip') manage_pkg('-y install', 'remote', 'libkrb5-dev') + #datalab.fab.conn.sudo('add-apt-repository ppa:deadsnakes/ppa') + #manage_pkg('-y install', 'remote', 'libpython3.7-dev') datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff') datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version'])) try: diff --git a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py index cf0679e..9ea4f12 100644 --- a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py +++ b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py @@ -47,6 +47,8 @@ args = parser.parse_args() spark_version = os.environ['notebook_spark_version'] hadoop_version = os.environ['notebook_hadoop_version'] +python_venv_version = os.environ['notebook_python_venv_version'] +python_venv_path = '/opt/python/python{0}/bin/python{1}'.format(python_venv_version, python_venv_version[:3]) if args.region == 'cn-north-1': spark_link = "http://mirrors.hust.edu.cn/apache/spark/spark-"; + spark_version + "/spark-" + spark_version + \ "-bin-hadoop" + hadoop_version + ".tgz" @@ -88,9 +90,13 @@ if __name__ == "__main__": print("Install Python 3 modules") ensure_python3_libraries(args.os_user) + # INSTALL PYTHON IN VIRTUALENV + print("Configure Python Virtualenv") + #ensure_python_venv(python_venv_version) + # INSTALL RSTUDIO print("Install RStudio") - install_rstudio(args.os_user, local_spark_path, args.rstudio_pass, args.rstudio_version) + install_rstudio(args.os_user, local_spark_path, args.rstudio_pass, args.rstudio_version, python_venv_path) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK print("Install local Spark") --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
