This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DLAB-1748 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit a17e982cc4a638bfb47efcf97cedaacb70b4f6eb Author: leonidfrolov <frolovl...@gmail.com> AuthorDate: Fri Jul 10 18:54:37 2020 +0300 [DLAB-1748]: added installaton of specific version for R --- .../src/general/lib/os/debian/notebook_lib.py | 2 +- .../src/general/lib/os/fab.py | 41 ++++++++++++++++++---- .../general/scripts/os/install_additional_libs.py | 4 +-- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py index c97ccb8..ea971e2 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py @@ -417,7 +417,7 @@ def get_available_os_pkgs(): os_pkgs = dict() ansi_escape = re.compile(r'\x1b[^m]*m') manage_pkg('update', 'remote', '') - apt_raw = sudo("apt list -a") + apt_raw = sudo("apt list") apt_list = ansi_escape.sub('', apt_raw).split("\r\n") for pkg in apt_list: if "/" in pkg: diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index ec9b6c2..7f80787 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -402,21 +402,49 @@ def ensure_ciphers(): def install_r_pkg(requisites): status = list() - error_parser = "ERROR:|error:|Cannot|failed|Please run|requires" + error_parser = "ERROR:|error:|Cannot|failed|Please run|requires|Error" try: for r_pkg in requisites: - if r_pkg == 'sparklyr': - run('sudo -i R -e \'install.packages("{0}", repos="https://cloud.r-project.org", dep=TRUE)\' 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(r_pkg, error_parser)) - sudo('R -e \'install.packages("{0}", repos="https://cloud.r-project.org", dep=TRUE)\' 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(r_pkg, error_parser)) + if r_pkg[1] == '': + r_pkg = pip_pkg[0] + if r_pkg == 'sparklyr': + run('sudo -i R -e \'install.packages("{0}", repos="https://cloud.r-project.org", dep=TRUE)\' 2>&1 | ' + 'tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" >' + ' /tmp/install_{0}.log;fi'.format(r_pkg, error_parser)) + else: + sudo('R -e \'install.packages("{0}", repos="https://cloud.r-project.org", dep=TRUE)\' 2>&1 | ' + 'tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" >' + ' /tmp/install_{0}.log;fi'.format(r_pkg, error_parser)) + else: + if r_pkg == 'sparklyr': + run('sudo -i R -e \'devtools::install_version("{0}", version = "{1}", repos = "http://cran.us.r-project.org", ' + 'dep=TRUE)\' 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > /tmp/install_{0}.log; ' + 'then echo "" > /tmp/install_{0}.log;fi'.format(r_pkg[0], r_pkg[1], error_parser)) + else: + sudo('R -e \'devtools::install_version("{0}", version = "{1}", repos = "http://cran.us.r-project.org", ' + 'dep=TRUE)\' 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > /tmp/install_{0}.log; ' + 'then echo "" > /tmp/install_{0}.log;fi'.format(r_pkg[0], r_pkg[1], args.version)) + dep = sudo('grep "(NA -> " /tmp/tee.tmp | awk \'{print $1}\'').replace('\r\n', ' ') + if dep == r_pkg or dep == '': + dep = [] + else: + dep = dep.split(' ') err = sudo('cat /tmp/install_{0}.log'.format(r_pkg)).replace('"', "'") sudo('R -e \'installed.packages()[,c(3:4)]\' | if ! grep -w {0} > /tmp/install_{0}.list; then echo "" > /tmp/install_{0}.list;fi'.format(r_pkg)) res = sudo('cat /tmp/install_{0}.list'.format(r_pkg)) if res: ansi_escape = re.compile(r'\x1b[^m]*m') version = ansi_escape.sub('', res).split("\r\n")[0].split('"')[1] - status.append({"group": "r_pkg", "name": r_pkg, "version": version, "status": "installed"}) + status.append({"group": "r_pkg", "name": r_pkg, "version": version, "status": "installed", "add_pkgs": dep}) else: - status.append({"group": "r_pkg", "name": r_pkg, "status": "failed", "error_message": err}) + if 'Error in download_version_url(package, version, repos, type) :' in err: + versions = sudo('R -e \'library(versions); available.versions("' + r_pkg + '")\' 2>&1 | grep -A 50 ' + '\'date available\' | awk \'{print $2}\'').replace('\r\n', ' ')[5:] + if versions == '': + versions = [] + else: + versions = versions.split(' ') + status.append({"group": "r_pkg", "name": r_pkg, "status": "failed", "error_message": err, "available_versions": versions}) return status except: return "Fail to install R packages" @@ -614,6 +642,7 @@ def install_r_packages(os_user): sudo('R -e "install.packages(\'ggplot2\', repos = \'https://cloud.r-project.org\')"') sudo('R -e "install.packages(c(\'devtools\',\'mplot\', \'googleVis\'), ' 'repos = \'https://cloud.r-project.org\'); require(devtools); install_github(\'ramnathv/rCharts\')"') + sudo('R -e \'install.packages("versions", repos="https://cloud.r-project.org", dep=TRUE)\'') sudo('touch /home/' + os_user + '/.ensure_dir/r_packages_ensured') diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py index e91ae0b..d2ce331 100644 --- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py +++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py @@ -58,11 +58,9 @@ if __name__ == "__main__": pkgs['libraries'][data[row]['group']].append( [data[row]['name'].split(':')[0], data[row]['name'].split(':')[1], data[row]['version'], data[row]['override']]) - elif data[row]['group'] == "pip2" or data[row]['group'] == "pip3" or data[row]['group'] == "os_pkg" or data[row]['group'] == "others": + else: pkgs['libraries'][data[row]['group']].append( [data[row]['name'], data[row]['version']]) - else: - pkgs['libraries'][data[row]['group']].append(data[row]['name']) except Exception as err: print('Error: {0}'.format(err)) append_result("Failed to parse libs list.", str(err)) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org