This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2429 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c720897f1b77a9901fccf96c8269310ad8261397 Author: leonidfrolov <[email protected]> AuthorDate: Thu Jun 17 16:36:30 2021 +0300 [DATALAB-2429]: fixed additional lib installation for gcp dataengine-service --- infrastructure-provisioning/src/general/lib/os/fab.py | 15 +++++++++------ .../scripts/gcp/dataengine-service_install_libs.py | 6 +++--- .../src/general/scripts/os/install_additional_libs.py | 7 ++++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index 10420b8..b69c515 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -99,12 +99,15 @@ def dataengine_dir_prepare(cluster_dir): subprocess.run('mkdir -p ' + cluster_dir, shell=True, check=True) -def install_pip_pkg(requisites, pip_version, lib_group): +def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = False): status = list() error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:" try: - venv_install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format( - os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3]) + if dataengine_service: + install_command = pip_version + else: + install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format( + os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3]) #if pip_version == 'pip3' and not exists(conn, '/bin/pip3'): # for v in range(4, 8): # if exists(conn, '/bin/pip3.{}'.format(v)): @@ -124,13 +127,13 @@ def install_pip_pkg(requisites, pip_version, lib_group): '''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | ''' '''tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; ''' '''then echo "" > /tmp/{4}install_{3}.log;fi' '''.format( - venv_install_command, pip_pkg, error_parser, name, pip_version)) + install_command, pip_pkg, error_parser, name, pip_version)) err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace( '"', "'").replace('\n', ' ') conn.sudo( '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; ''' '''then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format( - venv_install_command, name, pip_version)) + install_command, name, pip_version)) res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace( '\n', '') conn.sudo( @@ -145,7 +148,7 @@ def install_pip_pkg(requisites, pip_version, lib_group): changed_pip_pkg = changed_pip_pkg[0] conn.sudo( '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format( - venv_install_command, changed_pip_pkg, pip_version)) + install_command, changed_pip_pkg, pip_version)) res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace( '\n', '') if err and name not in installed_out: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py index bcc2eb7..32475e1 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py @@ -34,8 +34,8 @@ from fabric import * def install_libs(instance, data_engine): - data_engine['instance_ip'] = meta_lib.GCPMeta().get_private_ip_address(instance) - params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}"' \ + data_engine['instance_ip'] = GCPMeta().get_private_ip_address(instance) + params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}" --dataengine_service True' \ .format(data_engine['os_user'], data_engine['instance_ip'], data_engine['keyfile'], data_engine['libs']) try: @@ -65,7 +65,7 @@ if __name__ == "__main__": data_engine['gcp_project_id'] = os.environ['gcp_project_id'] data_engine['gcp_region'] = os.environ['gcp_region'] data_engine['gcp_zone'] = os.environ['gcp_zone'] - res = meta_lib.GCPMeta().get_list_instances(data_engine['gcp_zone'], data_engine['cluster_name']) + res = GCPMeta().get_list_instances(data_engine['gcp_zone'], data_engine['cluster_name']) data_engine['cluster_instances'] = [i.get('name') for i in res['items']] data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) data_engine['libs'] = os.environ['libs'] diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py index 764f86d..cf443c5 100644 --- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py +++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py @@ -34,6 +34,7 @@ parser.add_argument('--keyfile', type=str, default='') parser.add_argument('--instance_ip', type=str, default='') parser.add_argument('--os_user', type=str, default='') parser.add_argument('--libs', type=str, default='') +parser.add_argument('--dataengine_service', type=bool, default=False) args = parser.parse_args() @@ -78,14 +79,14 @@ if __name__ == "__main__": try: print('Installing pip2 packages: {}'.format(pkgs['libraries']['pip2'])) - status = install_pip_pkg(pkgs['libraries']['pip2'], 'pip2', 'pip2') + status = install_pip_pkg(pkgs['libraries']['pip2'], 'pip2', 'pip2', args.dataengine_service) general_status = general_status + status except KeyError: pass try: print('Installing pip3 packages: {}'.format(pkgs['libraries']['pip3'])) - status = install_pip_pkg(pkgs['libraries']['pip3'], 'pip3', 'pip3') + status = install_pip_pkg(pkgs['libraries']['pip3'], 'pip3', 'pip3', args.dataengine_service) general_status = general_status + status except KeyError: pass @@ -93,7 +94,7 @@ if __name__ == "__main__": try: print('Installing other packages (only tries pip3): {}'.format(pkgs['libraries']['others'])) for pkg in pkgs['libraries']['others']: - status_pip3 = install_pip_pkg([pkg], 'pip3', 'others') + status_pip3 = install_pip_pkg([pkg], 'pip3', 'others', args.dataengine_service) general_status = general_status + status_pip3 except KeyError: pass --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
