This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-408 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f159de70afa70126963e900079baec2582a6831b Author: leonidfrolov <[email protected]> AuthorDate: Wed Aug 4 15:20:42 2021 +0300 [DATALAB-408]: usage of docker images from nexus during ssn deployment --- .../scripts/deploy_datalab.py | 16 ++++-- .../src/ssn/scripts/configure_docker.py | 58 ++++++++++++++-------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py index e3bdd4c..45fde90 100644 --- a/infrastructure-provisioning/scripts/deploy_datalab.py +++ b/infrastructure-provisioning/scripts/deploy_datalab.py @@ -206,11 +206,21 @@ def generate_docker_command(): def build_docker_images(args): - # Building base and ssn docker images - subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file ' + if args.conf_repository_user and args.conf_repository_pass and args.conf_repository_address: + subprocess.run( 'sudo docker login -u {0} -p {1} {2}:8083' + .format(args.conf_repository_user, args.conf_repository_pass, args.conf_repository_address), shell=True, check=True) + subprocess.run('docker pull {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True) + subprocess.run('docker image tag {}:8083/docker.datalab-base-{} docker.datalab-base'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True) + subprocess.run('docker image rm {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True) + subprocess.run('docker pull {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True) + subprocess.run('docker image tag {}:8083/docker.datalab-ssn-{} docker.datalab-ssn'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True) + subprocess.run('docker image rm {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.cloud_provider), shell=True, check=True) + else: + # Building base and ssn docker images + subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file ' 'infrastructure-provisioning/src/general/files/{1}/' 'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True) - subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/' + subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/' 'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py index aa20a68..14db2a7 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py @@ -113,28 +113,42 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider): def build_docker_images(image_list, region, datalab_path): try: - host_string = '{}@{}'.format(args.os_user, args.hostname) - if os.environ['conf_cloud_provider'] == 'azure': - conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/' - 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path)) - conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json ' - '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user)) - if region == 'cn-north-1': - add_china_repository(datalab_path) - for image in image_list: - name = image['name'] - tag = image['tag'] - conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json ' - '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name)) - if name == 'base': - conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} " - "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} " - ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path)) - else: - conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} " - "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path)) - conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path)) - return True + if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ: + conn.sudo('sudo docker login -u {0} -p {1} {2}:8083' + .format(os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address'])) + for image in image_list: + name = image['name'] + tag = image['tag'] + conn.sudo('docker pull {}:8083/docker.datalab-{}-{}:{}' + .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag)) + conn.sudo('docker image tag {0}:8083/docker.datalab-{1}-{2}:{3} docker.datalab-{1}-{2}:{3}' + .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag)) + conn.sudo('docker image rm {}:8083/docker.datalab-{}-{}:{}' + .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag)) + return True + else: + host_string = '{}@{}'.format(args.os_user, args.hostname) + if os.environ['conf_cloud_provider'] == 'azure': + conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/' + 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path)) + conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json ' + '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user)) + if region == 'cn-north-1': + add_china_repository(datalab_path) + for image in image_list: + name = image['name'] + tag = image['tag'] + conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json ' + '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name)) + if name == 'base': + conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} " + "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} " + ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path)) + else: + conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} " + "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path)) + conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path)) + return True except: return False --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
