This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DLAB-1430 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 9b7f10bf8c72852712165ad99fd58e6e61292e95 Author: Mykola_Bodnar1 <bodnarmyk...@gmail.com> AuthorDate: Wed Feb 5 12:46:33 2020 +0200 [DLAB-1430]: Post-deployment configuration scripts for SSN prepared --- .../scripts/POST_DEPLOYMENT.md | 42 ++++++++++ ...uration.py => post-deployment_configuration.py} | 96 +++++++++++++++++----- .../scripts/post_deployment_configuration.sh | 56 ------------- .../src/ssn/scripts/docker_build.py | 4 +- 4 files changed, 119 insertions(+), 79 deletions(-) diff --git a/infrastructure-provisioning/scripts/POST_DEPLOYMENT.md b/infrastructure-provisioning/scripts/POST_DEPLOYMENT.md new file mode 100644 index 0000000..42d826f --- /dev/null +++ b/infrastructure-provisioning/scripts/POST_DEPLOYMENT.md @@ -0,0 +1,42 @@ +### Prerequisites for DLab post-deployment + +- Service account with following roles: +``` +Compute Admin +Compute Network Admin +Dataproc Administrator +Role Administrator +Service Account Admin +Service Account User +Project IAM Admin +Storage Admin +``` +- Google Cloud Storage JSON API should be enabled +- Keycloak server with specific client for Dlab UI (could be dpeloyed with Kecylaok deployment script) + +Service account should be created manually and attached to the instance with post-deployment script. + +### Executing post-deployment script + +To configure SSN node, following steps should be executed: + +- Connect to the instance via SSH and run the following commands: +``` +/usr/bin/python /opt/dlab/sources/infrastructure-provisioning/scripts/post-deployment_configuration.py + --keycloak_realm_name <value> + --keycloak_auth_server_url <value> + --keycloak_client_name <value> + --keycloak_client_secret <value> + --keycloak_user <value> + --keycloak_admin_password <value> +``` + +List of parameters for SSN node post-deployment script: +| Parameter | Description/Value | +|-------------------------------|-------------------------------------------------------------------------------------| +| keycloak\_realm\_name | Keycloak realm name | +| keycloak\_auth\_server\_url | Url of Keycloak auth server | +| keycloak\_client\_name | Name of client for Dlab UI | +| keycloak\_client\_secret | Secret of client for Dlab UI | +| kkeycloak\_user | Keycloak user with administrator permissions | +| keycloak\_admin\_password | Password for Keycloak user with administrator permissions | \ No newline at end of file diff --git a/infrastructure-provisioning/scripts/post_deployment_configuration.py b/infrastructure-provisioning/scripts/post-deployment_configuration.py similarity index 60% rename from infrastructure-provisioning/scripts/post_deployment_configuration.py rename to infrastructure-provisioning/scripts/post-deployment_configuration.py index 5e5271e..2be6807 100644 --- a/infrastructure-provisioning/scripts/post_deployment_configuration.py +++ b/infrastructure-provisioning/scripts/post-deployment_configuration.py @@ -24,17 +24,19 @@ from fabric.api import * import argparse import requests +import uuid +from Crypto.PublicKey import RSA if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--keycloak_realm_name', type=str, default='dlab', help='Keycloak Realm name') - parser.add_argument('--keycloak_auth_server_url', type=str, default='dlab', help='Keycloak auth server URL') - parser.add_argument('--keycloak_client_name', type=str, default='dlab', help='Keycloak client name') - parser.add_argument('--keycloak_client_secret', type=str, default='dlab', help='Keycloak client secret') - parser.add_argument('--keycloak_user', type=str, default='dlab', help='Keycloak user') - parser.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password', - help='Keycloak user password') + parser.add_argument('--keycloak_realm_name', type=str, default='KEYCLOAK_REALM_NAME', help='Keycloak Realm name') + parser.add_argument('--keycloak_auth_server_url', type=str, default='KEYCLOAK_AUTH_SERVER_URL', help='Keycloak auth server URL') + parser.add_argument('--keycloak_client_name', type=str, default='KEYCLOAK_CLIENT_NAME', help='Keycloak client name') + parser.add_argument('--keycloak_client_secret', type=str, default='KEYCLOAK_CLIENT_SECRET', help='Keycloak client secret') + parser.add_argument('--keycloak_user', type=str, default='KEYCLOAK_USER', help='Keycloak user') + parser.add_argument('--keycloak_admin_password', type=str, default='KEYCLOAK_ADMIN_PASSWORD', + help='Keycloak admin password') args = parser.parse_args() headers = { 'Metadata-Flavor': 'Google', @@ -51,6 +53,25 @@ if __name__ == "__main__": gcp_projectId = requests.get('http://metadata/computeMetadata/v1/project/project-id', headers=headers).text keycloak_redirectUri = 'http://{}'.format(server_external_ip) + print("Generationg SSH keyfile for dlab-user") + key = RSA.generate(2048) + local("sudo sh -c 'echo \"{}\" >> /home/dlab-user/keys/KEY-FILE.pem'".format(key.exportKey('PEM'))) + local("sudo chmod 600 /home/dlab-user/keys/KEY-FILE.pem") + pubkey = key.publickey() + local("sudo sh -c 'echo \"{}\" >> /home/dlab-user/.ssh/authorized_keys'".format(pubkey.exportKey('OpenSSH'))) + + print("Generationg MongoDB password") + mongo_pwd = uuid.uuid4().hex + try: + local("sudo echo -e 'db.changeUserPassword(\"admin\", \"{}\")' | mongo dlabdb --port 27017 -u admin -p MONGO_PASSWORD".format( + mongo_pwd)) + local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/dlab/conf/billing.yml'.format(mongo_pwd)) + + local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/dlab/conf/ssn.yml'.format(mongo_pwd)) + except: + print('Mongo password was already changed') + + print('Reserving external IP') static_address_exist = local( "sudo gcloud compute addresses list --filter='address={}'".format(server_external_ip), capture=True) @@ -68,13 +89,15 @@ if __name__ == "__main__": local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/conf/self-service.yml'.format( args.keycloak_auth_server_url)) local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/conf/self-service.yml'.format(args.keycloak_client_name)) - local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/self-service.yml'.format(args.keycloak_client_secret)) + local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/self-service.yml'.format( + args.keycloak_client_secret)) local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_realm_name)) local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/conf/provisioning.yml'.format( args.keycloak_auth_server_url)) local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_client_name)) - local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_client_secret)) + local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/provisioning.yml'.format( + args.keycloak_client_secret)) local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/conf/provisioning.yml'.format(dlab_sbn)) local('sudo sed -i "s|SUBNET_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(deployment_subnetId)) local('sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/conf/provisioning.yml'.format(dlab_region)) @@ -82,20 +105,49 @@ if __name__ == "__main__": local('sudo sed -i "s|SSN_VPC_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(deployment_vpcId)) local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(gcp_projectId)) local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_user)) - local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_user_password)) - - local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_sbn)) - local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(gcp_projectId)) - local('sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_region)) - local('sudo sed -i "s|DLAB_ZONE|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_zone)) - local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_realm_name)) - local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_auth_server_url)) - local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_name)) - local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_secret)) - local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_user)) - local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_user_password)) + local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/conf/provisioning.yml'.format( + args.keycloak_admin_password)) + + local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/conf/billing.yml'.format(dlab_sbn)) + + local( + 'sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + dlab_sbn)) + local( + 'sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + gcp_projectId)) + local( + 'sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + dlab_region)) + local( + 'sudo sed -i "s|DLAB_ZONE|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + dlab_zone)) + local( + 'sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + args.keycloak_realm_name)) + local( + 'sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + args.keycloak_auth_server_url)) + local( + 'sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + args.keycloak_client_name)) + local( + 'sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + args.keycloak_client_secret)) + local( + 'sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + args.keycloak_user)) + local( + 'sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format( + args.keycloak_admin_password)) local('sudo sed -i "s|SERVER_IP|{}|g" /etc/nginx/conf.d/nginx_proxy.conf'.format(server_external_ip)) local('sudo systemctl restart nginx') local('sudo supervisorctl restart all') - local('cd /opt/dlab/sources/infrastructure-provisioning/src/ && docker-build all') \ No newline at end of file + local('cd /opt/dlab/sources/infrastructure-provisioning/src/ && sudo docker-build all') + + print('SUMMARY') + print('Mongo password stored in /opt/dlab/conf/ssn.yml') + print('SSH key for dlab-user stored in /home/dlab-user/keys/KEY-FILE.pem') + if not args: + print('Keycloak parameters was not set, please configure Keycloak parameters manually') diff --git a/infrastructure-provisioning/scripts/post_deployment_configuration.sh b/infrastructure-provisioning/scripts/post_deployment_configuration.sh deleted file mode 100644 index 234e108..0000000 --- a/infrastructure-provisioning/scripts/post_deployment_configuration.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -server_external_ip=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip) -sed -i "s|SERVER_IP|$server_external_ip|g" /etc/nginx/conf.d/nginx_proxy.conf -systemctl restart nginx - -dlab_sbn=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/name) - -KEYCLOAK_REDIRECTURI='http://'$server_external_ip -KEYCLOAK_REALM_NAME='dlab' -KEYCLOAK_AUTH_SERVER_URL='https://idp.demo.dlabanalytics.com/auth' -KEYCLOAK_CLIENT_NAME=$dlab_sbn'-ui' -KEYCLOAK_CLIENT_SECRET='e235f2b6-a5e0-448a-837d-465d1a4990f7' -KEYCLOAK_USER='admin' -KEYCLOAK_USER_PASSWORD='v7rdj2ckHgAdJj54' - -sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/self-service.yml -sed -i "s|KEYCLOAK_REDIRECTURI|$KEYCLOAK_REDIRECTURI|g" /opt/dlab/conf/self-service.yml -sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/self-service.yml -sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/self-service.yml -sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/self-service.yml -sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/self-service.yml -sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/provisioning.yml -sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/provisioning.yml -sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/provisioning.yml -sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/provisioning.yml - -ssn_subnetId=$(sudo gcloud compute instances describe $dlab_sbn --zone us-west1-a | awk -F/ '/subnetwork: / {print $11}') -dlab_zone=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/zone | awk -F/ '{print $4}') -dlab_region=$(echo $dlab_zone | awk '{print substr($0, 1, length($0)-2)}') -ssn_vpcId=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | awk -F/ '{print $4}') -gcp_projectId=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/project/project-id) - -sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml -sed -i "s|SUBNET_ID|$ssn_subnetId|g" /opt/dlab/conf/provisioning.yml -sed -i "s|DLAB_REGION|$dlab_region|g" /opt/dlab/conf/provisioning.yml -sed -i "s|DLAB_ZONE|$dlab_zone|g" /opt/dlab/conf/provisioning.yml -sed -i "s|SSN_VPC_ID|$ssn_vpcId|g" /opt/dlab/conf/provisioning.yml -sed -i "s|GCP_PROJECT_ID|$gcp_projectId|g" /opt/dlab/conf/provisioning.yml -sed -i "s|KEYCLOAK_USER|$KEYCLOAK_USER|g" /opt/dlab/conf/provisioning.yml -sed -i "s|KEYCLOAK_USER_PASSWORD|$KEYCLOAK_USER_PASSWORD|g" /opt/dlab/conf/provisioning.yml - -sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|GCP_PROJECT_ID|$gcp_projectId|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|DLAB_REGION|$dlab_region|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|DLAB_ZONE|$dlab_zone|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|KEYCLOAK_USER|$KEYCLOAK_USER|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini -sed -i "s|KEYCLOAK_USER_PASSWORD|$KEYCLOAK_USER_PASSWORD|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini - -supervisorctl restart all - -cd /opt/dlab/sources/infrastructure-provisioning/src/ && docker-build all \ No newline at end of file diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py index 73b5a1d..ac4fee5 100644 --- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py +++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py @@ -40,7 +40,9 @@ if sys.argv[1] == 'all': 'tensor', 'tensor-rstudio', 'deeplearning', - 'dataengine' + 'dataengine', + 'dataengine-service', + 'superset' ] else: node = sys.argv[1:] --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org