This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch epm-v2.5.2.1 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit df4b173d769d201f3c5631d739b1034005eb9eaf Author: leonidfrolov <[email protected]> AuthorDate: Wed Oct 12 11:31:50 2022 +0300 fixed conflict --- .../src/base/scripts/configure_keycloak.py | 6 +- .../src/general/conf/datalab.ini | 4 +- .../scripts/gcp/common_terminate_notebook.py | 44 +++++++++++++ .../src/general/scripts/gcp/jupyter_configure.py | 72 +++++++++++----------- 4 files changed, 88 insertions(+), 38 deletions(-) diff --git a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py index 516a8ab52..449177c0a 100644 --- a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py +++ b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py @@ -94,7 +94,11 @@ if __name__ == "__main__": if not args.exploratory_name: keycloak_client_data["redirectUris"] = keycloak_redirectUris - if args.exploratory_name or not args.project_name: + if not args.project_name: + keycloak_client_data["serviceAccountsEnabled"] = "true" + + if args.exploratory_name: + keycloak_client_data["standardFlowEnabled"] = "false" keycloak_client_data["serviceAccountsEnabled"] = "true" try: diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini index 0cda67a6c..620fd78f1 100644 --- a/infrastructure-provisioning/src/general/conf/datalab.ini +++ b/infrastructure-provisioning/src/general/conf/datalab.ini @@ -372,7 +372,9 @@ nbconvert_version = 5.6.1 ### nbformat_version nbformat_version = 5.3.0 ### jupyterlab version -jupyterlab_version = 3.2.9 +jupyterlab_version = 3.4.3 +### jupyter keycloak client creation +create_keycloak_client = False #--- [emr] section contains all parameters that are using for emr provisioning ---# [emr] diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py index db40b05e2..5acc11abc 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py @@ -25,6 +25,7 @@ import datalab.actions_lib import datalab.fab import datalab.meta_lib import json +import requests from datalab.logger import logging import os import sys @@ -73,6 +74,45 @@ def terminate_nb(instance_name, bucket_name, region, zone, user_name): except Exception as err: datalab.fab.append_result("Failed to terminate instance", str(err)) sys.exit(1) + + if os.environ['notebook_create_keycloak_client'] == 'True': + logging.info("Terminating notebook keycloak client") + try: + keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format( + os.environ['keycloak_auth_server_url']) + keycloak_client_url = '{0}/admin/realms/{1}/clients'.format(os.environ['keycloak_auth_server_url'], + os.environ['keycloak_realm_name']) + + keycloak_auth_data = { + "username": os.environ['keycloak_user'], + "password": os.environ['keycloak_user_password'], + "grant_type": "password", + "client_id": "admin-cli", + } + + client_params = { + "clientId": "{}-{}-{}-{}".format(notebook_config['service_base_name'], notebook_config['project_name'], + notebook_config['endpoint_name'], notebook_config['exploratory_name']) + } + + keycloak_token = requests.post(keycloak_auth_server_url, data=keycloak_auth_data).json() + + keycloak_get_id_client = requests.get(keycloak_client_url, data=keycloak_auth_data, params=client_params, + headers={"Authorization": "Bearer " + keycloak_token.get("access_token"), + "Content-Type": "application/json"}) + json_keycloak_client_id = json.loads(keycloak_get_id_client.text) + keycloak_id_client = json_keycloak_client_id[0]['id'] + + keycloak_client_delete_url = '{0}/admin/realms/{1}/clients/{2}'.format(os.environ['keycloak_auth_server_url'], + os.environ['keycloak_realm_name'], + keycloak_id_client) + + requests.delete(keycloak_client_delete_url, + headers={"Authorization": "Bearer " + keycloak_token.get("access_token"), + "Content-Type": "application/json"}) + except Exception as err: + logging.error("Failed to remove project client from Keycloak", str(err)) + if __name__ == "__main__": @@ -91,6 +131,10 @@ if __name__ == "__main__": notebook_config['endpoint_name']) notebook_config['gcp_region'] = os.environ['gcp_region'] notebook_config['gcp_zone'] = os.environ['gcp_zone'] + try: + notebook_config['exploratory_name'] = (os.environ['exploratory_name']).replace('_', '-').lower() + except: + notebook_config['exploratory_name'] = '' try: logging.info('[TERMINATE NOTEBOOK]') diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py index 5e972b84e..4fa87fb5c 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py @@ -206,46 +206,46 @@ if __name__ == "__main__": datalab.fab.append_result("Failed to setup git credentials.", str(err)) GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone']) sys.exit(1) - - try: - logging.info('[SETUP KEYCLOAK CLIENT]') - notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\ - .format(notebook_config['service_base_name'], notebook_config['project_name'], - notebook_config['endpoint_name'], notebook_config['exploratory_name']) - notebook_config['keycloak_client_secret'] = str(uuid.uuid4()) - keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \ - "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \ - "--project_name {} --endpoint_name {} --exploratory_name {}"\ - .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'], - os.environ['keycloak_realm_name'], os.environ['keycloak_user'], - os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'], - notebook_config['project_name'], notebook_config['endpoint_name'], - notebook_config['exploratory_name']) - try: - subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True) - except: - datalab.fab.append_result("Failed setup keycloak client") - raise Exception + if os.environ['notebook_create_keycloak_client'] == 'True': try: - conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'], - notebook_config['ssh_key_path'], '', False) + logging.info('[SETUP KEYCLOAK CLIENT]') + notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\ + .format(notebook_config['service_base_name'], notebook_config['project_name'], + notebook_config['endpoint_name'], notebook_config['exploratory_name']) + notebook_config['keycloak_client_secret'] = str(uuid.uuid4()) + keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \ + "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \ + "--project_name {} --endpoint_name {} --exploratory_name {}"\ + .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'], + os.environ['keycloak_realm_name'], os.environ['keycloak_user'], + os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'], + notebook_config['project_name'], notebook_config['endpoint_name'], + notebook_config['exploratory_name']) + try: + subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True) + except: + datalab.fab.append_result("Failed setup keycloak client") + raise Exception - with open("/home/datalab-user/template.json") as py3kernel: - content = json.loads(py3kernel.read()) - content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name'] - content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret'] - print(content['env']) - with open("/home/datalab-user/template.json", 'w') as py3kernel: - py3kernel.write(json.dumps(content)) - except: - datalab.fab.append_result("Failed to write variables to .bashrc") - raise Exception + try: + conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'], + notebook_config['ssh_key_path'], '', False) + content = json.loads(conn.sudo("cat /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json" + .format(notebook_config['datalab_ssh_user'])).stdout) + content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name'] + content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret'] + conn.sudo("echo '{}' > /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json" + .format(json.dumps(content), notebook_config['datalab_ssh_user'])) + conn.sudo('systemctl restart jupyter-notebook') + except: + datalab.fab.append_result("Failed to write variables to .bashrc") + raise Exception - except Exception as err: - datalab.fab.append_result("Failed setup keycloak client ", str(err)) - GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone']) - sys.exit(1) + except Exception as err: + datalab.fab.append_result("Failed setup keycloak client ", str(err)) + GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) if notebook_config['image_enabled'] == 'true': try: --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
