This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-3079
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-3079 by this push:
new 96a7be03c [DATALAB-3079]: added unique set of 6 characters at the
beginning of hdinsight cluster name
96a7be03c is described below
commit 96a7be03c94f3a6c45f0bcce9f6498ef9799b8ec
Author: leonidfrolov <[email protected]>
AuthorDate: Wed Oct 19 18:44:25 2022 +0300
[DATALAB-3079]: added unique set of 6 characters at the beginning of
hdinsight cluster name
---
.../src/dataengine-service/fabfile.py | 6 ++++--
.../common_notebook_configure_dataengine-service.py | 10 +++++++---
.../scripts/azure/dataengine-service_configure.py | 17 ++++++++++++-----
.../general/scripts/azure/dataengine-service_prepare.py | 7 ++++++-
.../scripts/azure/dataengine-service_terminate.py | 7 +++++--
5 files changed, 34 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/dataengine-service/fabfile.py
b/infrastructure-provisioning/src/dataengine-service/fabfile.py
index 0bedc256d..8308c7bb4 100644
--- a/infrastructure-provisioning/src/dataengine-service/fabfile.py
+++ b/infrastructure-provisioning/src/dataengine-service/fabfile.py
@@ -27,11 +27,13 @@ import sys
import uuid
import secrets
import subprocess
+
from datalab.actions_lib import *
from datalab.fab import *
from datalab.meta_lib import *
from fabric import *
-
+import random
+import string
@task
def run(ctx):
@@ -42,7 +44,7 @@ def run(ctx):
level=logging.INFO,
filename=local_log_filepath)
dataengine_service_config = dict()
- dataengine_service_config['uuid'] = str(uuid.uuid4())[:5]
+ dataengine_service_config['uuid'] =
'{}{}'.format(random.choice(string.ascii_lowercase), str(uuid.uuid4())[:5])
dataengine_service_config['access_password'] = secrets.token_urlsafe(32)
try:
subprocess.run("~/scripts/{}.py --uuid {} --access_password {}"
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine-service.py
b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine-service.py
index 3bd9c05d1..31e9dca30 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine-service.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine-service.py
@@ -34,7 +34,8 @@ from fabric import *
def clear_resources():
-
AzureActions.terminate_hdinsight_cluster(notebook_config['resource_group_name'],
notebook_config['cluster_name'])
+
AzureActions.terminate_hdinsight_cluster(notebook_config['resource_group_name'],
+
notebook_config['full_cluster_name'])
for storage_account in
AzureMeta.list_storage_accounts(notebook_config['resource_group_name']):
if notebook_config['storage_account_name_tag'] ==
storage_account.tags["Name"]:
AzureActions.remove_storage_account(notebook_config['resource_group_name'],
storage_account.name)
@@ -74,8 +75,11 @@ if __name__ == "__main__":
notebook_config['project_name'], notebook_config['endpoint_tag'])
edge_instance_hostname =
AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
edge_instance_name)
+ for cluster in
AzureMeta.list_hdinsight_clusters(notebook_config['resource_group_name']):
+ if notebook_config['cluster_name'] == cluster.tags["Name"]:
+ notebook_config['full_cluster_name'] = cluster.name
notebook_config['headnode_ip'] =
datalab.fab.get_hdinsight_headnode_private_ip(os.environ['conf_os_user'],
-
notebook_config['cluster_name'],
+
notebook_config['full_cluster_name'],
notebook_config['key_path'])
if os.environ['application'] == 'deeplearning':
@@ -88,7 +92,7 @@ if __name__ == "__main__":
params = "--bucket {} --cluster_name {} --hdinsight_version {}
--keyfile {} --notebook_ip {} --region {} " \
"--project_name {} --os_user {} --edge_hostname {}
--proxy_port {} " \
"--scala_version {} --application {} --headnode_ip {}" \
- .format(notebook_config['storage_account_name_tag'],
notebook_config['cluster_name'], os.environ['hdinsight_version'],
+ .format(notebook_config['storage_account_name_tag'],
notebook_config['full_cluster_name'], os.environ['hdinsight_version'],
notebook_config['key_path'],
notebook_config['notebook_ip'], os.environ['gcp_region'],
notebook_config['project_name'],
os.environ['conf_os_user'],
edge_instance_hostname, '3128',
os.environ['notebook_scala_version'], os.environ['application'],
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py
index 78841d65a..5ceb8c660 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py
@@ -72,12 +72,18 @@ if __name__ == "__main__":
hdinsight_conf['computational_name'] =
os.environ['computational_name'].lower().replace('_', '-')
else:
hdinsight_conf['computational_name'] = ''
+
hdinsight_conf['cluster_name'] =
'{}-{}-{}-des-{}'.format(hdinsight_conf['service_base_name'],
hdinsight_conf['project_name'],
hdinsight_conf['endpoint_name'],
hdinsight_conf['computational_name'])
- hdinsight_conf["instance_id"] = hdinsight_conf["cluster_name"]
- hdinsight_conf['cluster_url'] =
'https://{}.azurehdinsight.net'.format(hdinsight_conf['cluster_name'])
+
+ hdinsight_conf['full_cluster_name'] =
'{}-{}-{}-{}-des-{}'.format(args.uuid, hdinsight_conf['service_base_name'],
+
hdinsight_conf['project_name'],
+
hdinsight_conf['endpoint_name'],
+
hdinsight_conf['computational_name'])
+ hdinsight_conf["instance_id"] = hdinsight_conf["full_cluster_name"]
+ hdinsight_conf['cluster_url'] =
'https://{}.azurehdinsight.net'.format(hdinsight_conf['full_cluster_name'])
hdinsight_conf['cluster_jupyter_url'] =
'{}/jupyter/'.format(hdinsight_conf['cluster_url'])
hdinsight_conf['cluster_sparkhistory_url'] =
'{}/sparkhistory/'.format(hdinsight_conf['cluster_url'])
hdinsight_conf['cluster_zeppelin_url'] =
'{}/zeppelin/'.format(hdinsight_conf['cluster_url'])
@@ -98,7 +104,7 @@ if __name__ == "__main__":
logging.info('[SUMMARY]')
logging.info("Service base name:
{}".format(hdinsight_conf['service_base_name']))
logging.info("Region: {}".format(hdinsight_conf['region']))
- logging.info("Cluster name: {}".format(hdinsight_conf['cluster_name']))
+ logging.info("Cluster name:
{}".format(hdinsight_conf['full_cluster_name']))
logging.info("Master node shape:
{}".format(hdinsight_conf['hdinsight_master_instance_type']))
logging.info("Slave node shape:
{}".format(hdinsight_conf['hdinsight_slave_instance_type']))
logging.info("Instance count:
{}".format(str(os.environ['hdinsight_count'])))
@@ -110,7 +116,7 @@ if __name__ == "__main__":
logging.info("Zeppelin URL:
{}".format(hdinsight_conf['cluster_zeppelin_url']))
with open("/root/result.json", 'w') as result:
- res = {"hostname": hdinsight_conf['cluster_name'],
+ res = {"hostname": hdinsight_conf['full_cluster_name'],
"key_name": hdinsight_conf['key_name'],
"instance_id": hdinsight_conf["instance_id"],
"Action": "Create new HDInsight cluster",
@@ -131,5 +137,6 @@ if __name__ == "__main__":
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Error with writing results", str(err))
-
AzureActions.terminate_hdinsight_cluster(hdinsight_conf['resource_group_name'],
hdinsight_conf['cluster_name'])
+
AzureActions.terminate_hdinsight_cluster(hdinsight_conf['resource_group_name'],
+
hdinsight_conf['full_cluster_name'])
sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index c57d0ff75..112d2d573 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -67,6 +67,11 @@ if __name__ == "__main__":
hdinsight_conf['project_name'],
hdinsight_conf['endpoint_name'],
hdinsight_conf['computational_name'])
+ hdinsight_conf['full_cluster_name'] =
'{}-{}-{}-{}-des-{}'.format(args.uuid,
+
hdinsight_conf['service_base_name'],
+
hdinsight_conf['project_name'],
+
hdinsight_conf['endpoint_name'],
+
hdinsight_conf['computational_name'])
hdinsight_conf['cluster_tags'] = {
"Name": hdinsight_conf['cluster_name'],
@@ -172,7 +177,7 @@ if __name__ == "__main__":
"--tags '{}' --public_key '{}' --vpc_id {} --subnet {}
--access_password {} " \
"--edge_storage_account_name {} --edge_storage_account_key
'{}' --edge_container_name {} " \
"--shared_storage_account_name {}
--shared_storage_account_key '{}' --shared_container_name {}"\
- .format(hdinsight_conf['resource_group_name'],
hdinsight_conf['cluster_name'],
+ .format(hdinsight_conf['resource_group_name'],
hdinsight_conf['full_cluster_name'],
hdinsight_conf['release_label'], hdinsight_conf['region'],
hdinsight_conf['hdinsight_master_instance_type'],
hdinsight_conf['hdinsight_slave_instance_type'],
hdinsight_conf['hdinsight_worker_count'],
hdinsight_conf['cluster_storage_account_name'],
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py
index 4a5a88b98..bbfddf491 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py
@@ -66,11 +66,14 @@ if __name__ == "__main__":
try:
logging.info('[TERMINATE HDINSIGHT CLUSTER AND ASSOCIATED RESOURCES]')
try:
+ for cluster in
AzureMeta.list_hdinsight_clusters(hdinsight_conf['resource_group_name']):
+ if hdinsight_conf['cluster_name'] == cluster.tags["Name"]:
+ hdinsight_conf['full_cluster_name'] = cluster.name
cluster =
AzureMeta.get_hdinsight_cluster(hdinsight_conf['resource_group_name'],
-
hdinsight_conf['cluster_name'])
+
hdinsight_conf['full_cluster_name'])
if cluster and cluster.properties.cluster_state == 'Running':
AzureActions.terminate_hdinsight_cluster(hdinsight_conf['resource_group_name'],
-
hdinsight_conf['cluster_name'])
+
hdinsight_conf['full_cluster_name'])
for storage_account in
AzureMeta.list_storage_accounts(hdinsight_conf['resource_group_name']):
if hdinsight_conf['storage_account_name_tag'] ==
storage_account.tags["Name"]:
AzureActions.remove_storage_account(hdinsight_conf['resource_group_name'],
storage_account.name)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
