This is an automated email from the ASF dual-hosted git repository. opolishchuk pushed a commit to branch DATALAB-3004 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit b9f115d4f0cc1bcc52b390dda87a1a7b491d6f8b Author: Oleksandr Polishchuk <[email protected]> AuthorDate: Fri Sep 2 15:37:02 2022 +0300 [DATALAB-3004]: added checking image status to status checker --- .../src/general/lib/aws/meta_lib.py | 20 ++++++++++++++++++++ .../src/general/lib/gcp/meta_lib.py | 16 ++++++++++++++++ .../src/general/scripts/aws/common_collect_data.py | 5 +++++ .../src/general/scripts/azure/common_collect_data.py | 5 +++++ .../src/general/scripts/gcp/common_collect_data.py | 6 ++++++ 5 files changed, 52 insertions(+) diff --git a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py index 6b6c5cb7a..a6f918b19 100644 --- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py +++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py @@ -754,6 +754,26 @@ def get_list_instance_statuses(instance_ids): return data +def get_list_image_statuses(image_ids, data=[]): + client = boto3.client('ec2') + for k in image_ids: + host = {} + try: + if 'id' in k: + response = client.describe_images(ImageIds=[k.get('id')]).get('Reservations') + for i in response: + img = i.get('Images') + for j in img: + host['id'] = j.get('ImageId') + host['status'] = j.get('State').get('Name') + data.append(host) + except Exception as err: + host['id'] = h.get('id') + host['status'] = 'terminated' + data.append(host) + return data + + def get_list_cluster_statuses(cluster_ids, data=[]): client = boto3.client('emr') for i in cluster_ids: diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py index be5d17b0c..b8f01e764 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py @@ -673,6 +673,22 @@ class GCPMeta: data.append(host) return data + def get_list_image_statuses(self, image_name_list): + data = [] + for image in image_name_list: + host = {} + try: + request = self.service.images().get(project=self.project, image=image) + result = request.execute() + host['id'] = image + host['status'] = result.get('status').lower().replace("terminated", "stopped") + data.append(host) + except: + host['id'] = image + host['status'] = 'terminated' + data.append(host) + return data + def get_cluster(self, cluster_name): try: request = self.dataproc.projects().regions().clusters().get(projectId=self.project, diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py index a81e37313..6cbb2fc9a 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py +++ b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py @@ -50,6 +50,11 @@ if __name__ == "__main__": statuses['cluster'] = data_clusters except: logging.info("Clusters JSON wasn't provided") + try: + data_images = get_list_image_statuses(data.get('image')) + statuses['image'] = data_images + except: + logging.info("Images JSON wasn't been provided") with open('/root/result.json', 'w') as outfile: json.dump(statuses, outfile) except Exception as err: diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py index ee8eda738..3394f041a 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py +++ b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py @@ -46,6 +46,11 @@ if __name__ == "__main__": statuses['host'] = data_instances except: logging.error("Hosts JSON wasn't been provided") + try: + data_images = AzureMeta().list_images(data.get('image')) + statuses['image'] = data_images + except: + logging.error("Images JSON wasn't been provided") with open('/root/result.json', 'w') as outfile: json.dump(statuses, outfile) except Exception as err: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py index 1f8527e48..e54828ac0 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py @@ -62,6 +62,12 @@ if __name__ == "__main__": statuses['cluster'] = data_clusters except: logging.error("Clusters JSON wasn't been provided") + try: + id_images = get_id_resourses(data.get('image')) + data_images = GCPMeta().get_list_image_statuses(id_images) + statuses['image'] = data_images + except: + logging.error("Images JSON wasn't been provided") with open('/root/result.json', 'w') as outfile: json.dump(statuses, outfile) except Exception as err: --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
