Repository: libcloud Updated Branches: refs/heads/trunk 42709622c -> 0c5bee8b1
Limit number of retries in destroy_node. Signed-off-by: Quentin Pradet <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a71e9557 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a71e9557 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a71e9557 Branch: refs/heads/trunk Commit: a71e95579dd461dd180c798091bce1216924165c Parents: 4270962 Author: Peter Amstutz <[email protected]> Authored: Mon Oct 16 16:12:45 2017 -0400 Committer: Quentin Pradet <[email protected]> Committed: Wed Nov 8 07:32:04 2017 +0400 ---------------------------------------------------------------------- libcloud/compute/drivers/azure_arm.py | 38 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/a71e9557/libcloud/compute/drivers/azure_arm.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/azure_arm.py b/libcloud/compute/drivers/azure_arm.py index fcc9ef3..db7ef65 100644 --- a/libcloud/compute/drivers/azure_arm.py +++ b/libcloud/compute/drivers/azure_arm.py @@ -691,7 +691,10 @@ class AzureNodeDriver(NodeDriver): else: return False - def destroy_node(self, node, ex_destroy_nic=True, ex_destroy_vhd=True): + def destroy_node(self, node, + ex_destroy_nic=True, + ex_destroy_vhd=True, + ex_retries=10): """ Destroy a node. @@ -706,6 +709,10 @@ class AzureNodeDriver(NodeDriver): this node (default True). :type node: ``bool`` + :param ex_retries: Number of times to retry checking if the node is gone, + destroying the NIC or destroying the VHD. + :type node: ``int`` + :return: True if the destroy was successful, raises exception otherwise. :rtype: ``bool`` @@ -733,12 +740,14 @@ class AzureNodeDriver(NodeDriver): # Poll until the node actually goes away (otherwise attempt to delete # NIC and VHD will fail with "resource in use" errors). - while do_node_polling: + retries = ex_retries + while do_node_polling and retries > 0: try: time.sleep(10) self.connection.request( node.id, params={"api-version": RESOURCE_API_VERSION}) + retries -= 1 except BaseHTTPError as h: if h.code in (204, 404): # Node is gone @@ -752,13 +761,16 @@ class AzureNodeDriver(NodeDriver): node.extra["properties"]["networkProfile"]["networkInterfaces"] if ex_destroy_nic: for nic in interfaces: - while True: + retries = ex_retries + while retries > 0: try: self.ex_destroy_nic(self._to_nic(nic)) break except BaseHTTPError as h: + retries -= 1 if (h.code == 400 and - h.message.startswith("[NicInUse]")): + h.message.startswith("[NicInUse]") and + retries > 0): time.sleep(10) else: raise @@ -767,19 +779,23 @@ class AzureNodeDriver(NodeDriver): vhd = node.extra["properties"]["storageProfile"]["osDisk"].get("vhd") if ex_destroy_vhd and vhd is not None: resourceGroup = node.id.split("/")[4] - while True: + retries = ex_retries + while retries > 0: try: - if self._ex_delete_old_vhd( - resourceGroup, - vhd["uri"]): + if self._ex_delete_old_vhd(resourceGroup, vhd["uri"]): break # Unfortunately lease errors usually result in it returning # "False" with no more information. Need to wait and try # again. except LibcloudError as e: - if "LeaseIdMissing" in str(e): - # If we get an lease error, need to wait and try again. - pass + retries -= 1 + if "LeaseIdMissing" in str(e) and retries > 0: + # Unfortunately lease errors + # (which occur if the vhd blob + # hasn't yet been released by the VM being destroyed) + # get raised as plain + # LibcloudError. Wait a bit and try again. + time.sleep(10) else: raise time.sleep(10)
