[vcl] 02/04: libvirt.pm: modified power_off: added check for "SIGKILL: Device or resource busy" in output of virsh destroy command - we've experienced times when a domain was considered busy but could be destroyed a few seconds later, this waits 30 seconds and tries again

[jfthomps]

This is an automated email from the ASF dual-hosted git repository.

jfthomps pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/vcl.git

commit 655566bd1fb0c9996e17e25fc2b6fa4c2bd164d5
Author: Josh Thompson <jftho...@ncsu.edu>
AuthorDate: Thu Jul 11 11:51:10 2019 -0400

    libvirt.pm: modified power_off: added check for "SIGKILL: Device or 
resource busy" in output of virsh destroy command - we've experienced times 
when a domain was considered busy but could be destroyed a few seconds later, 
this waits 30 seconds and tries again
---
 .../lib/VCL/Module/Provisioning/libvirt.pm         | 38 ++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/managementnode/lib/VCL/Module/Provisioning/libvirt.pm 
b/managementnode/lib/VCL/Module/Provisioning/libvirt.pm
index bee452d..04cccc6 100644
--- a/managementnode/lib/VCL/Module/Provisioning/libvirt.pm
+++ b/managementnode/lib/VCL/Module/Provisioning/libvirt.pm
@@ -779,6 +779,44 @@ sub power_off {
                notify($ERRORS{'DEBUG'}, 0, "'$domain_name' domain is not 
running on $node_name");
                return 1;
        }
+       elsif (grep(/SIGKILL: Device or resource busy/i, @$output)) {
+               notify($ERRORS{'DEBUG'}, 0, "Failed to stop '$domain_name' 
domain; \"SIGKILL: Device or resource busy\" encountered, waiting 30 seconds 
and trying again");
+               # libvirt could not kill VM, wait and try again
+               sleep 30;
+               my $command2 = "virsh -q list --all";
+               ($exit_status, $output) = $self->vmhost_os->execute($command2);
+               my ($id, $name, $state);
+               for my $line (@$output) {
+                       ($id, $name, $state) = $line =~ 
/^\s*([\d\-]+)\s+(.+?)\s+(\w+|shut off|in shutdown)$/g;
+                       next if (!defined($id));
+                       last if ($name eq $domain_name);
+               }
+               if ($name ne $domain_name) {
+                       notify($ERRORS{'WARNING'}, 0, "2nd try: $domain_name 
not found when running 'virsh list' on $node_name");
+                       return;
+               }
+               if ($id eq '-') {
+                       notify($ERRORS{'DEBUG'}, 0, "'$domain_name' domain now 
stopped");
+                       return 1;
+               }
+               ($exit_status, $output) = $self->vmhost_os->execute($command);
+               if (!defined($output)) {
+                       notify($ERRORS{'WARNING'}, 0, "2nd try: failed to 
execute virsh command to destroy '$domain_name' domain on $node_name");
+                       return;
+               }
+               elsif ($exit_status eq '0') {
+                       notify($ERRORS{'OK'}, 0, "2nd try: destroyed 
'$domain_name' domain on $node_name");
+                       return 1;
+               }
+               elsif (grep(/domain is not running/i, @$output)) {
+                       notify($ERRORS{'DEBUG'}, 0, "2nd try: '$domain_name' 
domain is not running on $node_name");
+                       return 1;
+               }
+               else {
+                       notify($ERRORS{'WARNING'}, 0, "2nd try: failed to 
destroy '$domain_name' domain on $node_name\ncommand: $command\nexit status: 
$exit_status\noutput:\n" . join("\n", @$output));
+                       return;
+               }
+       }
        else {
                notify($ERRORS{'WARNING'}, 0, "failed to destroy '$domain_name' 
domain on $node_name\ncommand: $command\nexit status: $exit_status\noutput:\n" 
. join("\n", @$output));
                return;