Public bug reported:
When removing guests that have been booted on Ceph, Nova will
occasionally put guests in to ERROR state with the following ...
Reported to the controller:
| fault | {"message": "error removing image",
"code": 500, "details": " File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 314, in
decorated_function |
| | return function(self, context,
*args, **kwargs)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2525, in
terminate_instance |
| | do_terminate_instance(instance,
bdms)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py\", line
272, in inner |
| | return f(*args, **kwargs)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2523, in
do_terminate_instance |
| |
self._set_instance_error_state(context, instance)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line
82, in __exit__ |
| | six.reraise(self.type_,
self.value, self.tb)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2513, in
do_terminate_instance |
| | self._delete_instance(context,
instance, bdms, quotas)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/hooks.py\", line 131, in inner
|
| | rv = f(*args, **kwargs)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2482, in
_delete_instance |
| | quotas.rollback()
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line
82, in __exit__ |
| | six.reraise(self.type_,
self.value, self.tb)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2459, in
_delete_instance |
| | self._shutdown_instance(context,
instance, bdms)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2389, in
_shutdown_instance |
| | requested_networks)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line
82, in __exit__ |
| | six.reraise(self.type_,
self.value, self.tb)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2378, in
_shutdown_instance |
| | block_device_info)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1058, in
destroy |
| | destroy_disks, migrate_data)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1173, in
cleanup |
| | self._cleanup_rbd(instance)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1218, in
_cleanup_rbd |
| |
LibvirtDriver._get_rbd_driver().cleanup_volumes(instance)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/rbd_utils.py\", line 266,
in cleanup_volumes |
| | rbd.RBD().remove(client.ioctx,
volume)
|
| | File
\"/usr/lib/python2.7/site-packages/rbd.py\", line 300, in remove
|
| | raise make_ex(ret, 'error removing
image')
|
| | ", "created": "2015-03-25T14:17:14Z"}
in nova-compute.log:
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 314, in
decorated_function
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher return
function(self, context, *args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2525, in
terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
do_terminate_instance(instance, bdms)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py", line
272, in inner
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher return
f(*args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2523, in
do_terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._set_instance_error_state(context, instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82,
in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2513, in
do_terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._delete_instance(context, instance, bdms, quotas)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/hooks.py", line 131, in inner
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher rv =
f(*args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2482, in
_delete_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
quotas.rollback()
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82,
in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2459, in
_delete_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._shutdown_instance(context, instance, bdms)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2389, in
_shutdown_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
requested_networks)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82,
in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2378, in
_shutdown_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
block_device_info)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1058, in
destroy
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
destroy_disks, migrate_data)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1173, in
cleanup
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._cleanup_rbd(instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1218, in
_cleanup_rbd
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
LibvirtDriver._get_rbd_driver().cleanup_volumes(instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/rbd_utils.py", line 266, in
cleanup_volumes
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
rbd.RBD().remove(client.ioctx, volume)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/rbd.py", line 300, in remove
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher raise
make_ex(ret, 'error removing image')
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher ImageBusy:
error removing image
** Affects: nova
Importance: Undecided
Status: New
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1438331
Title:
Nova fails to delete rbd image, puts guest in to ERROR state
Status in OpenStack Compute (Nova):
New
Bug description:
When removing guests that have been booted on Ceph, Nova will
occasionally put guests in to ERROR state with the following ...
Reported to the controller:
| fault | {"message": "error removing image",
"code": 500, "details": " File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 314, in
decorated_function |
| | return function(self, context,
*args, **kwargs)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2525, in
terminate_instance |
| | do_terminate_instance(instance,
bdms)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py\", line
272, in inner |
| | return f(*args, **kwargs)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2523, in
do_terminate_instance |
| |
self._set_instance_error_state(context, instance)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line
82, in __exit__ |
| | six.reraise(self.type_,
self.value, self.tb)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2513, in
do_terminate_instance |
| | self._delete_instance(context,
instance, bdms, quotas)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/hooks.py\", line 131, in inner
|
| | rv = f(*args, **kwargs)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2482, in
_delete_instance |
| | quotas.rollback()
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line
82, in __exit__ |
| | six.reraise(self.type_,
self.value, self.tb)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2459, in
_delete_instance |
| | self._shutdown_instance(context,
instance, bdms)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2389, in
_shutdown_instance |
| | requested_networks)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line
82, in __exit__ |
| | six.reraise(self.type_,
self.value, self.tb)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2378, in
_shutdown_instance |
| | block_device_info)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1058, in
destroy |
| | destroy_disks, migrate_data)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1173, in
cleanup |
| | self._cleanup_rbd(instance)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1218, in
_cleanup_rbd |
| |
LibvirtDriver._get_rbd_driver().cleanup_volumes(instance)
|
| | File
\"/usr/lib/python2.7/site-packages/nova/virt/libvirt/rbd_utils.py\", line 266,
in cleanup_volumes |
| | rbd.RBD().remove(client.ioctx,
volume)
|
| | File
\"/usr/lib/python2.7/site-packages/rbd.py\", line 300, in remove
|
| | raise make_ex(ret, 'error
removing image')
|
| | ", "created":
"2015-03-25T14:17:14Z"}
in nova-compute.log:
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 314, in
decorated_function
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher return
function(self, context, *args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2525, in
terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
do_terminate_instance(instance, bdms)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py", line
272, in inner
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher return
f(*args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2523, in
do_terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._set_instance_error_state(context, instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82,
in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2513, in
do_terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._delete_instance(context, instance, bdms, quotas)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/hooks.py", line 131, in inner
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher rv =
f(*args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2482, in
_delete_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
quotas.rollback()
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82,
in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2459, in
_delete_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._shutdown_instance(context, instance, bdms)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2389, in
_shutdown_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
requested_networks)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82,
in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2378, in
_shutdown_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
block_device_info)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1058, in
destroy
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
destroy_disks, migrate_data)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1173, in
cleanup
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
self._cleanup_rbd(instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1218, in
_cleanup_rbd
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
LibvirtDriver._get_rbd_driver().cleanup_volumes(instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/nova/virt/libvirt/rbd_utils.py", line 266, in
cleanup_volumes
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher
rbd.RBD().remove(client.ioctx, volume)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.7/site-packages/rbd.py", line 300, in remove
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher raise
make_ex(ret, 'error removing image')
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher ImageBusy:
error removing image
To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1438331/+subscriptions
--
Mailing list: https://launchpad.net/~yahoo-eng-team
Post to : yahoo-eng-team@lists.launchpad.net
Unsubscribe : https://launchpad.net/~yahoo-eng-team
More help : https://help.launchpad.net/ListHelp
