Public bug reported: icehouse Ceph as a backend for glance and cinder
when evacuate an instance from failed host to another. the command fails. 2014-06-20 20:21:36.430 12362 ERROR oslo.messaging.rpc.dispatcher [-] Exception during message handling: error removing image 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher Traceback (most recent call last): 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 133, in _dispatch_and_reply 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher incoming.message)) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 176, in _dispatch 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return self._do_dispatch(endpoint, method, ctxt, args) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 122, in _do_dispatch 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher result = getattr(endpoint, method)(ctxt, **new_args) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 88, in wrapped 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher payload) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 71, in wrapped 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return f(self, context, *args, **kw) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 327, in decorated_function 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher function(self, context, *args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 303, in decorated_function 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher e, sys.exc_info()) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 290, in decorated_function 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2251, in terminate_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher do_terminate_instance(instance, bdms) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/lockutils.py", line 249, in inner 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return f(*args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2249, in do_terminate_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self._set_instance_error_state(context, instance['uuid']) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2239, in do_terminate_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher reservations=reservations) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/hooks.py", line 103, in inner 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher rv = f(*args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2209, in _delete_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher user_id=user_id) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2179, in _delete_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self._shutdown_instance(context, db_inst, bdms) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2114, in _shutdown_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher requested_networks) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2104, in _shutdown_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher block_device_info) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 961, in destroy 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher destroy_disks, migrate_data) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 1079, in cleanup 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self._cleanup_rbd(instance) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 1086, in _cleanup_rbd 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher driver.cleanup_volumes(instance) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/rbd_utils.py", line 237, in cleanup_volumes 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self.rbd.RBD().remove(client.ioctx, volume) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/rbd.py", line 300, in remove 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher raise make_ex(ret, 'error removing image') 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher ImageBusy: error removing image 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher ** Affects: nova Importance: Undecided Status: New -- You received this bug notification because you are a member of Yahoo! Engineering Team, which is subscribed to OpenStack Compute (nova). https://bugs.launchpad.net/bugs/1332536 Title: ImageBusy: error removing image. when evacuate on ceph backed volume Status in OpenStack Compute (Nova): New Bug description: icehouse Ceph as a backend for glance and cinder when evacuate an instance from failed host to another. the command fails. 2014-06-20 20:21:36.430 12362 ERROR oslo.messaging.rpc.dispatcher [-] Exception during message handling: error removing image 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher Traceback (most recent call last): 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 133, in _dispatch_and_reply 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher incoming.message)) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 176, in _dispatch 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return self._do_dispatch(endpoint, method, ctxt, args) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 122, in _do_dispatch 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher result = getattr(endpoint, method)(ctxt, **new_args) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 88, in wrapped 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher payload) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 71, in wrapped 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return f(self, context, *args, **kw) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 327, in decorated_function 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher function(self, context, *args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 303, in decorated_function 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher e, sys.exc_info()) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 290, in decorated_function 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2251, in terminate_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher do_terminate_instance(instance, bdms) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/lockutils.py", line 249, in inner 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher return f(*args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2249, in do_terminate_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self._set_instance_error_state(context, instance['uuid']) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2239, in do_terminate_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher reservations=reservations) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/hooks.py", line 103, in inner 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher rv = f(*args, **kwargs) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2209, in _delete_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher user_id=user_id) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2179, in _delete_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self._shutdown_instance(context, db_inst, bdms) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2114, in _shutdown_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher requested_networks) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 2104, in _shutdown_instance 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher block_device_info) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 961, in destroy 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher destroy_disks, migrate_data) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 1079, in cleanup 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self._cleanup_rbd(instance) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 1086, in _cleanup_rbd 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher driver.cleanup_volumes(instance) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/rbd_utils.py", line 237, in cleanup_volumes 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher self.rbd.RBD().remove(client.ioctx, volume) 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/rbd.py", line 300, in remove 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher raise make_ex(ret, 'error removing image') 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher ImageBusy: error removing image 2014-06-20 20:21:36.430 12362 TRACE oslo.messaging.rpc.dispatcher To manage notifications about this bug go to: https://bugs.launchpad.net/nova/+bug/1332536/+subscriptions -- Mailing list: https://launchpad.net/~yahoo-eng-team Post to : yahoo-eng-team@lists.launchpad.net Unsubscribe : https://launchpad.net/~yahoo-eng-team More help : https://help.launchpad.net/ListHelp