Public bug reported: Glance api returns 500 Internal Server Error, if db deadlock occurs in glance-registry for some reason while deleting an image. The image 'status' is set to deleted and 'deleted' is set to False. As deleted is still False, the image is visible in image list but it can not be deleted any more.
If you try to delete this image it will raise 404 (Not Found) error for V1 api and 500 (HTTPInternalServerError) for V2 api. Note: To reproduce this issue I've explicitly raised "db_exception.DBDeadlock" exception from "_image_child_entry_delete_all" method under "\glance\db\sqlalchemy\api.py". glance-api.log -------------- 2014-10-06 00:53:10.037 6827 INFO glance.registry.client.v1.client [2b47d213-6f80-410f-9766-dc80607f0224 7e7c3a413f184dbcb9a65404dbfcc0f0 309c5ff4082c423 1bcc17d8c55c83997 - - -] Registry client request DELETE /images/f9f8a40d-530b-498c-9fbc-86f29da555f4 raised ServerError 2014-10-06 00:53:10.045 6827 INFO glance.wsgi.server [2b47d213-6f80-410f-9766-dc80607f0224 7e7c3a413f184dbcb9a65404dbfcc0f0 309c5ff4082c4231bcc17d8c55c83997 - - -] Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/eventlet/wsgi.py", line 433, in handle_one_response result = self.application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/osprofiler/web.py", line 106, in __call__ return request.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/keystonemiddleware/auth_token.py", line 748, in __call__ return self._call_app(env, start_response) File "/usr/local/lib/python2.7/dist-packages/keystonemiddleware/auth_token.py", line 684, in _call_app return self._app(env, _fake_start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/lib/python2.7/dist-packages/paste/urlmap.py", line 206, in __call__ return app(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ return resp(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/routes/middleware.py", line 131, in __call__ response = self.app(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ return resp(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 683, in __call__ request, **action_args) File "/opt/stack/glance/glance/common/wsgi.py", line 707, in dispatch return method(*args, **kwargs) File "/opt/stack/glance/glance/common/utils.py", line 449, in wrapped return func(self, req, *args, **kwargs) File "/opt/stack/glance/glance/api/v1/images.py", line 1058, in delete registry.delete_image_metadata(req.context, id) File "/opt/stack/glance/glance/registry/client/v1/api.py", line 175, in delete_image_metadata return c.delete_image(image_id) File "/opt/stack/glance/glance/registry/client/v1/client.py", line 207, in delete_image res = self.do_request("DELETE", "/images/%s" % image_id) File "/opt/stack/glance/glance/registry/client/v1/client.py", line 130, in do_request 'exc_name': exc_name}) File "/opt/stack/glance/glance/openstack/common/excutils.py", line 82, in __exit__ six.reraise(self.type_, self.value, self.tb) File "/opt/stack/glance/glance/registry/client/v1/client.py", line 115, in do_request **kwargs) File "/opt/stack/glance/glance/common/client.py", line 68, in wrapped return func(self, *args, **kwargs) File "/opt/stack/glance/glance/common/client.py", line 388, in do_request headers=copy.deepcopy(headers)) File "/opt/stack/glance/glance/common/client.py", line 85, in wrapped return func(self, method, url, body, headers) File "/opt/stack/glance/glance/common/client.py", line 546, in _do_request raise exception.ServerError() ServerError: The request returned 500 Internal Server Error. 2014-10-06 00:53:10.055 6827 INFO glance.wsgi.server [2b47d213-6f80-410f-9766-dc80607f0224 7e7c3a413f184dbcb9a65404dbfcc0f0 309c5ff4082c4231bcc17d8c55c83997 - - -] 10.69.4.178 - - [06/Oct/2014 00:53:10] "DELETE /v1/images/f9f8a40d-530b-498c-9fbc-86f29da555f4 HTTP/1.1" 500 139 0.239514 ** Affects: glance Importance: Undecided Assignee: Ankit Agrawal (ankitagrawal) Status: New ** Tags: ntt ** Changed in: glance Assignee: (unassigned) => Ankit Agrawal (ankitagrawal) -- You received this bug notification because you are a member of Yahoo! Engineering Team, which is subscribed to Glance. https://bugs.launchpad.net/bugs/1378215 Title: If db deadlock occurs for some reason while deleting an image, no one can delete the image any more Status in OpenStack Image Registry and Delivery Service (Glance): New Bug description: Glance api returns 500 Internal Server Error, if db deadlock occurs in glance-registry for some reason while deleting an image. The image 'status' is set to deleted and 'deleted' is set to False. As deleted is still False, the image is visible in image list but it can not be deleted any more. If you try to delete this image it will raise 404 (Not Found) error for V1 api and 500 (HTTPInternalServerError) for V2 api. Note: To reproduce this issue I've explicitly raised "db_exception.DBDeadlock" exception from "_image_child_entry_delete_all" method under "\glance\db\sqlalchemy\api.py". glance-api.log -------------- 2014-10-06 00:53:10.037 6827 INFO glance.registry.client.v1.client [2b47d213-6f80-410f-9766-dc80607f0224 7e7c3a413f184dbcb9a65404dbfcc0f0 309c5ff4082c423 1bcc17d8c55c83997 - - -] Registry client request DELETE /images/f9f8a40d-530b-498c-9fbc-86f29da555f4 raised ServerError 2014-10-06 00:53:10.045 6827 INFO glance.wsgi.server [2b47d213-6f80-410f-9766-dc80607f0224 7e7c3a413f184dbcb9a65404dbfcc0f0 309c5ff4082c4231bcc17d8c55c83997 - - -] Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/eventlet/wsgi.py", line 433, in handle_one_response result = self.application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/osprofiler/web.py", line 106, in __call__ return request.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/keystonemiddleware/auth_token.py", line 748, in __call__ return self._call_app(env, start_response) File "/usr/local/lib/python2.7/dist-packages/keystonemiddleware/auth_token.py", line 684, in _call_app return self._app(env, _fake_start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 394, in __call__ response = req.get_response(self.application) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send application, catch_exc_info=False) File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application app_iter = application(self.environ, start_response) File "/usr/lib/python2.7/dist-packages/paste/urlmap.py", line 206, in __call__ return app(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ return resp(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/routes/middleware.py", line 131, in __call__ response = self.app(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ return resp(environ, start_response) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ resp = self.call_func(req, *args, **self.kwargs) File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func return self.func(req, *args, **kwargs) File "/opt/stack/glance/glance/common/wsgi.py", line 683, in __call__ request, **action_args) File "/opt/stack/glance/glance/common/wsgi.py", line 707, in dispatch return method(*args, **kwargs) File "/opt/stack/glance/glance/common/utils.py", line 449, in wrapped return func(self, req, *args, **kwargs) File "/opt/stack/glance/glance/api/v1/images.py", line 1058, in delete registry.delete_image_metadata(req.context, id) File "/opt/stack/glance/glance/registry/client/v1/api.py", line 175, in delete_image_metadata return c.delete_image(image_id) File "/opt/stack/glance/glance/registry/client/v1/client.py", line 207, in delete_image res = self.do_request("DELETE", "/images/%s" % image_id) File "/opt/stack/glance/glance/registry/client/v1/client.py", line 130, in do_request 'exc_name': exc_name}) File "/opt/stack/glance/glance/openstack/common/excutils.py", line 82, in __exit__ six.reraise(self.type_, self.value, self.tb) File "/opt/stack/glance/glance/registry/client/v1/client.py", line 115, in do_request **kwargs) File "/opt/stack/glance/glance/common/client.py", line 68, in wrapped return func(self, *args, **kwargs) File "/opt/stack/glance/glance/common/client.py", line 388, in do_request headers=copy.deepcopy(headers)) File "/opt/stack/glance/glance/common/client.py", line 85, in wrapped return func(self, method, url, body, headers) File "/opt/stack/glance/glance/common/client.py", line 546, in _do_request raise exception.ServerError() ServerError: The request returned 500 Internal Server Error. 2014-10-06 00:53:10.055 6827 INFO glance.wsgi.server [2b47d213-6f80-410f-9766-dc80607f0224 7e7c3a413f184dbcb9a65404dbfcc0f0 309c5ff4082c4231bcc17d8c55c83997 - - -] 10.69.4.178 - - [06/Oct/2014 00:53:10] "DELETE /v1/images/f9f8a40d-530b-498c-9fbc-86f29da555f4 HTTP/1.1" 500 139 0.239514 To manage notifications about this bug go to: https://bugs.launchpad.net/glance/+bug/1378215/+subscriptions -- Mailing list: https://launchpad.net/~yahoo-eng-team Post to : yahoo-eng-team@lists.launchpad.net Unsubscribe : https://launchpad.net/~yahoo-eng-team More help : https://help.launchpad.net/ListHelp