Public bug reported: This is a newton devstack created today running neutron + ovs:
1. create an instance, wait for it to go active 2. list instances, it's fine 3. stop all neutron-server processes 4. list instances - it fails with a 500 trying to process security groups because it can't connect to neutron: http://paste.openstack.org/show/493416/ 2016-04-07 20:30:54.272 ERROR nova.api.openstack [req-5d867a48-2097-456f-a2ae-f93c982ac5d0 admin admin] Caught error: Unable to establish connection to http://9.5.124.163:9696/v2.0/ports.json?device_id=cda8a8ac-6eac-434d-bded-a0b34d285f41 2016-04-07 20:30:54.272 TRACE nova.api.openstack Traceback (most recent call last): 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/__init__.py", line 134, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return req.get_response(self.application) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1317, in send 2016-04-07 20:30:54.272 TRACE nova.api.openstack application, catch_exc_info=False) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1281, in call_application 2016-04-07 20:30:54.272 TRACE nova.api.openstack app_iter = application(self.environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.func(req, *args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystonemiddleware/auth_token/__init__.py", line 467, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack response = req.get_response(self._app) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1317, in send 2016-04-07 20:30:54.272 TRACE nova.api.openstack application, catch_exc_info=False) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1281, in call_application 2016-04-07 20:30:54.272 TRACE nova.api.openstack app_iter = application(self.environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/routes/middleware.py", line 141, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack response = self.app(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.func(req, *args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/wsgi.py", line 672, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack content_type, body, accept) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/wsgi.py", line 756, in _process_stack 2016-04-07 20:30:54.272 TRACE nova.api.openstack request, action_args) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/wsgi.py", line 619, in post_process_extensions 2016-04-07 20:30:54.272 TRACE nova.api.openstack **action_args) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/compute/security_groups.py", line 491, in detail 2016-04-07 20:30:54.272 TRACE nova.api.openstack self._extend_servers(req, list(resp_obj.obj['servers'])) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/compute/security_groups.py", line 461, in _extend_servers 2016-04-07 20:30:54.272 TRACE nova.api.openstack servers)) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/network/security_group/neutron_driver.py", line 381, in get_instances_security_groups_bindings 2016-04-07 20:30:54.272 TRACE nova.api.openstack ports = self._get_ports_from_server_list(servers, neutron) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/network/security_group/neutron_driver.py", line 340, in _get_ports_from_server_list 2016-04-07 20:30:54.272 TRACE nova.api.openstack ports.extend(neutron.list_ports(**search_opts).get('ports')) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 618, in list_ports 2016-04-07 20:30:54.272 TRACE nova.api.openstack **_params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 357, in list 2016-04-07 20:30:54.272 TRACE nova.api.openstack for r in self._pagination(collection, path, **params): 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 372, in _pagination 2016-04-07 20:30:54.272 TRACE nova.api.openstack res = self.get(path, params=params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 342, in get 2016-04-07 20:30:54.272 TRACE nova.api.openstack headers=headers, params=params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 319, in retry_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack headers=headers, params=params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 270, in do_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp, replybody = self.httpclient.do_request(action, method, body=body) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/client.py", line 306, in do_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.request(url, method, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/client.py", line 294, in request 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = super(SessionClient, self).request(*args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/adapter.py", line 98, in request 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.session.request(url, method, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/positional/__init__.py", line 94, in inner 2016-04-07 20:30:54.272 TRACE nova.api.openstack return func(*args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 452, in request 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = send(**kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 496, in _send_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack raise exceptions.ConnectFailure(msg) 2016-04-07 20:30:54.272 TRACE nova.api.openstack ConnectFailure: Unable to establish connection to http://9.5.124.163:9696/v2.0/ports.json?device_id=cda8a8ac-6eac-434d-bded-a0b34d285f41 2016-04-07 20:30:54.272 TRACE nova.api.openstack ** Affects: nova Importance: Medium Status: Confirmed ** Tags: api network neutron security-groups ** Changed in: nova Status: New => Confirmed ** Changed in: nova Importance: Undecided => Medium ** Tags added: network security-groups -- You received this bug notification because you are a member of Yahoo! Engineering Team, which is subscribed to OpenStack Compute (nova). https://bugs.launchpad.net/bugs/1567655 Title: 500 error when trying to list instances and neutron-server is down Status in OpenStack Compute (nova): Confirmed Bug description: This is a newton devstack created today running neutron + ovs: 1. create an instance, wait for it to go active 2. list instances, it's fine 3. stop all neutron-server processes 4. list instances - it fails with a 500 trying to process security groups because it can't connect to neutron: http://paste.openstack.org/show/493416/ 2016-04-07 20:30:54.272 ERROR nova.api.openstack [req-5d867a48-2097-456f-a2ae-f93c982ac5d0 admin admin] Caught error: Unable to establish connection to http://9.5.124.163:9696/v2.0/ports.json?device_id=cda8a8ac-6eac-434d-bded-a0b34d285f41 2016-04-07 20:30:54.272 TRACE nova.api.openstack Traceback (most recent call last): 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/__init__.py", line 134, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return req.get_response(self.application) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1317, in send 2016-04-07 20:30:54.272 TRACE nova.api.openstack application, catch_exc_info=False) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1281, in call_application 2016-04-07 20:30:54.272 TRACE nova.api.openstack app_iter = application(self.environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.func(req, *args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystonemiddleware/auth_token/__init__.py", line 467, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack response = req.get_response(self._app) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1317, in send 2016-04-07 20:30:54.272 TRACE nova.api.openstack application, catch_exc_info=False) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1281, in call_application 2016-04-07 20:30:54.272 TRACE nova.api.openstack app_iter = application(self.environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/routes/middleware.py", line 141, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack response = self.app(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack return resp(environ, start_response) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.func(req, *args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/wsgi.py", line 672, in __call__ 2016-04-07 20:30:54.272 TRACE nova.api.openstack content_type, body, accept) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/wsgi.py", line 756, in _process_stack 2016-04-07 20:30:54.272 TRACE nova.api.openstack request, action_args) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/wsgi.py", line 619, in post_process_extensions 2016-04-07 20:30:54.272 TRACE nova.api.openstack **action_args) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/compute/security_groups.py", line 491, in detail 2016-04-07 20:30:54.272 TRACE nova.api.openstack self._extend_servers(req, list(resp_obj.obj['servers'])) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/api/openstack/compute/security_groups.py", line 461, in _extend_servers 2016-04-07 20:30:54.272 TRACE nova.api.openstack servers)) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/network/security_group/neutron_driver.py", line 381, in get_instances_security_groups_bindings 2016-04-07 20:30:54.272 TRACE nova.api.openstack ports = self._get_ports_from_server_list(servers, neutron) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/nova/nova/network/security_group/neutron_driver.py", line 340, in _get_ports_from_server_list 2016-04-07 20:30:54.272 TRACE nova.api.openstack ports.extend(neutron.list_ports(**search_opts).get('ports')) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 618, in list_ports 2016-04-07 20:30:54.272 TRACE nova.api.openstack **_params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 357, in list 2016-04-07 20:30:54.272 TRACE nova.api.openstack for r in self._pagination(collection, path, **params): 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 372, in _pagination 2016-04-07 20:30:54.272 TRACE nova.api.openstack res = self.get(path, params=params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 342, in get 2016-04-07 20:30:54.272 TRACE nova.api.openstack headers=headers, params=params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 319, in retry_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack headers=headers, params=params) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/v2_0/client.py", line 270, in do_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp, replybody = self.httpclient.do_request(action, method, body=body) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/client.py", line 306, in do_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.request(url, method, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/opt/stack/python-neutronclient/neutronclient/client.py", line 294, in request 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = super(SessionClient, self).request(*args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/adapter.py", line 98, in request 2016-04-07 20:30:54.272 TRACE nova.api.openstack return self.session.request(url, method, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/positional/__init__.py", line 94, in inner 2016-04-07 20:30:54.272 TRACE nova.api.openstack return func(*args, **kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 452, in request 2016-04-07 20:30:54.272 TRACE nova.api.openstack resp = send(**kwargs) 2016-04-07 20:30:54.272 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 496, in _send_request 2016-04-07 20:30:54.272 TRACE nova.api.openstack raise exceptions.ConnectFailure(msg) 2016-04-07 20:30:54.272 TRACE nova.api.openstack ConnectFailure: Unable to establish connection to http://9.5.124.163:9696/v2.0/ports.json?device_id=cda8a8ac-6eac-434d-bded-a0b34d285f41 2016-04-07 20:30:54.272 TRACE nova.api.openstack To manage notifications about this bug go to: https://bugs.launchpad.net/nova/+bug/1567655/+subscriptions -- Mailing list: https://launchpad.net/~yahoo-eng-team Post to : yahoo-eng-team@lists.launchpad.net Unsubscribe : https://launchpad.net/~yahoo-eng-team More help : https://help.launchpad.net/ListHelp
