Public bug reported: Example of failure https://3e402c0e76741e83fc60-d00ff4f1a74cdbc5ea9d8044145b77c0.ssl.cf2.rackcdn.com/888574/3/check/neutron- functional-with-uwsgi/3bea3d7/job-output.txt
It seems that many tests are stuck on the interaction with netlink (add ip, device_exists, etc.) Example of stacktrace: 2023-07-14 16:22:38.548953 | controller | Captured traceback: 2023-07-14 16:22:38.548978 | controller | ~~~~~~~~~~~~~~~~~~~ 2023-07-14 16:22:38.549017 | controller | Traceback (most recent call last): 2023-07-14 16:22:38.549042 | controller | 2023-07-14 16:22:38.549101 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py", line 178, in func 2023-07-14 16:22:38.549143 | controller | return f(self, *args, **kwargs) 2023-07-14 16:22:38.549164 | controller | 2023-07-14 16:22:38.549202 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/extensions/test_ovs_agent_qos_extension.py", line 242, in test_port_creation_with_different_bandwidth_limits 2023-07-14 16:22:38.549283 | controller | self.setup_agent_and_ports(port_dicts) 2023-07-14 16:22:38.549305 | controller | 2023-07-14 16:22:38.549330 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/base.py", line 457, in setup_agent_and_ports 2023-07-14 16:22:38.549362 | controller | self._plug_ports(self.network, self.ports, self.agent, 2023-07-14 16:22:38.549390 | controller | 2023-07-14 16:22:38.549416 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/base.py", line 268, in _plug_ports 2023-07-14 16:22:38.549505 | controller | self.driver.init_l3(port.get('vif_name'), ip_cidrs, 2023-07-14 16:22:38.549533 | controller | 2023-07-14 16:22:38.549554 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/interface.py", line 152, in init_l3 2023-07-14 16:22:38.549583 | controller | device.addr.add(ip_cidr) 2023-07-14 16:22:38.549607 | controller | 2023-07-14 16:22:38.549628 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 544, in add 2023-07-14 16:22:38.551748 | controller | add_ip_address(cidr, self.name, self._parent.namespace, scope, 2023-07-14 16:22:38.551786 | controller | 2023-07-14 16:22:38.551817 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 837, in add_ip_address 2023-07-14 16:22:38.551841 | controller | privileged.add_ip_address( 2023-07-14 16:22:38.551859 | controller | 2023-07-14 16:22:38.551878 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/priv_context.py", line 271, in _wrap 2023-07-14 16:22:38.551899 | controller | return self.channel.remote_call(name, args, kwargs, 2023-07-14 16:22:38.551921 | controller | 2023-07-14 16:22:38.551942 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/daemon.py", line 204, in remote_call 2023-07-14 16:22:38.552058 | controller | result = self.send_recv((comm.Message.CALL.value, name, args, kwargs), 2023-07-14 16:22:38.552087 | controller | 2023-07-14 16:22:38.552106 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/comm.py", line 194, in send_recv 2023-07-14 16:22:38.552155 | controller | reply = future.result() 2023-07-14 16:22:38.552184 | controller | 2023-07-14 16:22:38.552215 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/comm.py", line 121, in result 2023-07-14 16:22:38.552241 | controller | if not self.condvar.wait(timeout=self.timeout): 2023-07-14 16:22:38.552264 | controller | 2023-07-14 16:22:38.552285 | controller | File "/usr/lib/python3.10/threading.py", line 320, in wait 2023-07-14 16:22:38.552302 | controller | waiter.acquire() 2023-07-14 16:22:38.552321 | controller | 2023-07-14 16:22:38.552339 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/semaphore.py", line 115, in acquire 2023-07-14 16:22:38.552358 | controller | hubs.get_hub().switch() 2023-07-14 16:22:38.552381 | controller | 2023-07-14 16:22:38.552456 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch 2023-07-14 16:22:38.552475 | controller | return self.greenlet.switch() 2023-07-14 16:22:38.552497 | controller | 2023-07-14 16:22:38.552513 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 365, in run 2023-07-14 16:22:38.552533 | controller | self.wait(sleep_time) 2023-07-14 16:22:38.552553 | controller | 2023-07-14 16:22:38.552568 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 80, in wait 2023-07-14 16:22:38.552595 | controller | presult = self.do_poll(seconds) 2023-07-14 16:22:38.552616 | controller | 2023-07-14 16:22:38.552634 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/epolls.py", line 31, in do_poll 2023-07-14 16:22:38.552653 | controller | return self.poll.poll(seconds) 2023-07-14 16:22:38.552671 | controller | 2023-07-14 16:22:38.552694 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/fixtures/_fixtures/timeout.py", line 52, in signal_handler 2023-07-14 16:22:38.552731 | controller | raise TimeoutException() 2023-07-14 16:22:38.552759 | controller | 2023-07-14 16:22:38.552785 | controller | fixtures._fixtures.timeout.TimeoutException It also looks similarly in https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c52/883246/16/check/neutron-fullstack-with-uwsgi/c524e84/job-output.txt ** Affects: neutron Importance: Critical Assignee: Miguel Lavalle (minsel) Status: Confirmed ** Tags: functional-tests gate-failure -- You received this bug notification because you are a member of Yahoo! Engineering Team, which is subscribed to neutron. https://bugs.launchpad.net/bugs/2028152 Title: Timeouts on netlink related operations in many functional tests causes job's timeout Status in neutron: Confirmed Bug description: Example of failure https://3e402c0e76741e83fc60-d00ff4f1a74cdbc5ea9d8044145b77c0.ssl.cf2.rackcdn.com/888574/3/check/neutron- functional-with-uwsgi/3bea3d7/job-output.txt It seems that many tests are stuck on the interaction with netlink (add ip, device_exists, etc.) Example of stacktrace: 2023-07-14 16:22:38.548953 | controller | Captured traceback: 2023-07-14 16:22:38.548978 | controller | ~~~~~~~~~~~~~~~~~~~ 2023-07-14 16:22:38.549017 | controller | Traceback (most recent call last): 2023-07-14 16:22:38.549042 | controller | 2023-07-14 16:22:38.549101 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py", line 178, in func 2023-07-14 16:22:38.549143 | controller | return f(self, *args, **kwargs) 2023-07-14 16:22:38.549164 | controller | 2023-07-14 16:22:38.549202 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/extensions/test_ovs_agent_qos_extension.py", line 242, in test_port_creation_with_different_bandwidth_limits 2023-07-14 16:22:38.549283 | controller | self.setup_agent_and_ports(port_dicts) 2023-07-14 16:22:38.549305 | controller | 2023-07-14 16:22:38.549330 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/base.py", line 457, in setup_agent_and_ports 2023-07-14 16:22:38.549362 | controller | self._plug_ports(self.network, self.ports, self.agent, 2023-07-14 16:22:38.549390 | controller | 2023-07-14 16:22:38.549416 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/base.py", line 268, in _plug_ports 2023-07-14 16:22:38.549505 | controller | self.driver.init_l3(port.get('vif_name'), ip_cidrs, 2023-07-14 16:22:38.549533 | controller | 2023-07-14 16:22:38.549554 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/interface.py", line 152, in init_l3 2023-07-14 16:22:38.549583 | controller | device.addr.add(ip_cidr) 2023-07-14 16:22:38.549607 | controller | 2023-07-14 16:22:38.549628 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 544, in add 2023-07-14 16:22:38.551748 | controller | add_ip_address(cidr, self.name, self._parent.namespace, scope, 2023-07-14 16:22:38.551786 | controller | 2023-07-14 16:22:38.551817 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 837, in add_ip_address 2023-07-14 16:22:38.551841 | controller | privileged.add_ip_address( 2023-07-14 16:22:38.551859 | controller | 2023-07-14 16:22:38.551878 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/priv_context.py", line 271, in _wrap 2023-07-14 16:22:38.551899 | controller | return self.channel.remote_call(name, args, kwargs, 2023-07-14 16:22:38.551921 | controller | 2023-07-14 16:22:38.551942 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/daemon.py", line 204, in remote_call 2023-07-14 16:22:38.552058 | controller | result = self.send_recv((comm.Message.CALL.value, name, args, kwargs), 2023-07-14 16:22:38.552087 | controller | 2023-07-14 16:22:38.552106 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/comm.py", line 194, in send_recv 2023-07-14 16:22:38.552155 | controller | reply = future.result() 2023-07-14 16:22:38.552184 | controller | 2023-07-14 16:22:38.552215 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/comm.py", line 121, in result 2023-07-14 16:22:38.552241 | controller | if not self.condvar.wait(timeout=self.timeout): 2023-07-14 16:22:38.552264 | controller | 2023-07-14 16:22:38.552285 | controller | File "/usr/lib/python3.10/threading.py", line 320, in wait 2023-07-14 16:22:38.552302 | controller | waiter.acquire() 2023-07-14 16:22:38.552321 | controller | 2023-07-14 16:22:38.552339 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/semaphore.py", line 115, in acquire 2023-07-14 16:22:38.552358 | controller | hubs.get_hub().switch() 2023-07-14 16:22:38.552381 | controller | 2023-07-14 16:22:38.552456 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch 2023-07-14 16:22:38.552475 | controller | return self.greenlet.switch() 2023-07-14 16:22:38.552497 | controller | 2023-07-14 16:22:38.552513 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 365, in run 2023-07-14 16:22:38.552533 | controller | self.wait(sleep_time) 2023-07-14 16:22:38.552553 | controller | 2023-07-14 16:22:38.552568 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 80, in wait 2023-07-14 16:22:38.552595 | controller | presult = self.do_poll(seconds) 2023-07-14 16:22:38.552616 | controller | 2023-07-14 16:22:38.552634 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/epolls.py", line 31, in do_poll 2023-07-14 16:22:38.552653 | controller | return self.poll.poll(seconds) 2023-07-14 16:22:38.552671 | controller | 2023-07-14 16:22:38.552694 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/fixtures/_fixtures/timeout.py", line 52, in signal_handler 2023-07-14 16:22:38.552731 | controller | raise TimeoutException() 2023-07-14 16:22:38.552759 | controller | 2023-07-14 16:22:38.552785 | controller | fixtures._fixtures.timeout.TimeoutException It also looks similarly in https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c52/883246/16/check/neutron-fullstack-with-uwsgi/c524e84/job-output.txt To manage notifications about this bug go to: https://bugs.launchpad.net/neutron/+bug/2028152/+subscriptions -- Mailing list: https://launchpad.net/~yahoo-eng-team Post to : yahoo-eng-team@lists.launchpad.net Unsubscribe : https://launchpad.net/~yahoo-eng-team More help : https://help.launchpad.net/ListHelp