Public bug reported:
Recently we have been seeing pretty often functional tests issues with
errors like:
ft1.13:
neutron.tests.functional.plugins.ml2.drivers.ovn.mech_driver.ovsdb.test_maintenance.TestMaintenance.test_check_for_port_security_unknown_addresstesttools.testresult.real._StringException:
traceback-1: {{{
Traceback (most recent call last):
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 363, in stop
self.mech_driver.nb_ovn.ovsdb_connection.stop()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 153, in nb_ovn
self._post_fork_event.wait()
File "/usr/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
File "/usr/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/semaphore.py",
line 120, in acquire
hubs.get_hub().switch()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py",
line 313, in switch
return self.greenlet.switch()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py",
line 365, in run
self.wait(sleep_time)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/poll.py",
line 80, in wait
presult = self.do_poll(seconds)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/epolls.py",
line 31, in do_poll
return self.poll.poll(seconds)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/fixtures/_fixtures/timeout.py",
line 52, in signal_handler
raise TimeoutException()
fixtures._fixtures.timeout.TimeoutException
}}}
traceback-2: {{{
Traceback (most recent call last):
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 351, in _collect_processes_logs
self._copy_log_file("%s.log" % northd_log, dst_northd)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 356, in _copy_log_file
shutil.copyfile(
File "/usr/lib/python3.8/shutil.py", line 264, in copyfile
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
FileNotFoundError: [Errno 2] No such file or directory:
'/tmp/tmps9cyr99c/ovn_northd.log'
}}}
Traceback (most recent call last):
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/plugins/ml2/drivers/ovn/mech_driver/ovsdb/test_maintenance.py",
line 41, in setUp
super(_TestMaintenanceHelper, self).setUp()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 217, in setUp
self._start_idls()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 320, in _start_idls
self.mech_driver.pre_fork_initialize(
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 254, in pre_fork_initialize
self._create_neutron_pg_drop()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 272, in _create_neutron_pg_drop
create_default_drop_port_group(pre_ovn_nb_api)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 1241, in create_default_drop_port_group
txn.add(nb_idl.pg_add_ports(pg_name, list(ports_with_pg)))
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/impl_idl_ovn.py",
line 274, in transaction
yield t
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py",
line 110, in transaction
del self._nested_txns_map[cur_thread_id]
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py",
line 61, in __exit__
self.result = self.commit()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py",
line 65, in commit
raise result.ex
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/connection.py",
line 131, in run
txn.results.put(txn.do_commit())
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py",
line 136, in do_commit
self.post_commit(txn)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py",
line 73, in post_commit
command.post_commit(txn)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/command.py",
line 94, in post_commit
row = self.api.tables[self.table_name].rows[real_uuid]
File "/usr/lib/python3.8/collections/__init__.py", line 1010, in __getitem__
raise KeyError(key)
KeyError: UUID('c9570a47-64da-4360-9c94-12ba459344c2')
Examples of such failures:
https://bd41d5ab9bd61c5e04d6-86672578d4e6ceb498f2d932b0da6815.ssl.cf5.rackcdn.com/798961/7/check/neutron-
functional-with-uwsgi/772270c/testr_results.html
https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_856/801310/2/check/neutron-
functional-with-uwsgi/856f539/testr_results.html
https://7ee9afbce159ed2d3c7a-ae7c228c39808cabcdb014f8881dacdb.ssl.cf5.rackcdn.com/800278/7/check/neutron-
functional-with-uwsgi/6332cdf/testr_results.html
https://fc20912585430bbde389-7e8720184c7a4cb82719c44bcc579362.ssl.cf5.rackcdn.com/802687/1/check/neutron-
functional-with-uwsgi/d5d2430/testr_results.html
In all cases the stacktrace is pretty the same but failing tests are
different.
** Affects: neutron
Importance: Critical
Status: Confirmed
** Tags: functional-tests gate-failure ovn
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1938766
Title:
Functional tests related to ovn failing with No such file or
directory: '/tmp/tmps9cyr99c/ovn_northd.log'
Status in neutron:
Confirmed
Bug description:
Recently we have been seeing pretty often functional tests issues with
errors like:
ft1.13:
neutron.tests.functional.plugins.ml2.drivers.ovn.mech_driver.ovsdb.test_maintenance.TestMaintenance.test_check_for_port_security_unknown_addresstesttools.testresult.real._StringException:
traceback-1: {{{
Traceback (most recent call last):
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 363, in stop
self.mech_driver.nb_ovn.ovsdb_connection.stop()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 153, in nb_ovn
self._post_fork_event.wait()
File "/usr/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
File "/usr/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/semaphore.py",
line 120, in acquire
hubs.get_hub().switch()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py",
line 313, in switch
return self.greenlet.switch()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py",
line 365, in run
self.wait(sleep_time)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/poll.py",
line 80, in wait
presult = self.do_poll(seconds)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/epolls.py",
line 31, in do_poll
return self.poll.poll(seconds)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/fixtures/_fixtures/timeout.py",
line 52, in signal_handler
raise TimeoutException()
fixtures._fixtures.timeout.TimeoutException
}}}
traceback-2: {{{
Traceback (most recent call last):
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 351, in _collect_processes_logs
self._copy_log_file("%s.log" % northd_log, dst_northd)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 356, in _copy_log_file
shutil.copyfile(
File "/usr/lib/python3.8/shutil.py", line 264, in copyfile
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
FileNotFoundError: [Errno 2] No such file or directory:
'/tmp/tmps9cyr99c/ovn_northd.log'
}}}
Traceback (most recent call last):
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/plugins/ml2/drivers/ovn/mech_driver/ovsdb/test_maintenance.py",
line 41, in setUp
super(_TestMaintenanceHelper, self).setUp()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 217, in setUp
self._start_idls()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py",
line 320, in _start_idls
self.mech_driver.pre_fork_initialize(
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 254, in pre_fork_initialize
self._create_neutron_pg_drop()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 272, in _create_neutron_pg_drop
create_default_drop_port_group(pre_ovn_nb_api)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py",
line 1241, in create_default_drop_port_group
txn.add(nb_idl.pg_add_ports(pg_name, list(ports_with_pg)))
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/impl_idl_ovn.py",
line 274, in transaction
yield t
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py",
line 110, in transaction
del self._nested_txns_map[cur_thread_id]
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py",
line 61, in __exit__
self.result = self.commit()
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py",
line 65, in commit
raise result.ex
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/connection.py",
line 131, in run
txn.results.put(txn.do_commit())
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py",
line 136, in do_commit
self.post_commit(txn)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py",
line 73, in post_commit
command.post_commit(txn)
File
"/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/command.py",
line 94, in post_commit
row = self.api.tables[self.table_name].rows[real_uuid]
File "/usr/lib/python3.8/collections/__init__.py", line 1010, in __getitem__
raise KeyError(key)
KeyError: UUID('c9570a47-64da-4360-9c94-12ba459344c2')
Examples of such failures:
https://bd41d5ab9bd61c5e04d6-86672578d4e6ceb498f2d932b0da6815.ssl.cf5.rackcdn.com/798961/7/check/neutron-
functional-with-uwsgi/772270c/testr_results.html
https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_856/801310/2/check/neutron-
functional-with-uwsgi/856f539/testr_results.html
https://7ee9afbce159ed2d3c7a-ae7c228c39808cabcdb014f8881dacdb.ssl.cf5.rackcdn.com/800278/7/check/neutron-
functional-with-uwsgi/6332cdf/testr_results.html
https://fc20912585430bbde389-7e8720184c7a4cb82719c44bcc579362.ssl.cf5.rackcdn.com/802687/1/check/neutron-
functional-with-uwsgi/d5d2430/testr_results.html
In all cases the stacktrace is pretty the same but failing tests are
different.
To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1938766/+subscriptions
--
Mailing list: https://launchpad.net/~yahoo-eng-team
Post to : yahoo-eng-team@lists.launchpad.net
Unsubscribe : https://launchpad.net/~yahoo-eng-team
More help : https://help.launchpad.net/ListHelp
