drajguru opened a new pull request, #37927: URL: https://github.com/apache/airflow/pull/37927
Fix for unhandled exceptions raised in the triggerer tests on the queue
listener thread.
The queue listener was being killed due to missing a reference to the
triggerer_job. Fixing this resulted in console logging during tests. So changes
also include disabling the console logging handler / queue listener during
tests.
Also fixed a bug in
tests/jobs/test_triggerer_job.py::test_trigger_create_race_condition_18392
which caused another unhandled exception.
----
tests/jobs/test_triggerer_job.py::test_trigger_logging_sensitive_info
/usr/local/lib/python3.8/site-packages/_pytest/threadexception.py:73:
PytestUnhandledThreadExceptionWarning: Exception in thread Thread-3
Traceback (most recent call last):
File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/logging/handlers.py", line 1487, in
_monitor
self.handle(record)
File "/usr/local/lib/python3.8/logging/handlers.py", line 1468, in
handle
handler.handle(record)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 107, in
handle
self.emit(record)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 96, in
emit
h = self._get_or_create_handler(record.trigger_id,
record.task_instance)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 92, in
_get_or_create_handler
self.handlers[trigger_id] = self._make_handler(ti)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 87, in
_make_handler
h.set_context(ti=ti)
File "/opt/airflow/airflow/utils/log/file_task_handler.py", line 243, in
set_context
local_loc = self._init_file(ti, identifier=identifier)
File "/opt/airflow/airflow/utils/log/file_task_handler.py", line 533, in
_init_file
full_path = self.add_triggerer_suffix(full_path=full_path,
job_id=ti.triggerer_job.id)
AttributeError: 'NoneType' object has no attribute 'id'
warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))
tests/jobs/test_triggerer_job.py::test_trigger_lifecycle
/usr/local/lib/python3.8/site-packages/_pytest/threadexception.py:73:
PytestUnhandledThreadExceptionWarning: Exception in thread Thread-11
Traceback (most recent call last):
File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/logging/handlers.py", line 1487, in
_monitor
self.handle(record)
File "/usr/local/lib/python3.8/logging/handlers.py", line 1468, in handle
handler.handle(record)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 107, in
handle
self.emit(record)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 96, in
emit
h = self._get_or_create_handler(record.trigger_id,
record.task_instance)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 92, in
_get_or_create_handler
self.handlers[trigger_id] = self._make_handler(ti)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 87, in
_make_handler
h.set_context(ti=ti)
File "/opt/airflow/airflow/utils/log/file_task_handler.py", line 243, in
set_context
local_loc = self._init_file(ti, identifier=identifier)
File "/opt/airflow/airflow/utils/log/file_task_handler.py", line 533, in
_init_file
full_path = self.add_triggerer_suffix(full_path=full_path,
job_id=ti.triggerer_job.id)
AttributeError: 'NoneType' object has no attribute 'id'
tests/jobs/test_triggerer_job.py::test_trigger_create_race_condition_18392
/usr/local/lib/python3.8/site-packages/_pytest/threadexception.py:73:
PytestUnhandledThreadExceptionWarning: Exception in thread Thread-19
Traceback (most recent call last):
File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/opt/airflow/airflow/jobs/triggerer_job_runner.py", line 341, in
_execute
self._run_trigger_loop()
File "/opt/airflow/airflow/jobs/triggerer_job_runner.py", line 364, in
_run_trigger_loop
self.load_triggers()
File "/opt/airflow/tests/jobs/test_triggerer_job.py", line 376, in
load_triggers
self.wait_for_runner_loop(runner_loop_count=2)
File "/opt/airflow/tests/jobs/test_triggerer_job.py", line 367, in
wait_for_runner_loop
pytest.fail("did not observe 2 loops in the runner thread")
File "/usr/local/lib/python3.8/site-packages/_pytest/outcomes.py", line
198, in fail
raise Failed(msg=reason, pytrace=pytrace)
Failed: did not observe 2 loops in the runner thread
warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))
tests/jobs/test_triggerer_job.py::test_trigger_runner_exception_stops_triggerer
/usr/local/lib/python3.8/site-packages/_pytest/threadexception.py:73:
PytestUnhandledThreadExceptionWarning: Exception in thread Thread-26
Traceback (most recent call last):
File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
File "/opt/airflow/airflow/jobs/triggerer_job_runner.py", line 462, in
run
asyncio.run(self.arun())
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in
run_until_complete
return future.result()
File "/opt/airflow/airflow/jobs/triggerer_job_runner.py", line 475, in
arun
await self.create_triggers()
File "/opt/airflow/tests/jobs/test_triggerer_job.py", line 493, in
create_triggers
raise MockTriggerException("Trigger creation failed")
tests.jobs.test_triggerer_job.test_trigger_runner_exception_stops_triggerer.<locals>.MockTriggerException:
Trigger creation failed
warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))
tests/jobs/test_triggerer_job.py::test_trigger_firing
/usr/local/lib/python3.8/site-packages/_pytest/threadexception.py:73:
PytestUnhandledThreadExceptionWarning: Exception in thread Thread-28
Traceback (most recent call last):
File "/usr/local/lib/python3.8/threading.py", line 932, in
_bootstrap_inner
self.run()
File "/usr/local/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/logging/handlers.py", line 1487, in
_monitor
self.handle(record)
File "/usr/local/lib/python3.8/logging/handlers.py", line 1468, in
handle
handler.handle(record)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 107, in
handle
self.emit(record)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 96, in
emit
h = self._get_or_create_handler(record.trigger_id,
record.task_instance)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 92, in
_get_or_create_handler
self.handlers[trigger_id] = self._make_handler(ti)
File "/opt/airflow/airflow/utils/log/trigger_handler.py", line 87, in
_make_handler
h.set_context(ti=ti)
File "/opt/airflow/airflow/utils/log/file_task_handler.py", line 243, in
set_context
local_loc = self._init_file(ti, identifier=identifier)
File "/opt/airflow/airflow/utils/log/file_task_handler.py", line 533, in
_init_file
full_path = self.add_triggerer_suffix(full_path=full_path,
job_id=ti.triggerer_job.id)
AttributeError: 'NoneType' object has no attribute 'id'
warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
