shivanshs9 commented on issue #11899:
URL: https://github.com/apache/airflow/issues/11899#issuecomment-736521240
@ashb
There's another deadlock case with different Traceback:
```
[2020-12-01 12:23:36,996] {{scheduler_job.py:1301}} ERROR - Exception when
executing SchedulerJob._run_scheduler_loop
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1277, in _execute_context
self.dialect.do_execute(
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py",
line 593, in do_execute
cursor.execute(statement, parameters)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
255, in execute
self.errorhandler(self, exc, value)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line
50, in defaulterrorhandler
raise errorvalue
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
252, in execute
res = self._query(query)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
379, in _query
self._do_get_result(db)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
182, in _do_get_result
self._result = result = self._get_result()
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
411, in _get_result
return self._get_db().store_result()
_mysql_exceptions.OperationalError: (1213, 'Deadlock found when trying to
get lock; try restarting transaction')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py",
line 1283, in _execute
self._run_scheduler_loop()
File
"/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py",
line 1385, in _run_scheduler_loop
num_queued_tis = self._do_scheduling(session)
File
"/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py",
line 1484, in _do_scheduling
self._create_dag_runs(query.all(), session)
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py",
line 3341, in all
return list(self)
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py",
line 3503, in __iter__
return self._execute_and_instances(context)
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py",
line 3528, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1014, in execute
return meth(self, multiparams, params)
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py",
line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1127, in _execute_clauseelement
ret = self._execute_context(
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1317, in _execute_context
self._handle_dbapi_exception(
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1511, in _handle_dbapi_exception
util.raise_(
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/util/compat.py",
line 178, in raise_
raise exception
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1277, in _execute_context
self.dialect.do_execute(
File
"/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py",
line 593, in do_execute
cursor.execute(statement, parameters)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
255, in execute
self.errorhandler(self, exc, value)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line
50, in defaulterrorhandler
raise errorvalue
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
252, in execute
res = self._query(query)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
379, in _query
self._do_get_result(db)
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
182, in _do_get_result
self._result = result = self._get_result()
File
"/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line
411, in _get_result
return self._get_db().store_result()
sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (1213,
'Deadlock found when trying to get lock; try restarting transaction')
[SQL: SELECT dag.dag_id AS dag_dag_id, dag.root_dag_id AS dag_root_dag_id,
dag.is_paused AS dag_is_paused, dag.is_subdag AS dag_is_subdag, dag.is_active
AS dag_is_active, dag.last_scheduler_run AS dag_last_scheduler_run,
dag.last_pickled AS dag_last_pickled, dag.last_expired AS dag_last_expired,
dag.scheduler_lock AS dag_scheduler_lock, dag.pickle_id AS dag_pickle_id,
dag.fileloc AS dag_fileloc, dag.owners AS dag_owners, dag.description AS
dag_description, dag.default_view AS dag_default_view, dag.schedule_interval AS
dag_schedule_interval, dag.concurrency AS dag_concurrency,
dag.has_task_concurrency_limits AS dag_has_task_concurrency_limits,
dag.next_dagrun AS dag_next_dagrun, dag.next_dagrun_create_after AS
dag_next_dagrun_create_after
FROM dag
WHERE dag.is_paused IS false AND dag.is_active IS true AND
dag.next_dagrun_create_after <= now() ORDER BY dag.next_dagrun_create_after
LIMIT %s FOR UPDATE]
[parameters: (10,)]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
[2020-12-01 12:23:38,016] {{process_utils.py:95}} INFO - Sending
Signals.SIGTERM to GPID 36
[2020-12-01 12:23:38,389] {{process_utils.py:61}} INFO - Process
psutil.Process(pid=522, status='terminated', started='12:23:36') (522)
terminated with exit code None
[2020-12-01 12:23:38,405] {{process_utils.py:201}} INFO - Waiting up to 5
seconds for processes to exit...
[2020-12-01 12:23:38,415] {{process_utils.py:61}} INFO - Process
psutil.Process(pid=36, status='terminated', exitcode=0, started='12:22:07')
(36) terminated with exit code 0
[2020-12-01 12:23:38,416] {{process_utils.py:61}} INFO - Process
psutil.Process(pid=526, status='terminated', started='12:23:36') (526)
terminated with exit code None
[2020-12-01 12:23:38,416] {{scheduler_job.py:1304}} INFO - Exited execute
loop
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
