Hi all,
Beginner here. Upon following
https://airflow.apache.org/docs/stable/howto/initialize-database.html, I found
I do not have access to set explicit_defaults_for_timestamp=1 in my
/etc/my.cnf. Thus, I asked the DB admin to set this for me and restart the
server. On the webserver, I tried a tutorial dag and got this:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid
start byte
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask/app.py",
line 2446, in wsgi_app
response = self.full_dispatch_request()
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask/app.py",
line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask/app.py",
line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask/_compat.py",
line 39, in reraise
raise value
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask/app.py",
line 1949, in full_dispatch_request
rv = self.dispatch_request()
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask/app.py",
line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask_admin/base.py",
line 69, in inner
return self._run_view(f, *args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask_admin/base.py",
line 368, in _run_view
return fn(self, *args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/flask_login/utils.py",
line 258, in decorated_view
return func(*args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/www/utils.py",
line 290, in wrapper
return f(*args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/www/utils.py",
line 337, in wrapper
return f(*args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/utils/db.py",
line 74, in wrapper
return func(*args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/www/views.py",
line 1213, in trigger
external_trigger=True
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/utils/db.py",
line 74, in wrapper
return func(*args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/models/dag.py",
line 1659, in create_dagrun
session=session)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/utils/db.py",
line 70, in wrapper
return func(*args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/models/dag.py",
line 1346, in create_dagrun
run.refresh_from_db()
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/utils/db.py",
line 74, in wrapper
return func(*args, **kwargs)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/airflow/models/dagrun.py",
line 109, in refresh_from_db
DR.run_id == self.run_id
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/orm/query.py",
line 3325, in one
ret = self.one_or_none()
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/orm/query.py",
line 3294, in one_or_none
ret = list(self)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/orm/query.py",
line 3367, in __iter__
return self._execute_and_instances(context)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/orm/query.py",
line 3392, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py",
line 982, in execute
return meth(self, multiparams, params)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/sql/elements.py",
line 287, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py",
line 1101, in _execute_clauseelement
distilled_params,
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py",
line 1250, in _execute_context
e, statement, parameters, cursor, context
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py",
line 1478, in _handle_dbapi_exception
util.reraise(*exc_info)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/util/compat.py",
line 153, in reraise
raise value
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py",
line 1246, in _execute_context
cursor, statement, parameters, context
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/sqlalchemy/engine/default.py",
line 581, in do_execute
cursor.execute(statement, parameters)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/mysql/connector/cursor_cext.py",
line 272, in execute
self._handle_result(result)
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/mysql/connector/cursor_cext.py",
line 163, in _handle_result
self._handle_resultset()
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/mysql/connector/cursor_cext.py",
line 651, in _handle_resultset
self._rows = self._cnx.get_rows()[0]
File
"/export/home/anaconda/2018.12/envs/dwan_98_airflow/lib/python3.7/site-packages/mysql/connector/connection_cext.py",
line 301, in get_rows
else self._cmysql.fetch_row()
SystemError: <method 'fetch_row' of '_mysql_connector.MySQL' objects> returned
a result with an error set
This is what my DB admin checked for me:
mysql> show variables like '%character%';
+--------------------------+------------------------------------------------------------+
| Variable_name | Value
|
+--------------------------+------------------------------------------------------------+
| character_set_client | utf8
|
| character_set_connection | utf8
|
| character_set_database | utf8
|
| character_set_filesystem | binary
|
| character_set_results | utf8
|
| character_set_server | utf8
|
| character_set_system | utf8
|
+--------------------------+------------------------------------------------------------+
mysql> show variables like '%collation%';
+----------------------+-----------------+
| Variable_name | Value |
+----------------------+-----------------+
| collation_connection | utf8_unicode_ci |
| collation_database | utf8_unicode_ci |
| collation_server | utf8_unicode_ci |
+----------------------+-----------------+
mysql> select version();
+------------+
| version() |
+------------+
| 5.7.18-log |
+------------+
1 row in set (0.00 sec)
I note that he didn't explicitly check explicit_defaults_for_timestamp=1, but I
observed that airflow initdb would an error otherwise, so I assume the
successful initialization means this has been set.
Allow me to copy some lines from the scheduler:
09:12 $ airflow scheduler
[2019-12-28 09:12:49,852] {settings.py:254} INFO - settings.configure_orm():
Using pool settings. pool_size=5, max_overflow=10, pool_recycle=1800, pid=23368
____________ _____________
____ |__( )_________ __/__ /________ __
____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / /
___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ /
_/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/
[2019-12-28 09:12:54,211] {__init__.py:51} INFO - Using executor LocalExecutor
[2019-12-28 09:12:54,924] {scheduler_job.py:1323} INFO - Starting the scheduler
[2019-12-28 09:12:54,924] {scheduler_job.py:1331} INFO - Running execute loop
for -1 seconds
[2019-12-28 09:12:54,924] {scheduler_job.py:1332} INFO - Processing each file
at most -1 times
[2019-12-28 09:12:54,924] {scheduler_job.py:1335} INFO - Searching for files in
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags
[2019-12-28 09:12:54,931] {scheduler_job.py:1337} INFO - There are 1 files in
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags
[2019-12-28 09:12:54,988] {scheduler_job.py:1388} INFO - Resetting orphaned
tasks for active dag runs
[2019-12-28 09:12:55,708] {dag_processing.py:556} INFO - Launched
DagFileProcessorManager with pid: 23558
[2019-12-28 09:12:55,716] {settings.py:55} INFO - Configured default timezone
<Timezone [UTC]>
[2019-12-28 09:12:55,723] {settings.py:254} INFO - settings.configure_orm():
Using pool settings. pool_size=5, max_overflow=10, pool_recycle=1800, pid=23558
and the scheduler log:
...
[2019-12-28 09:17:39,320] {scheduler_job.py:153} INFO - Started process
(PID=26499) to work on
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:39,321] {scheduler_job.py:1539} INFO - Processing file
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
for tasks to queue
[2019-12-28 09:17:39,321] {logging_mixin.py:112} INFO - [2019-12-28
09:17:39,321] {dagbag.py:403} INFO - Filling up the DagBag from
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:39,326] {scheduler_job.py:1551} INFO - DAG(s)
dict_keys(['dwan_tutorial']) retrieved from
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:45,022] {scheduler_job.py:161} INFO - Processing
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
took 5.703 seconds
[2019-12-28 09:17:45,331] {logging_mixin.py:112} INFO - [2019-12-28
09:17:45,330] {settings.py:254} INFO - settings.configure_orm(): Using pool
settings. pool_size=5, max_overflow=10, pool_recycle=1800, pid=26852
[2019-12-28 09:17:45,332] {scheduler_job.py:153} INFO - Started process
(PID=26852) to work on
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:45,334] {scheduler_job.py:1539} INFO - Processing file
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
for tasks to queue
[2019-12-28 09:17:45,334] {logging_mixin.py:112} INFO - [2019-12-28
09:17:45,334] {dagbag.py:403} INFO - Filling up the DagBag from
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:45,339] {scheduler_job.py:1551} INFO - DAG(s)
dict_keys(['dwan_tutorial']) retrieved from
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:51,036] {scheduler_job.py:161} INFO - Processing
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
took 5.704 seconds
[2019-12-28 09:17:51,054] {logging_mixin.py:112} INFO - [2019-12-28
09:17:51,054] {settings.py:254} INFO - settings.configure_orm(): Using pool
settings. pool_size=5, max_overflow=10, pool_recycle=1800, pid=27065
[2019-12-28 09:17:51,056] {scheduler_job.py:153} INFO - Started process
(PID=27065) to work on
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:51,057] {scheduler_job.py:1539} INFO - Processing file
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
for tasks to queue
[2019-12-28 09:17:51,058] {logging_mixin.py:112} INFO - [2019-12-28
09:17:51,058] {dagbag.py:403} INFO - Filling up the DagBag from
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
[2019-12-28 09:17:51,063] {scheduler_job.py:1551} INFO - DAG(s)
dict_keys(['dwan_tutorial']) retrieved from
/home/dwan/Workspace/dev/dev_py/quanterest/quanterest/airflow/dags/tutorial.py
Help much appreciated!
Regards,
Derek Wan
######################################################################
The information contained in this communication is confidential and
may contain information that is privileged or exempt from disclosure
under applicable law. If you are not a named addressee, please notify
the sender immediately and delete this email from your system.
If you have received this communication, and are not a named
recipient, you are hereby notified that any dissemination,
distribution or copying of this communication is strictly prohibited.
######################################################################
