Repository: incubator-airflow Updated Branches: refs/heads/master cae918e91 -> f1df9575b
[AIRFLOW-2587] Add TIMESTAMP type mapping to MySqlToHiveTransfer This PR fixes MySqlToHiveTransfer to map MySQL TIMESTAMP to Hive TIMESTAMP (not STRING) so that users are not required to cast it explicitly. Closes #3486 from sekikn/AIRFLOW-2587 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/f1df9575 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/f1df9575 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/f1df9575 Branch: refs/heads/master Commit: f1df9575bd3ac2bd81187d88d3c9665c2755d2c0 Parents: cae918e Author: Kengo Seki <sek...@apache.org> Authored: Mon Jun 11 20:50:04 2018 +0200 Committer: Fokko Driesprong <fokkodriespr...@godatadriven.com> Committed: Mon Jun 11 20:50:04 2018 +0200 ---------------------------------------------------------------------- airflow/operators/mysql_to_hive.py | 1 + tests/operators/operators.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f1df9575/airflow/operators/mysql_to_hive.py ---------------------------------------------------------------------- diff --git a/airflow/operators/mysql_to_hive.py b/airflow/operators/mysql_to_hive.py index ab7b7fa..34b5337 100644 --- a/airflow/operators/mysql_to_hive.py +++ b/airflow/operators/mysql_to_hive.py @@ -109,6 +109,7 @@ class MySqlToHiveTransfer(BaseOperator): t.SHORT: 'INT', t.TINY: 'SMALLINT', t.YEAR: 'INT', + t.TIMESTAMP: 'TIMESTAMP', } return d[mysql_type] if mysql_type in d else 'STRING' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f1df9575/tests/operators/operators.py ---------------------------------------------------------------------- diff --git a/tests/operators/operators.py b/tests/operators/operators.py index 07b1396..c343d5c 100644 --- a/tests/operators/operators.py +++ b/tests/operators/operators.py @@ -334,7 +334,8 @@ class TransferTests(unittest.TestCase): c1 SMALLINT, c2 MEDIUMINT, c3 INT, - c4 BIGINT + c4 BIGINT, + c5 TIMESTAMP ) """.format(mysql_table)) @@ -355,6 +356,7 @@ class TransferTests(unittest.TestCase): d["c2"] = "INT" d["c3"] = "BIGINT" d["c4"] = "DECIMAL(38,0)" + d["c5"] = "TIMESTAMP" self.assertEqual(mock_load_file.call_args[1]["field_dict"], d) finally: with m.get_conn() as c:
