Repository: incubator-airflow Updated Branches: refs/heads/master aa64f370b -> 2de4b7cfb
[AIRFLOW-1399] Fix cli reading logfile in memory Don't read logfile in memory if remote base isn't specified Closes #2433 from NielsZeilemaker/no_remote_base_no_read Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/2de4b7cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/2de4b7cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/2de4b7cf Branch: refs/heads/master Commit: 2de4b7cfb12f5a36eeaf5e78d3ee0fb12d67f3b2 Parents: aa64f37 Author: Niels Zeilemaker <nielszeilema...@godatadriven.com> Authored: Sun Jul 23 20:29:41 2017 +0200 Committer: Bolke de Bruin <bo...@xs4all.nl> Committed: Sun Jul 23 20:29:47 2017 +0200 ---------------------------------------------------------------------- airflow/bin/cli.py | 11 +++++++++-- tests/core.py | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/2de4b7cf/airflow/bin/cli.py ---------------------------------------------------------------------- diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index a8543d3..de8e9f3 100755 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -461,6 +461,10 @@ def run(args, dag=None): logging.root.handlers = [] # store logs remotely + _store_logs_remotely(log_base, filename) + + +def _store_logs_remotely(log_base, filename): remote_base = conf.get('core', 'REMOTE_BASE_LOG_FOLDER') # deprecated as of March 2016 @@ -472,7 +476,10 @@ def run(args, dag=None): DeprecationWarning) remote_base = conf.get('core', 'S3_LOG_FOLDER') - if os.path.exists(filename): + if remote_base == 'None': + remote_base = None + + if remote_base and os.path.exists(filename): # read log and remove old logs to get just the latest additions with open(filename, 'r') as logfile: @@ -487,7 +494,7 @@ def run(args, dag=None): elif remote_base.startswith('gs:/'): logging_utils.GCSLog().write(log, remote_log_location) # Other - elif remote_base and remote_base != 'None': + else: logging.error( 'Unsupported remote log location: {}'.format(remote_base)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/2de4b7cf/tests/core.py ---------------------------------------------------------------------- diff --git a/tests/core.py b/tests/core.py index 923e0c3..4e81534 100644 --- a/tests/core.py +++ b/tests/core.py @@ -1326,6 +1326,32 @@ class CliTests(unittest.TestCase): 'run', 'example_bash_operator', 'runme_0', '-l', DEFAULT_DATE.isoformat()])) + def test_cli_store_logs_remotely_no_remote_base(self): + with mock.patch('__main__.open', mock.mock_open(read_data='42'), create=True) as open_mock: + with mock.patch('os.path.exists') as path_mock: + path_mock.return_value = True + + cli._store_logs_remotely("42", "existing_file") + + # remote base not specified, hence no call to open + self.assertEqual(open_mock.call_count, 0) + + def test_cli_store_logs_remotely_with_remote_base(self): + orig_base_log_folder = configuration.get('core', 'REMOTE_BASE_LOG_FOLDER') + configuration.set("core", "REMOTE_BASE_LOG_FOLDER", "42") + + try: + with mock.patch('airflow.bin.cli.open', mock.mock_open(read_data='42'), create=True) as open_mock: + with mock.patch('os.path.exists') as path_mock: + path_mock.return_value = True + + cli._store_logs_remotely("42", "existing_file") + + # remote base specified, hence one call to open + self.assertEqual(open_mock.call_count, 1) + finally: + configuration.set("core", "REMOTE_BASE_LOG_FOLDER", orig_base_log_folder) + def test_task_state(self): cli.task_state(self.parser.parse_args([ 'task_state', 'example_bash_operator', 'runme_0',