Repository: incubator-airflow Updated Branches: refs/heads/master ce9c7bbdf -> 49556917d
[AIRFLOW-1057][AIRFLOW-1380][AIRFLOW-2362][2362] AIRFLOW Update DockerOperator to new API update import to docker's new API version >=2.0.0 changed dependency for docker package; now docker rather than docker-py updated test cases to align to new docker class Closes #3407 from Noremac201/fixer Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/49556917 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/49556917 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/49556917 Branch: refs/heads/master Commit: 49556917d9a3e9192d9043afd6cf3ca7939a2403 Parents: ce9c7bb Author: Willem van Asperen <wil...@van.asperen.org> Authored: Thu May 24 10:25:22 2018 +0200 Committer: Fokko Driesprong <fokkodriespr...@godatadriven.com> Committed: Thu May 24 10:25:22 2018 +0200 ---------------------------------------------------------------------- airflow/hooks/docker_hook.py | 4 ++-- airflow/operators/docker_operator.py | 4 ++-- setup.py | 2 +- tests/hooks/test_docker_hook.py | 3 +-- tests/operators/docker_operator.py | 36 +++++++++++++++---------------- 5 files changed, 24 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/airflow/hooks/docker_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/docker_hook.py b/airflow/hooks/docker_hook.py index 14a4a6e..5375c72 100644 --- a/airflow/hooks/docker_hook.py +++ b/airflow/hooks/docker_hook.py @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -from docker import Client +from docker import APIClient from docker.errors import APIError from airflow.exceptions import AirflowException @@ -60,7 +60,7 @@ class DockerHook(BaseHook, LoggingMixin): self.__reauth = False if extra_options.get('reauth') == 'no' else True def get_conn(self): - client = Client( + client = APIClient( base_url=self.__base_url, version=self.__version, tls=self.__tls http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/airflow/operators/docker_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py index 9d535ff..421ce49 100644 --- a/airflow/operators/docker_operator.py +++ b/airflow/operators/docker_operator.py @@ -24,7 +24,7 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults from airflow.utils.file import TemporaryDirectory -from docker import Client, tls +from docker import APIClient, tls import ast @@ -166,7 +166,7 @@ class DockerOperator(BaseOperator): if self.docker_conn_id: self.cli = self.get_hook().get_conn() else: - self.cli = Client( + self.cli = APIClient( base_url=self.docker_url, version=self.api_version, tls=tls_config http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 6d53ce5..dcd59af 100644 --- a/setup.py +++ b/setup.py @@ -136,7 +136,7 @@ doc = [ 'sphinx-rtd-theme>=0.1.6', 'Sphinx-PyPI-upload>=0.2.1' ] -docker = ['docker-py>=1.6.0'] +docker = ['docker>=2.0.0'] druid = ['pydruid>=0.4.1'] elasticsearch = [ 'elasticsearch>=5.0.0,<6.0.0', http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/tests/hooks/test_docker_hook.py ---------------------------------------------------------------------- diff --git a/tests/hooks/test_docker_hook.py b/tests/hooks/test_docker_hook.py index 5753aa5..b8a0132 100644 --- a/tests/hooks/test_docker_hook.py +++ b/tests/hooks/test_docker_hook.py @@ -26,7 +26,6 @@ from airflow.utils import db try: from airflow.hooks.docker_hook import DockerHook - from docker import Client except ImportError: pass @@ -39,7 +38,7 @@ except ImportError: mock = None -@mock.patch('airflow.hooks.docker_hook.Client', autospec=True) +@mock.patch('airflow.hooks.docker_hook.APIClient', autospec=True) class DockerHookTest(unittest.TestCase): def setUp(self): configuration.load_test_config() http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/tests/operators/docker_operator.py ---------------------------------------------------------------------- diff --git a/tests/operators/docker_operator.py b/tests/operators/docker_operator.py index 1ebcbe9..78a920c 100644 --- a/tests/operators/docker_operator.py +++ b/tests/operators/docker_operator.py @@ -23,7 +23,7 @@ import logging try: from airflow.operators.docker_operator import DockerOperator from airflow.hooks.docker_hook import DockerHook - from docker import Client + from docker import APIClient except ImportError: pass @@ -40,12 +40,12 @@ except ImportError: class DockerOperatorTestCase(unittest.TestCase): @mock.patch('airflow.utils.file.mkdtemp') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute(self, client_class_mock, mkdtemp_mock): host_config = mock.Mock() mkdtemp_mock.return_value = '/mkdtemp' - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = host_config client_mock.images.return_value = [] @@ -84,9 +84,9 @@ class DockerOperatorTestCase(unittest.TestCase): client_mock.wait.assert_called_with('some_id') @mock.patch('airflow.operators.docker_operator.tls.TLSConfig') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_tls(self, client_class_mock, tls_class_mock): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = mock.Mock() client_mock.images.return_value = [] @@ -104,15 +104,15 @@ class DockerOperatorTestCase(unittest.TestCase): operator.execute(None) tls_class_mock.assert_called_with(assert_hostname=None, ca_cert='ca.pem', - client_cert=('cert.pem', 'key.pem'), ssl_version=None, - verify=True) + client_cert=('cert.pem', 'key.pem'), + ssl_version=None, verify=True) - client_class_mock.assert_called_with(base_url='https://127.0.0.1:2376', tls=tls_mock, - version=None) + client_class_mock.assert_called_with(base_url='https://127.0.0.1:2376', + tls=tls_mock, version=None) - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_unicode_logs(self, client_class_mock): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = mock.Mock() client_mock.images.return_value = [] @@ -132,9 +132,9 @@ class DockerOperatorTestCase(unittest.TestCase): logging.raiseExceptions = originalRaiseExceptions print_exception_mock.assert_not_called() - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_container_fails(self, client_class_mock): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.create_host_config.return_value = mock.Mock() client_mock.images.return_value = [] @@ -150,7 +150,7 @@ class DockerOperatorTestCase(unittest.TestCase): operator.execute(None) def test_on_kill(self): - client_mock = mock.Mock(spec=Client) + client_mock = mock.Mock(spec=APIClient) operator = DockerOperator(image='ubuntu', owner='unittest', task_id='unittest') operator.cli = client_mock @@ -160,10 +160,10 @@ class DockerOperatorTestCase(unittest.TestCase): client_mock.stop.assert_called_with('some_id') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_no_docker_conn_id_no_hook(self, operator_client_mock): # Mock out a Docker client, so operations don't raise errors - client_mock = mock.Mock(name='DockerOperator.Client mock', spec=Client) + client_mock = mock.Mock(name='DockerOperator.APIClient mock', spec=APIClient) client_mock.images.return_value = [] client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.logs.return_value = [] @@ -194,11 +194,11 @@ class DockerOperatorTestCase(unittest.TestCase): ) @mock.patch('airflow.operators.docker_operator.DockerHook') - @mock.patch('airflow.operators.docker_operator.Client') + @mock.patch('airflow.operators.docker_operator.APIClient') def test_execute_with_docker_conn_id_use_hook(self, operator_client_mock, operator_docker_hook): # Mock out a Docker client, so operations don't raise errors - client_mock = mock.Mock(name='DockerOperator.Client mock', spec=Client) + client_mock = mock.Mock(name='DockerOperator.APIClient mock', spec=APIClient) client_mock.images.return_value = [] client_mock.create_container.return_value = {'Id': 'some_id'} client_mock.logs.return_value = []