[ https://issues.apache.org/jira/browse/AIRFLOW-6431?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Harish updated AIRFLOW-6431: ---------------------------- Description: airflow with celery executor. a DockerOprator task that runs [the hello-world docker|https://hub.docker.com/_/hello-world] {code:python} from airflow import DAG from airflow.operators.docker_operator import DockerOperator args = { 'owner': 'airflow', 'start_date': airflow.utils.dates.days_ago(2), } dag = DAG('example_xcom', schedule_interval="@once", default_args=args) d = DockerOperator( task_id='hello-world', image='hello-world', # auto_remove=True, do_xcom_push=True, xcom_all=True, tty=True, dag=dag ) {code} 3 issues: # the task log does not show messages printed by the docker to the standard output. # no xcom is created. # an exception is thrown if run with auto_remove=True {code:python} [2020-01-02 19:01:54,438] {docker_operator.py:214} INFO - Starting docker container from image hello-world [2020-01-02 19:01:54,799] {taskinstance.py:1078} ERROR - 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e") Traceback (most recent call last): File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 261, in _raise_for_status response.raise_for_status() File "/usr/local/lib/python3.6/site-packages/requests/models.py", line 940, in raise_for_status raise HTTPError(http_error_msg, response=self) requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http+docker://localhost/v1.35/containers/98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e/json During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/airflow/airflow/models/taskinstance.py", line 945, in _run_raw_task result = task_copy.execute(context=context) File "/opt/airflow/airflow/operators/docker_operator.py", line 283, in execute self._run_image() File "/opt/airflow/airflow/operators/docker_operator.py", line 256, in _run_image if self.xcom_all else line.encode('utf-8') File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped return f(self, resource_id, *args, **kwargs) File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 852, in logs output = self._get_result(container, stream, res) File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 451, in _get_result return self._get_result_tty(stream, res, self._check_is_tty(container)) File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped return f(self, resource_id, *args, **kwargs) File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 447, in _check_is_tty cont = self.inspect_container(container) File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped return f(self, resource_id, *args, **kwargs) File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 755, in inspect_container self._get(self._url("/containers/{0}/json", container)), True File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 267, in _result self._raise_for_status(response) File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 263, in _raise_for_status raise create_api_error_from_http_exception(e) File "/usr/local/lib/python3.6/site-packages/docker/errors.py", line 31, in create_api_error_from_http_exception raise cls(e, response=response, explanation=explanation) docker.errors.NotFound: 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e") [2020-01-02 19:01:54,800] {taskinstance.py:1109} INFO - Marking task as FAILED. [2020-01-02 19:01:59,349] {logging_mixin.py:89} INFO - [2020-01-02 19:01:59,349] {local_task_job.py:98} INFO - Task exited with return code 1 {code} was: airflow with celery executor. a DockerOprator task that runs [the hello-world docker|https://hub.docker.com/_/hello-world] {code:python} from airflow import DAG from airflow.operators.docker_operator import DockerOperatorargs = { 'owner': 'airflow', 'start_date': airflow.utils.dates.days_ago(2), } dag = DAG('example_xcom', schedule_interval="@once", default_args=args)d = DockerOperator( task_id='hello-world', image='hello-world', # auto_remove=True, do_xcom_push=True, xcom_all=True, tty=True, dag=dag ) {code} 3 issues: # the task log does not show messages printed by the docker to the standard output. # no xcom is created. # an exception is thrown if run with auto_remove=True {code:python} [2020-01-02 19:01:54,438] {docker_operator.py:214} INFO - Starting docker container from image hello-world [2020-01-02 19:01:54,799] {taskinstance.py:1078} ERROR - 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e") Traceback (most recent call last): File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 261, in _raise_for_status response.raise_for_status() File "/usr/local/lib/python3.6/site-packages/requests/models.py", line 940, in raise_for_status raise HTTPError(http_error_msg, response=self) requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http+docker://localhost/v1.35/containers/98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e/json During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/airflow/airflow/models/taskinstance.py", line 945, in _run_raw_task result = task_copy.execute(context=context) File "/opt/airflow/airflow/operators/docker_operator.py", line 283, in execute self._run_image() File "/opt/airflow/airflow/operators/docker_operator.py", line 256, in _run_image if self.xcom_all else line.encode('utf-8') File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped return f(self, resource_id, *args, **kwargs) File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 852, in logs output = self._get_result(container, stream, res) File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 451, in _get_result return self._get_result_tty(stream, res, self._check_is_tty(container)) File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped return f(self, resource_id, *args, **kwargs) File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 447, in _check_is_tty cont = self.inspect_container(container) File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped return f(self, resource_id, *args, **kwargs) File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 755, in inspect_container self._get(self._url("/containers/{0}/json", container)), True File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 267, in _result self._raise_for_status(response) File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 263, in _raise_for_status raise create_api_error_from_http_exception(e) File "/usr/local/lib/python3.6/site-packages/docker/errors.py", line 31, in create_api_error_from_http_exception raise cls(e, response=response, explanation=explanation) docker.errors.NotFound: 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e") [2020-01-02 19:01:54,800] {taskinstance.py:1109} INFO - Marking task as FAILED. [2020-01-02 19:01:59,349] {logging_mixin.py:89} INFO - [2020-01-02 19:01:59,349] {local_task_job.py:98} INFO - Task exited with return code 1 {code} > docker operator and xcom > ------------------------ > > Key: AIRFLOW-6431 > URL: https://issues.apache.org/jira/browse/AIRFLOW-6431 > Project: Apache Airflow > Issue Type: Bug > Components: operators, xcom > Affects Versions: 2.0.0 > Reporter: Harish > Priority: Major > > airflow with celery executor. > a DockerOprator task that runs [the hello-world > docker|https://hub.docker.com/_/hello-world] > {code:python} > from airflow import DAG > from airflow.operators.docker_operator import DockerOperator > args = { > 'owner': 'airflow', > 'start_date': airflow.utils.dates.days_ago(2), > } > dag = DAG('example_xcom', schedule_interval="@once", default_args=args) > d = DockerOperator( > task_id='hello-world', > image='hello-world', > # auto_remove=True, > do_xcom_push=True, > xcom_all=True, > tty=True, > dag=dag > ) > {code} > > 3 issues: > # the task log does not show messages printed by the docker to the standard > output. > # no xcom is created. > # an exception is thrown if run with auto_remove=True > {code:python} > [2020-01-02 19:01:54,438] {docker_operator.py:214} INFO - Starting docker > container from image hello-world > [2020-01-02 19:01:54,799] {taskinstance.py:1078} ERROR - 404 Client Error: > Not Found ("No such container: > 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e") > Traceback (most recent call last): > File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line > 261, in _raise_for_status > response.raise_for_status() > File "/usr/local/lib/python3.6/site-packages/requests/models.py", line 940, > in raise_for_status > raise HTTPError(http_error_msg, response=self) > requests.exceptions.HTTPError: 404 Client Error: Not Found for url: > http+docker://localhost/v1.35/containers/98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e/json > During handling of the above exception, another exception occurred: > Traceback (most recent call last): > File "/opt/airflow/airflow/models/taskinstance.py", line 945, in > _run_raw_task > result = task_copy.execute(context=context) > File "/opt/airflow/airflow/operators/docker_operator.py", line 283, in > execute > self._run_image() > File "/opt/airflow/airflow/operators/docker_operator.py", line 256, in > _run_image > if self.xcom_all else line.encode('utf-8') > File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", > line 19, in wrapped > return f(self, resource_id, *args, **kwargs) > File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line > 852, in logs > output = self._get_result(container, stream, res) > File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line > 451, in _get_result > return self._get_result_tty(stream, res, self._check_is_tty(container)) > File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", > line 19, in wrapped > return f(self, resource_id, *args, **kwargs) > File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line > 447, in _check_is_tty > cont = self.inspect_container(container) > File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", > line 19, in wrapped > return f(self, resource_id, *args, **kwargs) > File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line > 755, in inspect_container > self._get(self._url("/containers/{0}/json", container)), True > File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line > 267, in _result > self._raise_for_status(response) > File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line > 263, in _raise_for_status > raise create_api_error_from_http_exception(e) > File "/usr/local/lib/python3.6/site-packages/docker/errors.py", line 31, in > create_api_error_from_http_exception > raise cls(e, response=response, explanation=explanation) > docker.errors.NotFound: 404 Client Error: Not Found ("No such container: > 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e") > [2020-01-02 19:01:54,800] {taskinstance.py:1109} INFO - Marking task as > FAILED. > [2020-01-02 19:01:59,349] {logging_mixin.py:89} INFO - [2020-01-02 > 19:01:59,349] {local_task_job.py:98} INFO - Task exited with return code 1 > {code} > -- This message was sent by Atlassian Jira (v8.3.4#803005)