This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b9d12033985 Add standalone DAG processor to docker compose (#45650)
b9d12033985 is described below
commit b9d12033985c3066cd5d1232b80c66a4c7689be9
Author: Jed Cunningham <[email protected]>
AuthorDate: Tue Jan 14 13:04:47 2025 -0700
Add standalone DAG processor to docker compose (#45650)
We are moving to only support a standalone DAG processor in Airflow 3,
so let's switch our compose setup to have it. We can remove the env var
once that's the only option, but for now it let's us avoid a massive PR
:)
---
.../howto/docker-compose/docker-compose.yaml | 16 ++++++++++++++++
docs/apache-airflow/howto/docker-compose/index.rst | 1 +
2 files changed, 17 insertions(+)
diff --git a/docs/apache-airflow/howto/docker-compose/docker-compose.yaml
b/docs/apache-airflow/howto/docker-compose/docker-compose.yaml
index eb3656940e7..c64eac5b375 100644
--- a/docs/apache-airflow/howto/docker-compose/docker-compose.yaml
+++ b/docs/apache-airflow/howto/docker-compose/docker-compose.yaml
@@ -60,6 +60,7 @@ x-airflow-common:
AIRFLOW__CORE__FERNET_KEY: ''
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
AIRFLOW__CORE__LOAD_EXAMPLES: 'true'
+ AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR: 'true'
AIRFLOW__API__AUTH_BACKENDS: >-
airflow.providers.fab.auth_manager.api.auth.backend.basic_auth,airflow.api.auth.backend.session
# yamllint disable rule:line-length
@@ -148,6 +149,21 @@ services:
airflow-init:
condition: service_completed_successfully
+ airflow-dag-processor:
+ <<: *airflow-common
+ command: dag-processor
+ healthcheck:
+ test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob
--hostname "$${HOSTNAME}"']
+ interval: 30s
+ timeout: 10s
+ retries: 5
+ start_period: 30s
+ restart: always
+ depends_on:
+ <<: *airflow-common-depends-on
+ airflow-init:
+ condition: service_completed_successfully
+
airflow-worker:
<<: *airflow-common
command: celery worker
diff --git a/docs/apache-airflow/howto/docker-compose/index.rst
b/docs/apache-airflow/howto/docker-compose/index.rst
index 65151c22e06..54cde82635e 100644
--- a/docs/apache-airflow/howto/docker-compose/index.rst
+++ b/docs/apache-airflow/howto/docker-compose/index.rst
@@ -88,6 +88,7 @@ This file contains several service definitions:
- ``airflow-scheduler`` - The :doc:`scheduler
</administration-and-deployment/scheduler>` monitors all tasks and DAGs, then
triggers the
task instances once their dependencies are complete.
+- ``airflow-dag-processor`` - The DAG processor parses DAG files.
- ``airflow-webserver`` - The webserver is available at
``http://localhost:8080``.
- ``airflow-worker`` - The worker that executes the tasks given by the
scheduler.
- ``airflow-triggerer`` - The triggerer runs an event loop for deferrable
tasks.
