GitHub user ahipp13 added a comment to the discussion: Remote Logging to Azure
Blob Storage Still Broke in Airflow 3.0.4
All environment variables I have set are below, as well as the airflow.cfg that
appears on the pods. I have never set anything in my helm chart for a secrets
backend. NOTE some values are changed/omitted
```
env:
- name: "AIRFLOW_HOME"
value: "/opt/airflow"
- name: "AIRFLOW__LOGGING__LOGGING_LEVEL"
value: "INFO"
- name: "AIRFLOW__CORE__LOAD_EXAMPLES"
value: "False"
- name: "AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL"
value: "10"
# Increasing paralelism to run more tasks
- name: "AIRFLOW__CORE__PARALLELISM"
value: "48"
### EDW CHANGES ENV ###
- name: "AIRFLOW__WEBSERVER__BASE_URL"
value:
- name: "AIRFLOW__API__BASE_URL"
value:
### EDW CHANGES ENV ###
# Fixes IntegrityError
- name: "AIRFLOW__FAB__SESSION_BACKEND"
value: "securecookie"
# Setting these next 5 variables as Airflow 3.0 needs them
- name: "AIRFLOW__FAB__EXPOSE_HOSTNAME"
value: "False"
- name: "AIRFLOW__FAB__NAVBAR_COLOR"
value: "#fff"
- name: "AIRFLOW__FAB__NAVBAR_HOVER_COLOR"
value: "#eee"
- name: "AIRFLOW__FAB__NAVBAR_TEXT_COLOR"
value: "#51504f"
- name: "AIRFLOW__FAB__NAVBAR_TEXT_HOVER_COLOR"
value: "#51504f"
# Vars to tell what ENV we are in for DAGs
### EDW CHANGES ENV ###
- name: "ENV"
value: "sand"
- name: "ENV_CAP"
value: "SAND"
- name: "ENV_CAP_CONCAT"
value: "SND"
### EDW CHANGES ENV ###
# Email list for each team
### EDW CHANGES ENV ###
- name: "E_EMAILLIST"
value:
- name: "SE_EMAILLIST"
value:
### EDW CHANGES ENV ###
- name: "ASSE_EMAILLIST"
value:
# Remote Logging
### EDW CHANGES ENV ###
- name: "AIRFLOW_CONN_AZURE_BLOB_STORAGE"
value:
### EDW CHANGES ENV ###
- name: "AIRFLOW__LOGGING__REMOTE_LOGGING"
value: "True"
- name: "AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER"
value: "wasb-airflow-logs"
- name: "AIRFLOW__AZURE_REMOTE_LOGGING__REMOTE_WASB_LOG_CONTAINER"
value: "airflow-logs"
- name: "AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID"
value: "AZURE_BLOB_STORAGE"
- name: "AIRFLOW__LOGGING__DELETE_LOCAL_LOGS"
value: "True"
# Timeout for UI
- name: "AIRFLOW__WEBSERVER__SESSION_LIFETIME_MINUTES"
value: "30"
- name: "AIRFLOW__FAB__SESSION_LIFETIME_MINUTES"
value: "30"
# SMTP Email configs (for email alerts on DAGs)
### EDW CHANGES ENV ###
- name: "AIRFLOW__SMTP__SMTP_HOST"
value:
### EDW CHANGES ENV ###
- name: "AIRFLOW__SMTP__SMTP_STARTTLS"
value: "False"
### EDW CHANGES ENV ###
- name: "AIRFLOW__SMTP__SMTP_MAIL_FROM"
value:
### EDW CHANGES ENV ###
- name: "AIRFLOW__SMTP__SMTP_PORT"
value: "25"
- name: AIRFLOW__EMAIL__SUBJECT_TEMPLATE
value: "/opt/airflow/EDW_subject_template_file"
# Increasing Import Timeout on DAGs
- name: AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT
value: "200.0"
# Azure variables for EDW
- name: "ED_AZURE_TENANT_ID"
value:
### EDW CHANGES ENV ###
- name: "EI_DEV_BLOBSTORAGE_CLIENT_ID"
value:
### EDW CHANGES ENV ###
# Azure key vault env variables for Security Platform
- name: "S_AZURE_TENANT_ID"
value:
- name: "S_AZURE_CLIENT_ID"
value:
- name: "S_AZURE_CLIENT_SECRET"
value: "S_CLIENT_SECRET"
- name: "S_AZURE_KEY_VAULT_URI"
value:
# AWS key vault env variables for Security Platform
- name: "S_AWS_ACCESS_KEY"
value: "S_ACCESS_KEY"
- name: "S_AWS_SECRET_ACCESS_KEY"
value: "S_SECRET_ACCESS_KEY"
# Azure key vault env variables for EIS Reporting with "SecPlat"
- name: "E"
value: ""
- name: "E_AZURE_CLIENT_ID"
value:
- name: "E_AZURE_CLIENT_SECRET"
value: "E_CLIENT_SECRET"
- name: "E_AZURE_KEY_VAULT_URI"
value:
### EDW CHANGES ENV ###
- name: "EDW_HV_URL"
value:
- name: "EDW_HV_ENV"
value: "nonprod"
### EDW CHANGES ENV ###
- name: "AIRFLOW__CORE__SENSITIVE_VAR_CONN_NAMES"
value: "tpu-auth"
```
airflow.cfg:
```
[celery]
flower_url_prefix =
worker_concurrency = 16
[celery_kubernetes_executor]
kubernetes_queue = kubernetes
[core]
auth_manager =
airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
colored_console_log = False
dags_folder = /opt/airflow/dag_files/dags
execution_api_server_url =
http://airflow-sand-deployment-api-server:8080/execution/
executor = KubernetesExecutor
load_examples = False
remote_logging = False
[elasticsearch]
json_format = True
log_id_template = {dag_id}_{task_id}_{execution_date}_{try_number}
[elasticsearch_configs]
max_retries = 3
retry_timeout = True
timeout = 30
[fab]
enable_proxy_fix = True
[kerberos]
ccache = /var/kerberos-ccache/cache
keytab = /etc/airflow.keytab
principal = [email protected]
reinit_frequency = 3600
[kubernetes]
airflow_configmap = airflow-sand-deployment-config
airflow_local_settings_configmap = airflow-sand-deployment-config
multi_namespace_mode = False
namespace = eis-dw-sand-k8s
pod_template_file = /opt/airflow/pod_templates/pod_template_file.yaml
worker_container_repository =REPO/airflow/airflow
worker_container_tag = sandbuild-50_16967071098_2025-08-14
[kubernetes_executor]
multi_namespace_mode = False
namespace = NAMESPACE
pod_template_file = /opt/airflow/pod_templates/pod_template_file.yaml
worker_container_repository = REPO/airflow/airflow
worker_container_tag = sandbuild-50_16967071098_2025-08-14
[logging]
colored_console_log = False
remote_logging = False
[metrics]
statsd_host = airflow-sand-deployment-statsd
statsd_on = True
statsd_port = 9125
statsd_prefix = airflow
[scheduler]
run_duration = 41460
standalone_dag_processor = True
statsd_host = airflow-sand-deployment-statsd
statsd_on = True
statsd_port = 9125
statsd_prefix = airflow
[webserver]
enable_proxy_fix = True
```
GitHub link:
https://github.com/apache/airflow/discussions/54489#discussioncomment-14119162
----
This is an automatically sent email for [email protected].
To unsubscribe, please send an email to: [email protected]
