nhuantho commented on PR #64032: URL: https://github.com/apache/airflow/pull/64032#issuecomment-4474120647
### Patch Validation on Kubernetes * Kubernetes version: v1.31.4 * Airflow version: 3.2.1 * Chart version: latest `main` branch including this patch * Result: `helmfile apply` completed successfully, and the patch works as expected. <img width="1412" height="714" alt="image" src="https://github.com/user-attachments/assets/6e3e426b-024a-4788-a748-445b1dc7fc5b"; /> <img width="1933" height="171" alt="image" src="https://github.com/user-attachments/assets/42a3ae0e-27fa-461c-b497-963c3bc85de4"; /> * values.yaml ```yaml airflowHome: /opt/airflow defaultAirflowRepository: registry.xxx.com/dataops/airflow # Default airflow tag to deploy defaultAirflowTag: "3.2.1-python3.12-dbt1.11-dag-factory1.1.0" # Airflow version (Used to make some decisions based on Airflow Version being deployed) airflowVersion: "3.2.1" images: statsd: repository: quay.io/prometheus/statsd-exporter tag: v0.28.0 pullPolicy: IfNotPresent redis: repository: redis tag: 7.2-bookworm pullPolicy: IfNotPresent gitSync: repository: registry.k8s.io/git-sync/git-sync tag: v4.4.2 pullPolicy: IfNotPresent # Ingress configuration ingress: apiServer: enabled: true annotations: ingress.kubernetes.io/force-ssl-redirect: "true" nginx.ingress.kubernetes.io/cors-allow-methods: GET nginx.ingress.kubernetes.io/enable-cors: "true" kubernetes.io/tls-acme: "true" nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" hosts: - name: cronjobs.xxx.xxx.com tls: enabled: true secretName: cronjobs.xxx.xxx.com-tls ingressClassName: nginx # Enable RBAC (default on most clusters these days) rbac: # Specifies whether RBAC resources should be created create: true # Airflow executor # One or multiple of: LocalExecutor, CeleryExecutor, KubernetesExecutor # For Airflow <3.0, LocalKubernetesExecutor and CeleryKubernetesExecutor are also supported. # Specify executors in a prioritized list to leverage multiple execution environments as needed: # https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/executor/index.html#using-multiple-executors-concurrently executor: "CeleryExecutor" # If this is true and using LocalExecutor/KubernetesExecutor/CeleryKubernetesExecutor, the scheduler's # service account will have access to communicate with the api-server and launch pods. # If this is true and using CeleryExecutor/KubernetesExecutor/CeleryKubernetesExecutor, the workers # will be able to launch pods. allowPodLaunching: true # Enables selected built-in secrets that are set via environment variables by default. # Those secrets are provided by the Helm Chart secrets by default but in some cases you # might want to provide some of those variables with _CMD or _SECRET variable, and you should # in this case disable setting of those variables by setting the relevant configuration to false. enableBuiltInSecretEnvVars: AIRFLOW__ELASTICSEARCH__HOST: false AIRFLOW__OPENSEARCH__HOST: false extraEnv: | - name: AIRFLOW__CORE__TEST_CONNECTION value: 'Enabled' - name: ENV_CRONJOBS value: 'xxx' # Airflow database data: metadataConnection: user: {{ .Values.database.username | fetchSecretValue | quote}} pass: {{ .Values.database.password | fetchSecretValue | quote}} protocol: postgresql host: infradb.xxx.xxx.com port: 6432 db: cronjobs sslmode: disable # Fernet key settings # Note: fernetKey can only be set during install, not upgrade fernetKey: {{ .Values.fernetKey | fetchSecretValue }} # Flask secret key for Airflow 3+ Api: `[api] secret_key` in airflow.cfg apiSecretKey: {{ .Values.apiServer.secretKey | fetchSecretValue }} # Secret key used to encode and decode JWTs: `[api_auth] jwt_secret` in airflow.cfg jwtSecret: {{ .Values.jwtSecret | fetchSecretValue }} # Airflow scheduler settings scheduler: # Airflow 2.0 allows users to run multiple schedulers, # However this feature is only recommended for MySQL 8+ and Postgres replicas: 1 resources: requests: cpu: 0.5 memory: 2.0Gi limits: cpu: 2.0 memory: 4.0Gi logGroomerSidecar: resources: requests: cpu: 0.25 memory: 0.5Gi limits: cpu: 0.5 memory: 1.0Gi apiServer: # Number of airflow apiServer in the deployment replicas: 1 resources: requests: cpu: 100m memory: 2.0Gi limits: cpu: 2.0 memory: 4.0Gi apiServerConfig: | {{ tpl (readFile "webserver_config.py") . | nindent 4 }} webserver: enabled: false # Airflow Triggerer Config triggerer: # Number of airflow triggerers in the deployment replicas: 1 persistence: # Enable persistent volumes enabled: true # Volume size for triggerer StatefulSet size: 2Gi # If using a custom storageClass, pass name ref to all statefulSets here storageClassName: ceph-rbd-hdd resources: requests: cpu: 100m memory: 1.0Gi limits: cpu: 2.0 memory: 2.0Gi logGroomerSidecar: resources: requests: cpu: 0.25 memory: 0.5Gi limits: cpu: 0.5 memory: 1.0Gi # Airflow Worker Config workers: celery: # Number of airflow celery workers in StatefulSet replicas: 1 persistence: # Enable persistent volumes enabled: true # Volume size for worker StatefulSet size: 2Gi # If using a custom storageClass, pass name ref to all statefulSets here storageClassName: ceph-rbd-hdd resources: requests: cpu: 0.5 memory: 4.0Gi limits: cpu: 2.0 memory: 8.0Gi logGroomerSidecar: resources: requests: cpu: 0.25 memory: 0.5Gi limits: cpu: 0.5 memory: 1.0Gi # Airflow Dag Processor Config dagProcessor: enabled: true # Number of airflow dag processors in the deployment replicas: 1 resources: requests: cpu: 500m memory: 1.0Gi limits: cpu: 2.0 memory: 2.0Gi logGroomerSidecar: resources: requests: cpu: 0.25 memory: 0.5Gi limits: cpu: 0.5 memory: 1.0Gi # Flower settings flower: # Enable flower. # If True, and using CeleryExecutor/CeleryKubernetesExecutor, will deploy flower app. enabled: true resources: requests: cpu: 0.2 memory: 2.0Gi limits: cpu: 1.0 memory: 4.0Gi # StatsD settings statsd: enabled: true resources: requests: cpu: 0.5 memory: 1.0Gi limits: cpu: 1 memory: 4.0Gi # Configuration for the redis provisioned by the chart redis: enabled: true persistence: # Enable persistent volumes enabled: true # Volume size for worker StatefulSet size: 2Gi # If using a custom storageClass, pass name ref to all statefulSets here storageClassName: ceph-rbd-hdd resources: requests: cpu: 0.2 memory: 1.0Gi limits: cpu: 1.0 memory: 2.0Gi password: {{ .Values.redis.password | fetchSecretValue }} postgresql: enabled: false # Config settings to go into the mounted airflow.cfg config: core: hide_sensitive_var_conn_fields: false killed_task_cleanup_time: 300 max_active_tasks_per_dag: 16 max_active_runs_per_dag: 16 parallelism: 128 auth_manager: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager logging: base_log_folder: /opt/airflow/logs/ logging_level: INFO remote_base_log_folder: s3://apps-airflow/logs/ remote_log_conn_id: airflow-logs-conn remote_logging: true fab: enable_proxy_fix: 'True' celery: worker_concurrency: 16 email: default_email_on_retry: false default_email_on_failure: false smtp: smtp_host: smtp.sendgrid.net smtp_port: 587 smtp_starttls: true smtp_user: {{ .Values.smtp.username | fetchSecretValue }} smtp_password: {{ .Values.smtp.password | fetchSecretValue }} smtp_mail_from: "Data's Airflow service <[email protected]>" scheduler: max_dagruns_to_create_per_loop: 100 # default 10 max_dagruns_per_loop_to_schedule: 200 # default 20 schedule_after_task_execution: false # https://stackoverflow.com/a/71196200 scheduler_idle_sleep_time: 2 # Longer sleep time, less cpu usage api: auth_backends: airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session base_url: https://cronjobs.xxx.xxx.com enable_swagger_ui: 'True' expose_config: 'True' api_auth: jwt_secret: {{ .Values.jwtSecret | fetchSecretValue }} # Git sync dags: persistence: enabled: false gitSync: enabled: true repo: https://{{ .Values.git.username | fetchSecretValue }}:{{ .Values.git.password | fetchSecretValue }}@git.xxx.vn/xxx/cronjobs branch: main subPath: "" period: 60s containerName: git-sync resources: requests: cpu: 0.25 memory: 0.5Gi limits: cpu: 0.5 memory: 1.0Gi emptyDirConfig: sizeLimit: 1Gi logs: persistence: # Enable persistent volume for storing logs enabled: false ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
