This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f3cc5f505b feat: support mysql backend as keda trigger (#36167)
f3cc5f505b is described below
commit f3cc5f505bf6dad696d3cb496a77bc3a0f5bf45d
Author: Forlot Romain <[email protected]>
AuthorDate: Sat Jan 20 22:10:35 2024 +0100
feat: support mysql backend as keda trigger (#36167)
---
chart/templates/_helpers.yaml | 2 +-
.../secrets/metadata-connection-secret.yaml | 4 +++
chart/templates/workers/worker-kedaautoscaler.yaml | 11 ++++++-
helm_tests/other/test_keda.py | 36 ++++++++++++++++++++++
4 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 69e7dae634..59a28fd9b7 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -84,7 +84,7 @@ If release name contains chart name it will be used as a full
name.
name: {{ template "airflow_metadata_secret" . }}
key: connection
{{- end }}
- {{- if and .Values.workers.keda.enabled .Values.pgbouncer.enabled (not
.Values.workers.keda.usePgbouncer) }}
+ {{- if and .Values.workers.keda.enabled (or (eq
.Values.data.metadataConnection.protocol "mysql") (and
.Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer))) }}
- name: KEDA_DB_CONN
valueFrom:
secretKeyRef:
diff --git a/chart/templates/secrets/metadata-connection-secret.yaml
b/chart/templates/secrets/metadata-connection-secret.yaml
index c25156d118..220a516682 100644
--- a/chart/templates/secrets/metadata-connection-secret.yaml
+++ b/chart/templates/secrets/metadata-connection-secret.yaml
@@ -51,5 +51,9 @@ data:
{{- with .Values.data.metadataConnection }}
kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf
"%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s"
$metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query"
$query) | b64enc | quote }}
{{- end }}
+ {{- else if and .Values.workers.keda.enabled (eq
.Values.data.metadataConnection.protocol "mysql") }}
+ {{- with .Values.data.metadataConnection }}
+ kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" (.user |
urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost
$metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc
| quote }}
+ {{- end }}
{{- end }}
{{- end }}
diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml
b/chart/templates/workers/worker-kedaautoscaler.yaml
index f93484efd0..95adb3e126 100644
--- a/chart/templates/workers/worker-kedaautoscaler.yaml
+++ b/chart/templates/workers/worker-kedaautoscaler.yaml
@@ -47,7 +47,14 @@ spec:
advanced: {{- toYaml .Values.workers.keda.advanced | nindent 4 }}
{{- end }}
triggers:
- - type: postgresql
+ {{- if eq .Values.data.metadataConnection.protocol "mysql" }}
+ - type: "mysql"
+ metadata:
+ queryValue: "1"
+ connectionStringFromEnv: KEDA_DB_CONN
+ query: {{ tpl .Values.workers.keda.query . | quote }}
+ {{- else }}
+ - type: "postgresql"
metadata:
targetQueryValue: "1"
{{- if and .Values.pgbouncer.enabled (not
.Values.workers.keda.usePgbouncer) }}
@@ -56,4 +63,6 @@ spec:
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
{{- end }}
query: {{ tpl .Values.workers.keda.query . | quote }}
+ {{- end }}
+
{{- end }}
diff --git a/helm_tests/other/test_keda.py b/helm_tests/other/test_keda.py
index bf15cbfaa6..07c07fc91e 100644
--- a/helm_tests/other/test_keda.py
+++ b/helm_tests/other/test_keda.py
@@ -275,3 +275,39 @@ class TestKeda:
"spec.triggers[0].metadata.connectionFromEnv", keda_autoscaler
)
assert autoscaler_connection_env_var == "KEDA_DB_CONN"
+
+ def test_mysql_keda_db_connection(self):
+ """Verify keda db connection when pgbouncer is enabled."""
+ docs = render_chart(
+ values={
+ "data": {"metadataConnection": {"protocol": "mysql", "port":
3306}},
+ "workers": {"keda": {"enabled": True}},
+ "executor": "CeleryExecutor",
+ },
+ show_only=[
+ "templates/workers/worker-deployment.yaml",
+ "templates/workers/worker-kedaautoscaler.yaml",
+ "templates/secrets/metadata-connection-secret.yaml",
+ ],
+ )
+ worker_deployment = docs[0]
+ keda_autoscaler = docs[1]
+ metadata_connection_secret = docs[2]
+
+ worker_container_env_vars = jmespath.search(
+ "spec.template.spec.containers[?name=='worker'].env[].name",
worker_deployment
+ )
+ assert "AIRFLOW_CONN_AIRFLOW_DB" in worker_container_env_vars
+ assert "KEDA_DB_CONN" in worker_container_env_vars
+
+ keda_autoscaler_metadata =
jmespath.search("spec.triggers[0].metadata", keda_autoscaler)
+ assert "queryValue" in keda_autoscaler_metadata
+
+ secret_data = jmespath.search("data", metadata_connection_secret)
+ assert "connection" in secret_data.keys()
+ assert "kedaConnection" in secret_data.keys()
+
+ autoscaler_connection_env_var = jmespath.search(
+ "spec.triggers[0].metadata.connectionStringFromEnv",
keda_autoscaler
+ )
+ assert autoscaler_connection_env_var == "KEDA_DB_CONN"
