This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new 5fbbb908236 [v3-4-test] Cover YAML-generated TypeScript Dags in the
ts_sdk e2e test (#75054) (#75595)
5fbbb908236 is described below
commit 5fbbb908236610855f8c697ec46f6ed71332ed13
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Oct 25 08:23:31 2026 +0000
[v3-4-test] Cover YAML-generated TypeScript Dags in the ts_sdk e2e test
(#75054) (#75595)
The Dag processor runs a native TypeScript Dag's bundle on every
re-parse, and the worker runs the same bundle for each task. So a
bundle can build its Dags from a file in the Dag bundle, and swapping
only that file should change the live Dags with no rebuild. The go_sdk
and java_sdk e2e tests cover this, but nothing covered it for
TypeScript.
(cherry picked from commit f5329a6c8043c10cd5f3e06c436df6295144fc20)
Co-authored-by: Andrew Chang <[email protected]>
---
airflow-e2e-tests/docker/ts.yml | 6 ++
.../ts_sdk_tests/test_ts_sdk_dynamic_dag.py | 89 ++++++++++++++++++++++
ts-sdk/example/package.json | 1 +
ts-sdk/example/src/dynamic.ts | 51 +++++++++++++
ts-sdk/example/src/main.ts | 8 +-
ts-sdk/pnpm-lock.yaml | 3 +
6 files changed, 156 insertions(+), 2 deletions(-)
diff --git a/airflow-e2e-tests/docker/ts.yml b/airflow-e2e-tests/docker/ts.yml
index 9306f57e3dd..90a1642b4e9 100644
--- a/airflow-e2e-tests/docker/ts.yml
+++ b/airflow-e2e-tests/docker/ts.yml
@@ -34,6 +34,12 @@ services:
- nodejs-bin:/opt/nodejs
airflow-dag-processor:
+ environment:
+ # The dynamic Dag test swaps a YAML file and waits for the Dag IDs to
change. Re-parse the
+ # bundle every few seconds, and mark a Dag stale soon after it leaves
the YAML.
+ AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL: '5'
+ AIRFLOW__DAG_PROCESSOR__STALE_DAG_THRESHOLD: '10'
+ AIRFLOW__SCHEDULER__PARSING_CLEANUP_INTERVAL: '5'
volumes:
- ./ts-bundles:/opt/airflow/ts-bundles:ro
- nodejs-bin:/opt/nodejs:ro
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/ts_sdk_tests/test_ts_sdk_dynamic_dag.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/ts_sdk_tests/test_ts_sdk_dynamic_dag.py
new file mode 100644
index 00000000000..2bdf64661b4
--- /dev/null
+++
b/airflow-e2e-tests/tests/airflow_e2e_tests/ts_sdk_tests/test_ts_sdk_dynamic_dag.py
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""E2E test for the native TypeScript Dags that
``ts-sdk/example/src/dynamic.ts`` generates from a YAML file.
+
+Run with::
+
+ E2E_TEST_MODE=ts_sdk uv run --project airflow-e2e-tests pytest \\
+ tests/airflow_e2e_tests/ts_sdk_tests/test_ts_sdk_dynamic_dag.py -xvs
+
+``conftest._setup_ts_sdk_integration`` puts the example bundle into the Dags
folder, with no ``dags.yaml``
+next to it yet. The bundle generates one Dag for each entry of the YAML the
test writes there, so the test
+swaps only the YAML, with no rebuild, and checks that the live Dag IDs follow
it and that each live Dag
+runs.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from airflow_e2e_tests.e2e_test_utils.clients import AirflowClient
+from airflow_e2e_tests.e2e_test_utils.dynamic_dags import swap_yaml_and_check
+
+# The Dag processor runs node on the bundle to parse it, so the Dags appear
some time after the stack is up.
+_PARSE_TIMEOUT = 600
+# After that, the Dag processor re-parses every few seconds (see ts.yml), and
a Dag that left the
+# YAML goes stale a few seconds after that.
+_RELOAD_TIMEOUT = 180
+_TS_TASK_TIMEOUT = 600
+
+_TAGS = ["typescript_dynamic"]
+
+
+def test_swapping_the_yaml_changes_the_live_dags(airflow_dags_path: Path):
+ client = AirflowClient()
+ yaml_path = airflow_dags_path / "typescript" / "dags.yaml"
+
+ live = swap_yaml_and_check(
+ client,
+ yaml_path,
+ {
+ "typescript_dynamic_a": "hello from a",
+ "typescript_dynamic_b": "hello from b",
+ "typescript_dynamic_c": "hello from c",
+ },
+ previous=set(),
+ tags=_TAGS,
+ timeout=_PARSE_TIMEOUT,
+ task_timeout=_TS_TASK_TIMEOUT,
+ )
+ live = swap_yaml_and_check(
+ client,
+ yaml_path,
+ {
+ "typescript_dynamic_a": "hello from a",
+ "typescript_dynamic_b": "hello from b",
+ "typescript_dynamic_c": "hello from c",
+ "typescript_dynamic_d": "hello from d",
+ "typescript_dynamic_e": "hello from e",
+ },
+ previous=live,
+ tags=_TAGS,
+ timeout=_RELOAD_TIMEOUT,
+ task_timeout=_TS_TASK_TIMEOUT,
+ )
+ # a keeps its Dag ID but gets a new greeting. The greeting is not in the
serialized Dag, so its run
+ # shows that the worker built the Dag from the new YAML, not that the Dag
processor rebuilt it.
+ swap_yaml_and_check(
+ client,
+ yaml_path,
+ {"typescript_dynamic_a": "hello again from a", "typescript_dynamic_f":
"hello from f"},
+ previous=live,
+ tags=_TAGS,
+ timeout=_RELOAD_TIMEOUT,
+ task_timeout=_TS_TASK_TIMEOUT,
+ )
diff --git a/ts-sdk/example/package.json b/ts-sdk/example/package.json
index d5e10ab3f57..35c74956eab 100644
--- a/ts-sdk/example/package.json
+++ b/ts-sdk/example/package.json
@@ -16,6 +16,7 @@
"devDependencies": {
"@types/node": "^26.1.2",
"esbuild": "^0.28.2",
+ "js-yaml": "^5.4.1",
"typescript": "^6.0.2"
}
}
diff --git a/ts-sdk/example/src/dynamic.ts b/ts-sdk/example/src/dynamic.ts
new file mode 100644
index 00000000000..fbb0edba580
--- /dev/null
+++ b/ts-sdk/example/src/dynamic.ts
@@ -0,0 +1,51 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Native Dags generated from the `dags.yaml` next to the bundle, one per
entry.
+//
+// Both the Dag processor and the worker run the bundle, and each run reads the
+// YAML again, so swapping only the YAML changes the Dags with no rebuild.
+// Without a `dags.yaml` next to the bundle, as when `airflow-ts-pack` runs it
to
+// read its metadata, there are no Dags, and the bundle's other Dags still
parse.
+
+import { existsSync, readFileSync } from "node:fs";
+
+import { Dag } from "apache-airflow-ts-sdk";
+import { load } from "js-yaml";
+
+interface DagsFile {
+ readonly dags?: readonly { readonly dag_id: string; readonly greeting:
string }[];
+}
+
+export function loadDynamicDags(): Dag[] {
+ const path = new URL("dags.yaml", import.meta.url);
+ if (!existsSync(path)) {
+ return [];
+ }
+ const file = load(readFileSync(path, "utf-8")) as DagsFile;
+ return (file.dags ?? []).map(({ dag_id, greeting }) => {
+ const dag = new Dag(dag_id, {
+ catchup: false,
+ tags: ["typescript", "typescript_dynamic"],
+ queue: "typescript",
+ });
+ dag.task("greet", async () => greeting)();
+ return dag;
+ });
+}
diff --git a/ts-sdk/example/src/main.ts b/ts-sdk/example/src/main.ts
index 756ec0fb762..2aea8ee5875 100644
--- a/ts-sdk/example/src/main.ts
+++ b/ts-sdk/example/src/main.ts
@@ -21,10 +21,12 @@
//
// The two Dags in `dags/` are declared in Python with `@task.stub` tasks
routed to the Node
// coordinator, so this side only supplies their task bodies. The third, in
`native.ts`, is
-// declared in TypeScript outright — graph, schedule and all.
+// declared in TypeScript outright — graph, schedule and all. `dynamic.ts`
adds a native Dag for
+// each entry of a YAML file next to the bundle.
import { Bundle, getClient, getContext, NEVER_EXPIRE, TaskHandler } from
"apache-airflow-ts-sdk";
+import { loadDynamicDags } from "./dynamic.js";
import { dag as nativeDag } from "./native.js";
import { buildSummaryMessage, report, summarize } from "./taskflow.js";
@@ -108,10 +110,12 @@ export async function readConnection() {
// is bound to and never by the task_id alone.
//
// One bundle serves both authoring modes: handlers for the Dags a Python file
declares, and the
-// natively declared Dag from `native.ts`, whose graph this side owns outright.
+// natively declared Dag from `native.ts`, whose graph this side owns
outright. It also serves the
+// native Dags `dynamic.ts` generates from a YAML file next to the bundle,
when there is one.
const bundle = new Bundle();
bundle.register(
nativeDag,
+ ...loadDynamicDags(),
new TaskHandler("typescript_example", "build_message", buildMessage),
new TaskHandler("typescript_example", "read_connection", readConnection),
new TaskHandler("typescript_example", "write_and_delete_variable",
writeAndDeleteVariable),
diff --git a/ts-sdk/pnpm-lock.yaml b/ts-sdk/pnpm-lock.yaml
index 99964f27854..072408df53c 100644
--- a/ts-sdk/pnpm-lock.yaml
+++ b/ts-sdk/pnpm-lock.yaml
@@ -58,6 +58,9 @@ importers:
esbuild:
specifier: ^0.28.2
version: 0.28.2
+ js-yaml:
+ specifier: ^5.4.1
+ version: 5.4.1
typescript:
specifier: ^6.0.2
version: 6.0.3
