This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9b37c269b4a Use only file:// URIs in the OpenLineage exclude_datasets
authoring system test (#75961)
9b37c269b4a is described below
commit 9b37c269b4a170a0f0baa3b630bb22cbd25c308a
Author: Kacper Muda <[email protected]>
AuthorDate: Thu Oct 29 15:21:58 2026 +0100
Use only file:// URIs in the OpenLineage exclude_datasets authoring system
test (#75961)
The t_identity task of openlineage_policy_exclude_datasets_authoring_dag
registered s3:// assets. Their OpenLineage translation lives in the amazon
provider, so when it is not installed the s3:// assets are silently dropped
and the task emits fewer inputs and outputs than expected.
Use file:// URIs on another host instead, so the test passes with and
without
the amazon provider installed.
Co-authored-by: Claude <[email protected]>
---
..._openlineage_policy_exclude_datasets_authoring_dag.py | 16 ++++++++--------
...penlineage_policy_exclude_datasets_authoring_dag.json | 8 ++++----
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git
a/providers/openlineage/tests/system/openlineage/example_openlineage_policy_exclude_datasets_authoring_dag.py
b/providers/openlineage/tests/system/openlineage/example_openlineage_policy_exclude_datasets_authoring_dag.py
index 0c024237eee..5ccb495062a 100644
---
a/providers/openlineage/tests/system/openlineage/example_openlineage_policy_exclude_datasets_authoring_dag.py
+++
b/providers/openlineage/tests/system/openlineage/example_openlineage_policy_exclude_datasets_authoring_dag.py
@@ -120,14 +120,14 @@ def _identity():
_register_uris(
QuietHook(),
inputs=[
- "s3://ol-exclude-bucket/staging/part_1.csv",
- "s3://ol-exclude-bucket/final/part_1.csv",
+ "file://host2/staging/part_1.csv",
+ "file://host2/final/part_1.csv",
"file://host1/slash_in.txt",
"file://host1/keep_in.txt",
],
outputs=[
- "s3://ol-exclude-bucket/staging/part_1_out.csv",
- "s3://ol-exclude-bucket/final/part_1_out.csv",
+ "file://host2/staging/part_1_out.csv",
+ "file://host2/final/part_1_out.csv",
"file://host1/slash_out.txt",
"file://host1/keep_out.txt",
],
@@ -224,12 +224,12 @@ with DAG(
".*/authored_full_.*",
],
)
- # The authored patterns match <namespace>/<name>: COMPLETE drops the s3
staging files and
- # file://host1/slash_* (no doubled slash before a name starting with /),
and keeps the s3 final files and
- # keep.
+ # The authored patterns match <namespace>/<name>: COMPLETE drops the
file://host2 staging files and
+ # file://host1/slash_* (no doubled slash before a name starting with /),
and keeps the file://host2 final files
+ # and keep.
t_identity = extend_global_openlineage_emission_policy(
PythonOperator(task_id="t_identity", python_callable=_identity),
- exclude_datasets=["s3://ol-exclude-bucket/staging/part_.*",
"file://host1/slash_.*"],
+ exclude_datasets=["file://host2/staging/part_.*",
"file://host1/slash_.*"],
)
# Conf: a locked Dag + task rule (locked_excluded) and an unlocked Dag +
task + AuthHook rule
# (specific_excluded), which wins for AuthHook. The authored list is
ignored with a warning: COMPLETE
diff --git
a/providers/openlineage/tests/system/openlineage/expected_events/openlineage_policy_exclude_datasets_authoring_dag.json
b/providers/openlineage/tests/system/openlineage/expected_events/openlineage_policy_exclude_datasets_authoring_dag.json
index 87721cf332b..770453c6ead 100644
---
a/providers/openlineage/tests/system/openlineage/expected_events/openlineage_policy_exclude_datasets_authoring_dag.json
+++
b/providers/openlineage/tests/system/openlineage/expected_events/openlineage_policy_exclude_datasets_authoring_dag.json
@@ -333,8 +333,8 @@
"eventType": "COMPLETE",
"inputs": [
{
- "namespace": "s3://ol-exclude-bucket",
- "name": "final/part_1.csv"
+ "namespace": "file://host2",
+ "name": "/final/part_1.csv"
},
{
"namespace": "file://host1",
@@ -343,8 +343,8 @@
],
"outputs": [
{
- "namespace": "s3://ol-exclude-bucket",
- "name": "final/part_1_out.csv"
+ "namespace": "file://host2",
+ "name": "/final/part_1_out.csv"
},
{
"namespace": "file://host1",
