This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 915e61cc40d Do not show @timestamp in logs in UI (#70790)
915e61cc40d is described below
commit 915e61cc40ddd802956d4d823ea2dafde6f95cf2
Author: Kirill Popov <[email protected]>
AuthorDate: Sat Aug 1 06:36:30 2026 +0300
Do not show @timestamp in logs in UI (#70790)
* Do not show @timestamp in logs in UI
* Keep naming OpenSearch in the OpenSearch handler docstring
---------
Co-authored-by: Kirill Popov <[email protected]>
Co-authored-by: Jarek Potiuk <[email protected]>
---
.../src/airflow/providers/elasticsearch/log/es_task_handler.py | 8 ++++----
.../tests/unit/elasticsearch/log/test_es_task_handler.py | 8 +++++++-
.../src/airflow/providers/opensearch/log/os_task_handler.py | 8 ++++----
.../opensearch/tests/unit/opensearch/log/test_os_task_handler.py | 8 +++++++-
4 files changed, 22 insertions(+), 10 deletions(-)
diff --git
a/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py
b/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py
index 2fb610186e9..9b4261072dc 100644
---
a/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py
+++
b/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py
@@ -123,11 +123,11 @@ def _format_error_detail(error_detail: Any) -> str | None:
def _build_log_fields(hit_dict: dict[str, Any]) -> dict[str, Any]:
"""Filter an ES hit to ``TASK_LOG_FIELDS`` and ensure compatibility with
StructuredLogMessage."""
- fields = {k: v for k, v in hit_dict.items() if k.lower() in
TASK_LOG_FIELDS or k == "@timestamp"}
+ fields = {k: v for k, v in hit_dict.items() if k.lower() in
TASK_LOG_FIELDS}
- # Map @timestamp to timestamp
- if "@timestamp" in fields and "timestamp" not in fields:
- fields["timestamp"] = fields.pop("@timestamp")
+ # Map @timestamp to timestamp but not include `@timestamp` in log fields
+ if "@timestamp" in hit_dict and "timestamp" not in fields:
+ fields["timestamp"] = hit_dict["@timestamp"]
# Map levelname to level
if "levelname" in fields and "level" not in fields:
diff --git
a/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py
b/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py
index a2cb4c315d8..589174c0c71 100644
---
a/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py
+++
b/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py
@@ -975,12 +975,18 @@ class TestBuildStructuredLogFields:
assert result["level"] == "ERROR"
assert "levelname" not in result
- def test_at_timestamp_mapped_to_timestamp(self):
+ def test_at_timestamp_mapped_to_timestamp_if_no_timestamp_present(self):
hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z"}
result = _build_log_fields(hit)
assert result["timestamp"] == "2024-01-01T00:00:00Z"
assert "@timestamp" not in result
+ def test_at_timestamp_not_included_if_timestamp_present(self):
+ hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z",
"timestamp": "2024-01-01T00:00:00Z"}
+ result = _build_log_fields(hit)
+ assert result["timestamp"] == "2024-01-01T00:00:00Z"
+ assert "@timestamp" not in result
+
def test_error_detail_is_kept_as_list(self):
error_detail = [
{
diff --git
a/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py
b/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py
index a6fee7ba601..6d9722478a5 100644
---
a/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py
+++
b/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py
@@ -104,11 +104,11 @@ def _format_error_detail(error_detail: Any) -> str | None:
def _build_log_fields(hit_dict: dict[str, Any]) -> dict[str, Any]:
"""Filter an OpenSearch hit to ``TASK_LOG_FIELDS`` and ensure
compatibility with StructuredLogMessage."""
- fields = {k: v for k, v in hit_dict.items() if k.lower() in
TASK_LOG_FIELDS or k == "@timestamp"}
+ fields = {k: v for k, v in hit_dict.items() if k.lower() in
TASK_LOG_FIELDS}
- # Map @timestamp to timestamp
- if "@timestamp" in fields and "timestamp" not in fields:
- fields["timestamp"] = fields.pop("@timestamp")
+ # Map @timestamp to timestamp but not include `@timestamp` in log fields
+ if "@timestamp" in hit_dict and "timestamp" not in fields:
+ fields["timestamp"] = hit_dict["@timestamp"]
# Map levelname to level
if "levelname" in fields and "level" not in fields:
diff --git
a/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py
b/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py
index 7ec7281f039..ca49103b8a5 100644
--- a/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py
+++ b/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py
@@ -879,12 +879,18 @@ class TestBuildStructuredLogFields:
assert result["level"] == "ERROR"
assert "levelname" not in result
- def test_at_timestamp_mapped_to_timestamp(self):
+ def test_at_timestamp_mapped_to_timestamp_if_no_timestamp_present(self):
hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z"}
result = _build_log_fields(hit)
assert result["timestamp"] == "2024-01-01T00:00:00Z"
assert "@timestamp" not in result
+ def test_at_timestamp_not_included_if_timestamp_present(self):
+ hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z",
"timestamp": "2024-01-01T00:00:00Z"}
+ result = _build_log_fields(hit)
+ assert result["timestamp"] == "2024-01-01T00:00:00Z"
+ assert "@timestamp" not in result
+
def test_error_detail_is_kept_as_list(self):
error_detail = [
{
