This is an automated email from the ASF dual-hosted git repository.
hussein-awala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4f459eab707 Report Datadog timer metrics in milliseconds (#74848)
4f459eab707 is described below
commit 4f459eab7073566b783fdc0fff08f2ca291548e4
Author: Timothé Barbe <[email protected]>
AuthorDate: Thu Nov 19 16:04:04 2026 +0100
Report Datadog timer metrics in milliseconds (#74848)
* Report Datadog timer metrics in milliseconds
Stats.timer() metrics are documented in milliseconds, but the Datadog
backend relied on DogStatsd.timed(), which reports seconds unless the
client enables use_ms, and then rounds to whole milliseconds. Datadog
users saw timers such as scheduler.scheduler_loop_duration 1000x smaller
than on StatsD or OpenTelemetry, and sub-millisecond timers reported as 0.
* Add newsfragment for Datadog timer units
* Fail fast on unsupported Datadog timer arguments
Arguments that DogStatsd.timing() does not accept used to surface as a
TypeError when the timer stopped, from the with block's __exit__, which
could hide an exception raised by the timed code. Stopping a timer that
was never started also raised AttributeError. The Datadog timer change
breaks existing dashboards and monitors, so it belongs in the upgrade notes.
* Share the timer that emits its duration between metric backends
The Datadog and OpenTelemetry timers duplicated the logic deciding when a
measured duration is sent, so a fix to one could easily miss the other.
---
airflow-core/newsfragments/74848.significant.rst | 18 +++++++++++
.../observability/metrics/datadog_logger.py | 12 +++++--
.../observability/metrics/otel_logger.py | 37 ++++++----------------
.../observability/metrics/protocols.py | 19 +++++++++--
.../tests/observability/metrics/test_stats.py | 32 +++++++++++++++----
5 files changed, 78 insertions(+), 40 deletions(-)
diff --git a/airflow-core/newsfragments/74848.significant.rst
b/airflow-core/newsfragments/74848.significant.rst
new file mode 100644
index 00000000000..07ef185c2b4
--- /dev/null
+++ b/airflow-core/newsfragments/74848.significant.rst
@@ -0,0 +1,18 @@
+Datadog timer metrics are now reported in milliseconds
+
+With the Datadog (DogStatsD) metrics backend, ``Stats.timer()`` metrics such as
+``scheduler.scheduler_loop_duration`` were sent in seconds, while they are
documented in milliseconds
+and reported in milliseconds by the StatsD and OpenTelemetry backends. They
are now sent in
+milliseconds, so their values become 1000x larger after upgrading. Update
Datadog dashboards, monitors
+and SLOs that use these timers accordingly.
+
+* Types of change
+
+ * [ ] Dag changes
+ * [ ] Config changes
+ * [ ] API changes
+ * [ ] CLI changes
+ * [x] Behaviour changes
+ * [ ] Plugin changes
+ * [ ] Dependency changes
+ * [ ] Code interface changes
diff --git
a/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
b/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
index efe621b4104..3d35bcd34e1 100644
---
a/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
+++
b/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
@@ -22,7 +22,7 @@ import logging
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
-from .protocols import Timer
+from .protocols import DurationEmittingTimer, Timer
from .validators import (
PatternAllowListValidator,
PatternBlockListValidator,
@@ -134,12 +134,18 @@ class SafeDogStatsdLogger:
self,
stat: str | None = None,
tags: dict[str, str] | None = None,
- **kwargs,
+ sample_rate: float | None = None,
) -> Timer:
"""Timer metric that can be cancelled."""
tags_list = self._build_tags_list(tags)
if stat and self.metrics_validator.test(stat):
- return Timer(self.dogstatsd.timed(stat, tags=tags_list, **kwargs))
+ # DogStatsd.timed() reports seconds unless the client sets use_ms,
and rounds to whole
+ # milliseconds when it does, so the duration measured by Timer is
sent instead.
+ return DurationEmittingTimer(
+ lambda duration: self.dogstatsd.timing(
+ metric=stat, value=duration, tags=tags_list,
sample_rate=sample_rate
+ )
+ )
return Timer()
diff --git
a/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py
b/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py
index bc8b819f144..e3c32562153 100644
---
a/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py
+++
b/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py
@@ -37,7 +37,7 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource
from ..common import get_otel_data_exporter
from ..exceptions import InvalidStatsNameException
from ..otel_env_config import load_metrics_env_config
-from .protocols import Timer
+from .protocols import DurationEmittingTimer, Timer
from .validators import (
OTEL_NAME_MAX_LENGTH,
ListValidator,
@@ -154,31 +154,6 @@ def _skip_due_to_rate(rate: float) -> bool:
return rate < 1 and random.random() > rate
-class _OtelTimer(Timer):
- """
- An implementation of stats.Timer() which records the result in the OTel
Metrics Map.
-
- OpenTelemetry does not have a native timer; values are stored as a
Histogram so that
- all observations (count, sum, bucket distribution) are preserved across
multiple recordings.
-
- :param name: The name of the timer.
- :param tags: Tags to append to the timer.
- """
-
- def __init__(self, otel_logger: SafeOtelLogger, name: str | None, tags:
Attributes):
- super().__init__()
- self.otel_logger = otel_logger
- self.name = name
- self.tags = tags
-
- def stop(self, send: bool = True) -> None:
- super().stop(send)
- if self.name and send and self.duration is not None:
- self.otel_logger.metrics_map.record_histogram_value(
- full_name(prefix=self.otel_logger.prefix, name=self.name),
self.duration, self.tags
- )
-
-
class SafeOtelLogger:
"""Otel Logger."""
@@ -317,8 +292,14 @@ class SafeOtelLogger:
**kwargs,
) -> Timer:
"""Timer context manager returns the duration and can be cancelled."""
- safe_stat = stat if self._is_recordable(stat) else None
- return _OtelTimer(self, safe_stat, tags)
+ if not (stat and self._is_recordable(stat)):
+ return Timer()
+ # OpenTelemetry has no native timer; a Histogram preserves count, sum
and bucket distribution.
+ return DurationEmittingTimer(
+ lambda duration: self.metrics_map.record_histogram_value(
+ full_name(prefix=self.prefix, name=stat), duration, tags
+ )
+ )
class InternalGauge:
diff --git
a/shared/observability/src/airflow_shared/observability/metrics/protocols.py
b/shared/observability/src/airflow_shared/observability/metrics/protocols.py
index fad6e6a570b..9b71b9f2fff 100644
--- a/shared/observability/src/airflow_shared/observability/metrics/protocols.py
+++ b/shared/observability/src/airflow_shared/observability/metrics/protocols.py
@@ -22,6 +22,8 @@ import time
from typing import TYPE_CHECKING, Protocol
if TYPE_CHECKING:
+ from collections.abc import Callable
+
from .. import Self
DeltaType = int | float | datetime.timedelta
@@ -105,8 +107,8 @@ class Timer(TimerProtocol):
# pystatsd and dogstatsd both have a timer class, but present different API
# so we can't use this as a mixin on those, instead this class contains
the "real" timer
- _start_time: float | None
- duration: float | None
+ _start_time: float | None = None
+ duration: float | None = None
def __init__(self, real_timer: BackendTimerProtocol | None = None) -> None:
self.real_timer = real_timer
@@ -130,3 +132,16 @@ class Timer(TimerProtocol):
self.duration = 1000.0 * (time.perf_counter() - self._start_time)
# Convert to milliseconds.
if send and self.real_timer:
self.real_timer.stop()
+
+
+class DurationEmittingTimer(Timer):
+ """Timer that passes its measured duration, in milliseconds, to ``emit``
when stopped."""
+
+ def __init__(self, emit: Callable[[float], object]) -> None:
+ super().__init__()
+ self._emit = emit
+
+ def stop(self, send: bool = True) -> None:
+ super().stop(send)
+ if send and self.duration is not None:
+ self._emit(self.duration)
diff --git a/shared/observability/tests/observability/metrics/test_stats.py
b/shared/observability/tests/observability/metrics/test_stats.py
index c2a4a931021..d9fe1ee4504 100644
--- a/shared/observability/tests/observability/metrics/test_stats.py
+++ b/shared/observability/tests/observability/metrics/test_stats.py
@@ -253,19 +253,37 @@ class TestDogStats:
metric="empty_key", sample_rate=1, tags=[], value=1
)
- @mock.patch.object(time, "perf_counter", side_effect=[0.0, 100.0])
- def test_timer(self, time_mock):
+ @mock.patch.object(time, "perf_counter", side_effect=[0.0, 0.0004])
+ def test_timer_sends_duration_in_milliseconds(self, time_mock):
with self.dogstatsd.timer("empty_timer") as timer:
pass
- self.dogstatsd_client.timed.assert_called_once_with("empty_timer",
tags=[])
- expected_duration = 1000.0 * 100.0
- assert expected_duration == timer.duration
- assert time_mock.call_count == 2
+ assert timer.duration == pytest.approx(0.4)
+ self.dogstatsd_client.timing.assert_called_once_with(
+ metric="empty_timer", value=pytest.approx(0.4), tags=[],
sample_rate=None
+ )
+ self.dogstatsd_client.timed.assert_not_called()
+
+ @mock.patch.object(time, "perf_counter", side_effect=[0.0, 1.0])
+ def test_timer_stop_without_send(self, time_mock):
+ timer = self.dogstatsd.timer("empty_timer").start()
+ timer.stop(send=False)
+ assert timer.duration == 1000.0
+ self.dogstatsd_client.timing.assert_not_called()
+
+ def test_timer_stop_without_start(self):
+ timer = self.dogstatsd.timer("empty_timer")
+ timer.stop()
+ assert timer.duration is None
+ self.dogstatsd_client.timing.assert_not_called()
+
+ def test_timer_rejects_unsupported_arguments_on_creation(self):
+ with pytest.raises(TypeError):
+ self.dogstatsd.timer("empty_timer", use_ms=True)
def test_empty_timer(self):
with self.dogstatsd.timer():
pass
- self.dogstatsd_client.timed.assert_not_called()
+ self.dogstatsd_client.timing.assert_not_called()
def test_timing(self):
import datetime