This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new f2fd10811 refactor: split metrics related crates to new crates (#6954)
f2fd10811 is described below
commit f2fd10811677588948a696dca1dacd934a268677
Author: Xuanwo <[email protected]>
AuthorDate: Mon Dec 8 21:34:37 2025 +0800
refactor: split metrics related crates to new crates (#6954)
Signed-off-by: Xuanwo <[email protected]>
---
core/Cargo.lock | 64 ++++++++++++++-
core/Cargo.toml | 19 +++--
core/core/Cargo.toml | 18 -----
core/core/src/layers/mod.rs | 33 --------
core/core/src/layers/observe/mod.rs | 93 ----------------------
core/core/src/raw/http_util/body.rs | 4 +-
core/core/src/raw/http_util/client.rs | 2 +-
core/layers/fastmetrics/Cargo.toml | 36 +++++++++
.../fastmetrics/src/lib.rs} | 16 ++--
core/layers/metrics/Cargo.toml | 36 +++++++++
.../metrics.rs => layers/metrics/src/lib.rs} | 8 +-
core/layers/observe-metrics-common/Cargo.toml | 36 +++++++++
.../observe-metrics-common/src/lib.rs} | 62 ++++++++++++++-
core/layers/otelmetrics/Cargo.toml | 36 +++++++++
.../otelmetrics/src/lib.rs} | 12 +--
core/layers/prometheus-client/Cargo.toml | 36 +++++++++
.../prometheus-client/src/lib.rs} | 12 +--
core/layers/prometheus/Cargo.toml | 36 +++++++++
.../prometheus.rs => layers/prometheus/src/lib.rs} | 20 ++---
core/src/lib.rs | 10 +++
20 files changed, 393 insertions(+), 196 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 501ccb4bb..6e507649a 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5478,6 +5478,12 @@ dependencies = [
"opendal-layer-async-backtrace",
"opendal-layer-await-tree",
"opendal-layer-capability-check",
+ "opendal-layer-fastmetrics",
+ "opendal-layer-metrics",
+ "opendal-layer-observe-metrics-common",
+ "opendal-layer-otelmetrics",
+ "opendal-layer-prometheus",
+ "opendal-layer-prometheus-client",
"opendal-service-aliyun-drive",
"opendal-service-azblob",
"opendal-service-azdls",
@@ -5542,7 +5548,6 @@ dependencies = [
"divan",
"dotenvy",
"etcd-client",
- "fastmetrics",
"fastpool",
"fastrace",
"fastrace-jaeger",
@@ -5563,7 +5568,6 @@ dependencies = [
"log",
"md-5",
"mea",
- "metrics",
"mime_guess",
"mini-moka",
"moka",
@@ -5580,8 +5584,6 @@ dependencies = [
"persy",
"pretty_assertions",
"probe",
- "prometheus 0.14.0",
- "prometheus-client",
"prost 0.13.5",
"quick-xml",
"rand 0.8.5",
@@ -5677,6 +5679,60 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-layer-fastmetrics"
+version = "0.55.0"
+dependencies = [
+ "fastmetrics",
+ "opendal-core",
+ "opendal-layer-observe-metrics-common",
+]
+
+[[package]]
+name = "opendal-layer-metrics"
+version = "0.55.0"
+dependencies = [
+ "metrics",
+ "opendal-core",
+ "opendal-layer-observe-metrics-common",
+]
+
+[[package]]
+name = "opendal-layer-observe-metrics-common"
+version = "0.55.0"
+dependencies = [
+ "futures",
+ "http 1.4.0",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-otelmetrics"
+version = "0.55.0"
+dependencies = [
+ "opendal-core",
+ "opendal-layer-observe-metrics-common",
+ "opentelemetry",
+]
+
+[[package]]
+name = "opendal-layer-prometheus"
+version = "0.55.0"
+dependencies = [
+ "opendal-core",
+ "opendal-layer-observe-metrics-common",
+ "prometheus 0.14.0",
+]
+
+[[package]]
+name = "opendal-layer-prometheus-client"
+version = "0.55.0"
+dependencies = [
+ "opendal-core",
+ "opendal-layer-observe-metrics-common",
+ "prometheus-client",
+]
+
[[package]]
name = "opendal-service-aliyun-drive"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 390abab99..da605cd87 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -26,8 +26,6 @@ members = [
"benches/vs_*",
"services/*",
"layers/*",
- "layers/await-tree",
- "layers/capability-check",
]
[workspace.package]
@@ -84,7 +82,6 @@ default = [
"reqwest-rustls-tls",
"executors-tokio",
"services-memory",
- "layers-capability-check",
]
executors-tokio = ["opendal-core/executors-tokio"]
internal-path-cache = ["opendal-core/internal-path-cache"]
@@ -94,14 +91,14 @@ layers-await-tree = ["dep:opendal-layer-await-tree"]
layers-capability-check = ["dep:opendal-layer-capability-check"]
layers-chaos = ["opendal-core/layers-chaos"]
layers-dtrace = ["opendal-core/layers-dtrace"]
-layers-fastmetrics = ["opendal-core/layers-fastmetrics"]
+layers-fastmetrics = ["dep:opendal-layer-fastmetrics"]
layers-fastrace = ["opendal-core/layers-fastrace"]
-layers-metrics = ["opendal-core/layers-metrics"]
+layers-metrics = ["dep:opendal-layer-metrics"]
layers-mime-guess = ["opendal-core/layers-mime-guess"]
-layers-otel-metrics = ["opendal-core/layers-otel-metrics"]
+layers-otel-metrics = ["dep:opendal-layer-otelmetrics"]
layers-otel-trace = ["opendal-core/layers-otel-trace"]
-layers-prometheus = ["opendal-core/layers-prometheus"]
-layers-prometheus-client = ["opendal-core/layers-prometheus-client"]
+layers-prometheus = ["dep:opendal-layer-prometheus"]
+layers-prometheus-client = ["dep:opendal-layer-prometheus-client"]
layers-throttle = ["opendal-core/layers-throttle"]
layers-tracing = ["opendal-core/layers-tracing"]
reqwest-rustls-tls = ["opendal-core/reqwest-rustls-tls"]
@@ -193,6 +190,12 @@ opendal-core = { path = "core", version = "0.55.0",
default-features = false }
opendal-layer-async-backtrace = { path = "layers/async-backtrace", version =
"0.55.0", optional = true, default-features = false }
opendal-layer-await-tree = { path = "layers/await-tree", version = "0.55.0",
optional = true, default-features = false }
opendal-layer-capability-check = { path = "layers/capability-check", version =
"0.55.0", optional = true, default-features = false }
+opendal-layer-observe-metrics-common = { path =
"layers/observe-metrics-common", version = "0.55.0", optional = true,
default-features = false }
+opendal-layer-metrics = { path = "layers/metrics", version = "0.55.0",
optional = true, default-features = false }
+opendal-layer-otelmetrics = { path = "layers/otelmetrics", version = "0.55.0",
optional = true, default-features = false }
+opendal-layer-prometheus = { path = "layers/prometheus", version = "0.55.0",
optional = true, default-features = false }
+opendal-layer-prometheus-client = { path = "layers/prometheus-client", version
= "0.55.0", optional = true, default-features = false }
+opendal-layer-fastmetrics = { path = "layers/fastmetrics", version = "0.55.0",
optional = true, default-features = false }
opendal-service-aliyun-drive = { path = "services/aliyun-drive", version =
"0.55.0", optional = true, default-features = false }
opendal-service-azblob = { path = "services/azblob", version = "0.55.0",
optional = true, default-features = false }
opendal-service-azdls = { path = "services/azdls", version = "0.55.0",
optional = true, default-features = false }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index 3846596f1..136f2f9c5 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -72,22 +72,12 @@ executors-tokio = ["tokio/rt"]
# Enable layers chaos support
layers-chaos = ["dep:rand"]
-# Enable layers metrics support
-layers-metrics = ["dep:metrics"]
# Enable layers mime_guess support
layers-mime-guess = ["dep:mime_guess"]
-# Enable layers prometheus support, with tikv/prometheus-rs crate
-layers-prometheus = ["dep:prometheus"]
-# Enable layers prometheus support, with prometheus-client crate
-layers-prometheus-client = ["dep:prometheus-client"]
-# Enable layers fastmetrics support.
-layers-fastmetrics = ["dep:fastmetrics"]
# Enable layers fastrace support.
layers-fastrace = ["dep:fastrace"]
# Enable layers tracing support.
layers-tracing = ["dep:tracing"]
-# Enable layers otelmetrics support.
-layers-otel-metrics = ["dep:opentelemetry", "opentelemetry/metrics"]
# Enable layers oteltrace support.
layers-otel-trace = ["dep:opentelemetry", "opentelemetry/trace"]
# Enable layers throttle support.
@@ -324,20 +314,12 @@ web-sys = { version = "0.3.77", optional = true, features
= [
# Layers
# for layers-throttle
governor = { version = "0.10.1", optional = true, features = ["std"] }
-# for layers-metrics
-metrics = { version = "0.24", optional = true }
# for layers-mime-guess
mime_guess = { version = "2.0.5", optional = true }
# for layers-fastrace
fastrace = { version = "0.7.14", optional = true }
# for layers-opentelemetry
opentelemetry = { version = "0.31.0", optional = true }
-# for layers-prometheus
-prometheus = { version = "0.14", features = ["process"], optional = true }
-# for layers-prometheus-client
-prometheus-client = { version = "0.24", optional = true }
-# for fastmetrics
-fastmetrics = { version = "0.4.1", optional = true }
# for layers-tracing
tracing = { version = "0.1", optional = true }
# for layers-dtrace
diff --git a/core/core/src/layers/mod.rs b/core/core/src/layers/mod.rs
index 85cccd55b..4b1012ff6 100644
--- a/core/core/src/layers/mod.rs
+++ b/core/core/src/layers/mod.rs
@@ -47,37 +47,11 @@ mod chaos;
#[cfg(feature = "layers-chaos")]
pub use chaos::ChaosLayer;
-#[cfg(feature = "layers-metrics")]
-mod metrics;
-#[cfg(feature = "layers-metrics")]
-pub use self::metrics::MetricsLayer;
-
#[cfg(feature = "layers-mime-guess")]
mod mime_guess;
#[cfg(feature = "layers-mime-guess")]
pub use self::mime_guess::MimeGuessLayer;
-#[cfg(feature = "layers-prometheus")]
-mod prometheus;
-#[cfg(feature = "layers-prometheus")]
-pub use self::prometheus::PrometheusLayer;
-#[cfg(feature = "layers-prometheus")]
-pub use self::prometheus::PrometheusLayerBuilder;
-
-#[cfg(feature = "layers-prometheus-client")]
-mod prometheus_client;
-#[cfg(feature = "layers-prometheus-client")]
-pub use self::prometheus_client::PrometheusClientLayer;
-#[cfg(feature = "layers-prometheus-client")]
-pub use self::prometheus_client::PrometheusClientLayerBuilder;
-
-#[cfg(feature = "layers-fastmetrics")]
-mod fastmetrics;
-#[cfg(feature = "layers-fastmetrics")]
-pub use self::fastmetrics::FastmetricsLayer;
-#[cfg(feature = "layers-fastmetrics")]
-pub use self::fastmetrics::FastmetricsLayerBuilder;
-
mod retry;
pub use self::retry::RetryInterceptor;
pub use self::retry::RetryLayer;
@@ -96,11 +70,6 @@ mod fastrace;
#[cfg(feature = "layers-fastrace")]
pub use self::fastrace::FastraceLayer;
-#[cfg(feature = "layers-otel-metrics")]
-mod otelmetrics;
-#[cfg(feature = "layers-otel-metrics")]
-pub use self::otelmetrics::OtelMetricsLayer;
-
#[cfg(feature = "layers-otel-trace")]
mod oteltrace;
#[cfg(feature = "layers-otel-trace")]
@@ -116,8 +85,6 @@ mod dtrace;
#[cfg(all(target_os = "linux", feature = "layers-dtrace"))]
pub use self::dtrace::DtraceLayer;
-pub mod observe;
-
mod correctness_check;
pub(crate) use correctness_check::CorrectnessCheckLayer;
diff --git a/core/core/src/layers/observe/mod.rs
b/core/core/src/layers/observe/mod.rs
deleted file mode 100644
index 7837e1e33..000000000
--- a/core/core/src/layers/observe/mod.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! OpenDAL Observability
-//!
-//! This module offers essential components to facilitate the implementation
of observability in OpenDAL.
-//!
-//! # OpenDAL Metrics Reference
-//!
-//! This document describes all metrics exposed by OpenDAL.
-//!
-//! ## Operation Metrics
-//!
-//! These metrics track operations at the storage abstraction level.
-//!
-//! | Metric Name | Type | Description
| Labels
|
-//!
|----------------------------------|-----------|--------------------------------------------------------------------------------------------|-------------------------------------------------|
-//! | operation_bytes | Histogram | Current operation size in
bytes, represents the size of data being processed | scheme,
namespace, root, operation, path |
-//! | operation_bytes_rate | Histogram | Histogram of data
processing rates in bytes per second within individual operations |
scheme, namespace, root, operation, path |
-//! | operation_entries | Histogram | Current operation size in
entries, represents the entries being processed | scheme,
namespace, root, operation, path |
-//! | operation_entries_rate | Histogram | Histogram of entries
processing rates in entries per second within individual operations | scheme,
namespace, root, operation, path |
-//! | operation_duration_seconds | Histogram | Duration of operations in
seconds, measured from start to completion | scheme,
namespace, root, operation, path |
-//! | operation_errors_total | Counter | Total number of failed
operations | scheme,
namespace, root, operation, path, error |
-//! | operation_executing | Gauge | Number of operations
currently being executed | scheme,
namespace, root, operation |
-//! | operation_ttfb_seconds | Histogram | Time to first byte in
seconds for operations | scheme,
namespace, root, operation, path |
-//!
-//! ## HTTP Metrics
-//!
-//! These metrics track the underlying HTTP requests made by OpenDAL services
that use HTTP.
-//!
-//! | Metric Name | Type | Description
| Labels
|
-//!
|----------------------------------|-----------|--------------------------------------------------------------------------------------------|-------------------------------------------------|
-//! | http_connection_errors_total | Counter | Total number of HTTP
requests that failed before receiving a response | scheme,
namespace, root, operation, error |
-//! | http_status_errors_total | Counter | Total number of HTTP
requests that received error status codes (non-2xx responses) | scheme,
namespace, root, operation, status |
-//! | http_executing | Gauge | Number of HTTP requests
currently in flight from this client | scheme,
namespace, root |
-//! | http_request_bytes | Histogram | Histogram of HTTP request
body sizes in bytes | scheme,
namespace, root, operation |
-//! | http_request_bytes_rate | Histogram | Histogram of HTTP request
bytes per second rates | scheme,
namespace, root, operation |
-//! | http_request_duration_seconds | Histogram | Histogram of time spent
sending HTTP requests, from first byte sent to first byte received | scheme,
namespace, root, operation |
-//! | http_response_bytes | Histogram | Histogram of HTTP
response body sizes in bytes |
scheme, namespace, root, operation |
-//! | http_response_bytes_rate | Histogram | Histogram of HTTP
response bytes per second rates |
scheme, namespace, root, operation |
-//! | http_response_duration_seconds | Histogram | Histogram of time spent
receiving HTTP responses, from first byte to last byte received | scheme,
namespace, root, operation |
-//!
-//! ## Label Descriptions
-//!
-//! | Label | Description
| Example Values |
-//!
|-----------|---------------------------------------------------------------|----------------------------------------|
-//! | scheme | The storage service scheme
| s3, gcs, azblob, fs, memory |
-//! | namespace | The storage service namespace (bucket, container, etc.)
| my-bucket, my-container |
-//! | root | The root path within the namespace
| /data, /backup |
-//! | operation | The operation being performed
| read, write, stat, list, delete |
-//! | path | The path of the object being operated on
| /path/to/file.txt |
-//! | error | The error type or message for error metrics
| not_found, permission_denied |
-//! | status | The HTTP status code for HTTP error metrics
| 404, 403, 500 |
-//!
-//! ## Metric Types
-//!
-//! * **Histogram**: Distribution of values with configurable buckets,
includes count, sum and quantiles
-//! * **Counter**: Cumulative metric that only increases over time (resets on
restart)
-//! * **Gauge**: Point-in-time metric that can increase and decrease
-
-mod metrics;
-
-pub use metrics::DEFAULT_BYTES_BUCKETS;
-pub use metrics::DEFAULT_BYTES_RATE_BUCKETS;
-pub use metrics::DEFAULT_DURATION_SECONDS_BUCKETS;
-pub use metrics::DEFAULT_ENTRIES_BUCKETS;
-pub use metrics::DEFAULT_ENTRIES_RATE_BUCKETS;
-pub use metrics::DEFAULT_TTFB_BUCKETS;
-pub use metrics::LABEL_ERROR;
-pub use metrics::LABEL_NAMESPACE;
-pub use metrics::LABEL_OPERATION;
-pub use metrics::LABEL_ROOT;
-pub use metrics::LABEL_SCHEME;
-pub use metrics::LABEL_STATUS_CODE;
-pub use metrics::MetricLabels;
-pub use metrics::MetricValue;
-pub use metrics::MetricsAccessor;
-pub use metrics::MetricsIntercept;
-pub use metrics::MetricsLayer;
diff --git a/core/core/src/raw/http_util/body.rs
b/core/core/src/raw/http_util/body.rs
index 34b1a22e5..f694633db 100644
--- a/core/core/src/raw/http_util/body.rs
+++ b/core/core/src/raw/http_util/body.rs
@@ -76,7 +76,7 @@ impl HttpBody {
/// Map the inner stream.
#[cfg(not(target_arch = "wasm32"))]
- pub(crate) fn map_inner(
+ pub fn map_inner(
mut self,
f: impl FnOnce(
Box<dyn Stream<Item = Result<Buffer>> + Send + Sync + Unpin +
'static>,
@@ -89,7 +89,7 @@ impl HttpBody {
/// Map the inner stream.
#[cfg(target_arch = "wasm32")]
- pub(crate) fn map_inner(
+ pub fn map_inner(
mut self,
f: impl FnOnce(
Box<dyn Stream<Item = Result<Buffer>> + Unpin + 'static>,
diff --git a/core/core/src/raw/http_util/client.rs
b/core/core/src/raw/http_util/client.rs
index f00c47443..abab39396 100644
--- a/core/core/src/raw/http_util/client.rs
+++ b/core/core/src/raw/http_util/client.rs
@@ -91,7 +91,7 @@ impl HttpClient {
}
/// Get the inner http client.
- pub(crate) fn into_inner(self) -> HttpFetcher {
+ pub fn into_inner(self) -> HttpFetcher {
self.fetcher
}
diff --git a/core/layers/fastmetrics/Cargo.toml
b/core/layers/fastmetrics/Cargo.toml
new file mode 100644
index 000000000..c60193292
--- /dev/null
+++ b/core/layers/fastmetrics/Cargo.toml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL metrics layer using fastmetrics"
+name = "opendal-layer-fastmetrics"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+fastmetrics = { version = "0.4.1" }
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+opendal-layer-observe-metrics-common = { path = "../observe-metrics-common",
version = "0.55.0", default-features = false }
diff --git a/core/core/src/layers/fastmetrics.rs
b/core/layers/fastmetrics/src/lib.rs
similarity index 98%
rename from core/core/src/layers/fastmetrics.rs
rename to core/layers/fastmetrics/src/lib.rs
index f3a9f00a7..d6b03f774 100644
--- a/core/core/src/layers/fastmetrics.rs
+++ b/core/layers/fastmetrics/src/lib.rs
@@ -28,10 +28,9 @@ use fastmetrics::registry::Register;
use fastmetrics::registry::Registry;
use fastmetrics::registry::RegistryError;
use fastmetrics::registry::with_global_registry_mut;
-
-use crate::layers::observe;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
+use opendal_layer_observe_metrics_common as observe;
/// Add [fastmetrics](https://docs.rs/fastmetrics/) for every operation.
///
@@ -42,7 +41,7 @@ use crate::*;
/// ```no_run
/// # use fastmetrics::format::text;
/// # use log::info;
-/// # use opendal_core::layers::FastmetricsLayer;
+/// # use opendal_layer_fastmetrics::FastmetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -87,7 +86,7 @@ use crate::*;
/// # use fastmetrics::format::text;
/// # use fastmetrics::registry::with_global_registry;
/// # use log::info;
-/// # use opendal_core::layers::FastmetricsLayer;
+/// # use opendal_layer_fastmetrics::FastmetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -230,7 +229,7 @@ impl FastmetricsLayerBuilder {
/// # Example
///
/// ```no_run
- /// # use opendal_core::layers::FastmetricsLayer;
+ /// # use opendal_layer_fastmetrics::FastmetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -325,7 +324,7 @@ impl FastmetricsLayerBuilder {
/// # Example
///
/// ```no_run
- /// # use opendal_core::layers::FastmetricsLayer;
+ /// # use opendal_layer_fastmetrics::FastmetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -495,6 +494,7 @@ impl observe::MetricsIntercept for FastmetricsInterceptor {
self.http_status_errors_total
.with_or_new(&labels, |counter| counter.inc());
}
+ _ => {}
};
}
}
diff --git a/core/layers/metrics/Cargo.toml b/core/layers/metrics/Cargo.toml
new file mode 100644
index 000000000..8461ebff6
--- /dev/null
+++ b/core/layers/metrics/Cargo.toml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL metrics layer built on metrics crate"
+name = "opendal-layer-metrics"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+metrics = "0.24"
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+opendal-layer-observe-metrics-common = { path = "../observe-metrics-common",
version = "0.55.0", default-features = false }
diff --git a/core/core/src/layers/metrics.rs b/core/layers/metrics/src/lib.rs
similarity index 97%
rename from core/core/src/layers/metrics.rs
rename to core/layers/metrics/src/lib.rs
index 6de9bc347..2d098d5f5 100644
--- a/core/core/src/layers/metrics.rs
+++ b/core/layers/metrics/src/lib.rs
@@ -19,9 +19,8 @@ use metrics::Label;
use metrics::counter;
use metrics::gauge;
use metrics::histogram;
-
-use crate::layers::observe;
-use crate::raw::*;
+use opendal_core::raw::*;
+use opendal_layer_observe_metrics_common as observe;
/// Add [metrics](https://docs.rs/metrics/) for every operation.
///
@@ -38,7 +37,7 @@ use crate::raw::*;
/// # Examples
///
/// ```no_run
-/// # use opendal_core::layers::MetricsLayer;
+/// # use opendal_layer_metrics::MetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -148,6 +147,7 @@ impl observe::MetricsIntercept for MetricsInterceptor {
observe::MetricValue::HttpStatusErrorsTotal => {
counter!(value.name(), labels).increment(1)
}
+ _ => {}
}
}
}
diff --git a/core/layers/observe-metrics-common/Cargo.toml
b/core/layers/observe-metrics-common/Cargo.toml
new file mode 100644
index 000000000..5206cd453
--- /dev/null
+++ b/core/layers/observe-metrics-common/Cargo.toml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL observe metrics common components"
+name = "opendal-layer-observe-metrics-common"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+futures = { version = "0.3", default-features = false, features = ["std",
"async-await"] }
+http = { workspace = true }
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
diff --git a/core/core/src/layers/observe/metrics.rs
b/core/layers/observe-metrics-common/src/lib.rs
similarity index 85%
rename from core/core/src/layers/observe/metrics.rs
rename to core/layers/observe-metrics-common/src/lib.rs
index 82f390453..d03480204 100644
--- a/core/core/src/layers/observe/metrics.rs
+++ b/core/layers/observe-metrics-common/src/lib.rs
@@ -15,6 +15,63 @@
// specific language governing permissions and limitations
// under the License.
+//! OpenDAL Observability
+//!
+//! This module offers essential components to facilitate the implementation
of observability in OpenDAL.
+//!
+//! # OpenDAL Metrics Reference
+//!
+//! This document describes all metrics exposed by OpenDAL.
+//!
+//! ## Operation Metrics
+//!
+//! These metrics track operations at the storage abstraction level.
+//!
+//! | Metric Name | Type | Description
| Labels
|
+//!
|----------------------------------|-----------|--------------------------------------------------------------------------------------------|-------------------------------------------------|
+//! | operation_bytes | Histogram | Current operation size in
bytes, represents the size of data being processed | scheme,
namespace, root, operation, path |
+//! | operation_bytes_rate | Histogram | Histogram of data
processing rates in bytes per second within individual operations |
scheme, namespace, root, operation, path |
+//! | operation_entries | Histogram | Current operation size in
entries, represents the entries being processed | scheme,
namespace, root, operation, path |
+//! | operation_entries_rate | Histogram | Histogram of entries
processing rates in entries per second within individual operations | scheme,
namespace, root, operation, path |
+//! | operation_duration_seconds | Histogram | Duration of operations in
seconds, measured from start to completion | scheme,
namespace, root, operation, path |
+//! | operation_errors_total | Counter | Total number of failed
operations | scheme,
namespace, root, operation, path, error |
+//! | operation_executing | Gauge | Number of operations
currently being executed | scheme,
namespace, root, operation |
+//! | operation_ttfb_seconds | Histogram | Time to first byte in
seconds for operations | scheme,
namespace, root, operation, path |
+//!
+//! ## HTTP Metrics
+//!
+//! These metrics track the underlying HTTP requests made by OpenDAL services
that use HTTP.
+//!
+//! | Metric Name | Type | Description
| Labels
|
+//!
|----------------------------------|-----------|--------------------------------------------------------------------------------------------|-------------------------------------------------|
+//! | http_connection_errors_total | Counter | Total number of HTTP
requests that failed before receiving a response | scheme,
namespace, root, operation, error |
+//! | http_status_errors_total | Counter | Total number of HTTP
requests that received error status codes (non-2xx responses) | scheme,
namespace, root, operation, status |
+//! | http_executing | Gauge | Number of HTTP requests
currently in flight from this client | scheme,
namespace, root |
+//! | http_request_bytes | Histogram | Histogram of HTTP request
body sizes in bytes | scheme,
namespace, root, operation |
+//! | http_request_bytes_rate | Histogram | Histogram of HTTP request
bytes per second rates | scheme,
namespace, root, operation |
+//! | http_request_duration_seconds | Histogram | Histogram of time spent
sending HTTP requests, from first byte sent to first byte received | scheme,
namespace, root, operation |
+//! | http_response_bytes | Histogram | Histogram of HTTP
response body sizes in bytes |
scheme, namespace, root, operation |
+//! | http_response_bytes_rate | Histogram | Histogram of HTTP
response bytes per second rates |
scheme, namespace, root, operation |
+//! | http_response_duration_seconds | Histogram | Histogram of time spent
receiving HTTP responses, from first byte to last byte received | scheme,
namespace, root, operation |
+//!
+//! ## Label Descriptions
+//!
+//! | Label | Description
| Example Values |
+//!
|-----------|---------------------------------------------------------------|----------------------------------------|
+//! | scheme | The storage service scheme
| s3, gcs, azblob, fs, memory |
+//! | namespace | The storage service namespace (bucket, container, etc.)
| my-bucket, my-container |
+//! | root | The root path within the namespace
| /data, /backup |
+//! | operation | The operation being performed
| read, write, stat, list, delete |
+//! | path | The path of the object being operated on
| /path/to/file.txt |
+//! | error | The error type or message for error metrics
| not_found, permission_denied |
+//! | status | The HTTP status code for HTTP error metrics
| 404, 403, 500 |
+//!
+//! ## Metric Types
+//!
+//! * **Histogram**: Distribution of values with configurable buckets,
includes count, sum and quantiles
+//! * **Counter**: Cumulative metric that only increases over time (resets on
restart)
+//! * **Gauge**: Point-in-time metric that can increase and decrease
+
use std::fmt::Debug;
use std::fmt::Formatter;
use std::pin::Pin;
@@ -26,9 +83,8 @@ use std::task::ready;
use futures::Stream;
use futures::StreamExt;
use http::StatusCode;
-
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
const KIB: f64 = 1024.0;
const MIB: f64 = 1024.0 * KIB;
diff --git a/core/layers/otelmetrics/Cargo.toml
b/core/layers/otelmetrics/Cargo.toml
new file mode 100644
index 000000000..3441b07a1
--- /dev/null
+++ b/core/layers/otelmetrics/Cargo.toml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL OpenTelemetry metrics layer"
+name = "opendal-layer-otelmetrics"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+opendal-layer-observe-metrics-common = { path = "../observe-metrics-common",
version = "0.55.0", default-features = false }
+opentelemetry = { version = "0.31.0", default-features = false, features =
["metrics"] }
diff --git a/core/core/src/layers/otelmetrics.rs
b/core/layers/otelmetrics/src/lib.rs
similarity index 98%
rename from core/core/src/layers/otelmetrics.rs
rename to core/layers/otelmetrics/src/lib.rs
index cd4962abb..5b084a59c 100644
--- a/core/core/src/layers/otelmetrics.rs
+++ b/core/layers/otelmetrics/src/lib.rs
@@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
+use opendal_core::raw::*;
+use opendal_layer_observe_metrics_common as observe;
use opentelemetry::KeyValue;
use opentelemetry::metrics::Counter;
use opentelemetry::metrics::Histogram;
use opentelemetry::metrics::Meter;
use opentelemetry::metrics::UpDownCounter;
-use crate::layers::observe;
-use crate::raw::*;
-
/// Add
[opentelemetry::metrics](https://docs.rs/opentelemetry/latest/opentelemetry/metrics/index.html)
for every operation.
///
/// # Examples
///
/// ```no_run
-/// # use opendal_core::layers::OtelMetricsLayer;
+/// # use opendal_layer_otelmetrics::OtelMetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -53,7 +52,7 @@ impl OtelMetricsLayer {
/// # Examples
///
/// ```no_run
- /// # use opendal_core::layers::OtelMetricsLayer;
+ /// # use opendal_layer_otelmetrics::OtelMetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -150,7 +149,7 @@ impl OtelMetricsLayerBuilder {
/// # Examples
///
/// ```no_run
- /// # use opendal_core::layers::OtelMetricsLayer;
+ /// # use opendal_layer_otelmetrics::OtelMetricsLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -421,6 +420,7 @@ impl observe::MetricsIntercept for OtelMetricsInterceptor {
observe::MetricValue::HttpStatusErrorsTotal => {
self.http_status_errors_total.add(1, &attributes)
}
+ _ => {}
}
}
}
diff --git a/core/layers/prometheus-client/Cargo.toml
b/core/layers/prometheus-client/Cargo.toml
new file mode 100644
index 000000000..2fe48a82a
--- /dev/null
+++ b/core/layers/prometheus-client/Cargo.toml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL metrics layer using prometheus-client"
+name = "opendal-layer-prometheus-client"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+opendal-layer-observe-metrics-common = { path = "../observe-metrics-common",
version = "0.55.0", default-features = false }
+prometheus-client = { version = "0.24" }
diff --git a/core/core/src/layers/prometheus_client.rs
b/core/layers/prometheus-client/src/lib.rs
similarity index 98%
rename from core/core/src/layers/prometheus_client.rs
rename to core/layers/prometheus-client/src/lib.rs
index 1198cd582..85f31848b 100644
--- a/core/core/src/layers/prometheus_client.rs
+++ b/core/layers/prometheus-client/src/lib.rs
@@ -17,6 +17,9 @@
use std::fmt;
+use opendal_core::raw::*;
+use opendal_core::*;
+use opendal_layer_observe_metrics_common as observe;
use prometheus_client::encoding::EncodeLabel;
use prometheus_client::encoding::EncodeLabelSet;
use prometheus_client::encoding::LabelSetEncoder;
@@ -29,10 +32,6 @@ use prometheus_client::registry::Metric;
use prometheus_client::registry::Registry;
use prometheus_client::registry::Unit;
-use crate::layers::observe;
-use crate::raw::*;
-use crate::*;
-
/// Add [prometheus-client](https://docs.rs/prometheus-client) for every
operation.
///
/// # Prometheus Metrics
@@ -44,7 +43,7 @@ use crate::*;
///
/// ```no_run
/// # use log::info;
-/// # use opendal_core::layers::PrometheusClientLayer;
+/// # use opendal_layer_prometheus_client::PrometheusClientLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -181,7 +180,7 @@ impl PrometheusClientLayerBuilder {
/// # Example
///
/// ```no_run
- /// # use opendal_core::layers::PrometheusClientLayer;
+ /// # use opendal_layer_prometheus_client::PrometheusClientLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -477,6 +476,7 @@ impl observe::MetricsIntercept for
PrometheusClientInterceptor {
observe::MetricValue::HttpStatusErrorsTotal => {
self.http_status_errors_total.get_or_create(&labels).inc();
}
+ _ => {}
};
}
}
diff --git a/core/layers/prometheus/Cargo.toml
b/core/layers/prometheus/Cargo.toml
new file mode 100644
index 000000000..45f588a22
--- /dev/null
+++ b/core/layers/prometheus/Cargo.toml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL Prometheus metrics layer"
+name = "opendal-layer-prometheus"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+opendal-layer-observe-metrics-common = { path = "../observe-metrics-common",
version = "0.55.0", default-features = false }
+prometheus = { version = "0.14", features = ["process"] }
diff --git a/core/core/src/layers/prometheus.rs
b/core/layers/prometheus/src/lib.rs
similarity index 98%
rename from core/core/src/layers/prometheus.rs
rename to core/layers/prometheus/src/lib.rs
index eb6dfae4a..003e3d5ed 100644
--- a/core/core/src/layers/prometheus.rs
+++ b/core/layers/prometheus/src/lib.rs
@@ -15,6 +15,10 @@
// specific language governing permissions and limitations
// under the License.
+use opendal_core::raw::Access;
+use opendal_core::raw::*;
+use opendal_core::*;
+use opendal_layer_observe_metrics_common as observe;
use prometheus::HistogramVec;
use prometheus::Registry;
use prometheus::core::AtomicI64;
@@ -25,11 +29,6 @@ use prometheus::register_histogram_vec_with_registry;
use prometheus::register_int_counter_vec_with_registry;
use prometheus::register_int_gauge_vec_with_registry;
-use crate::layers::observe;
-use crate::raw::Access;
-use crate::raw::*;
-use crate::*;
-
/// Add [prometheus](https://docs.rs/prometheus) for every operation.
///
/// # Prometheus Metrics
@@ -43,7 +42,7 @@ use crate::*;
///
/// ```no_run
/// # use log::info;
-/// # use opendal_core::layers::PrometheusLayer;
+/// # use opendal_layer_prometheus::PrometheusLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -94,7 +93,7 @@ use crate::*;
/// ```no_run
/// # use std::sync::OnceLock;
/// # use log::info;
-/// # use opendal_core::layers::PrometheusLayer;
+/// # use opendal_layer_prometheus::PrometheusLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -146,7 +145,7 @@ impl PrometheusLayer {
/// # Example
///
/// ```no_run
- /// # use opendal_core::layers::PrometheusLayer;
+ /// # use opendal_layer_prometheus::PrometheusLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -261,7 +260,7 @@ impl PrometheusLayerBuilder {
/// # Example
///
/// ```no_run
- /// # use opendal_core::layers::PrometheusLayer;
+ /// # use opendal_layer_prometheus::PrometheusLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -499,7 +498,7 @@ impl PrometheusLayerBuilder {
/// # Example
///
/// ```no_run
- /// # use opendal_core::layers::PrometheusLayer;
+ /// # use opendal_layer_prometheus::PrometheusLayer;
/// # use opendal_core::services;
/// # use opendal_core::Operator;
/// # use opendal_core::Result;
@@ -624,6 +623,7 @@ impl observe::MetricsIntercept for PrometheusInterceptor {
.http_status_errors_total
.with_label_values(&labels.values())
.inc(),
+ _ => {}
}
}
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 3f3da0356..5a6625e97 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -54,4 +54,14 @@ pub mod layers {
pub use opendal_layer_await_tree::*;
#[cfg(feature = "layers-capability-check")]
pub use opendal_layer_capability_check::*;
+ #[cfg(feature = "layers-fastmetrics")]
+ pub use opendal_layer_fastmetrics::*;
+ #[cfg(feature = "layers-metrics")]
+ pub use opendal_layer_metrics::*;
+ #[cfg(feature = "layers-otel-metrics")]
+ pub use opendal_layer_otelmetrics::*;
+ #[cfg(feature = "layers-prometheus")]
+ pub use opendal_layer_prometheus::*;
+ #[cfg(feature = "layers-prometheus-client")]
+ pub use opendal_layer_prometheus_client::*;
}
