wcy-fdu commented on code in PR #1930: URL: https://github.com/apache/incubator-opendal/pull/1930#discussion_r1164427692
########## core/src/layers/prometheus.rs: ########## @@ -0,0 +1,990 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; +use std::io; +use std::sync::Arc; +use std::task::Context; +use std::task::Poll; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::FutureExt; +use futures::TryFutureExt; +use log::debug; + +use prometheus::core::AtomicU64; +use prometheus::core::GenericCounter; +use prometheus::exponential_buckets; +use prometheus::histogram_opts; +use prometheus::register_histogram_with_registry; +use prometheus::register_int_counter_with_registry; +use prometheus::Histogram; +use prometheus::Registry; + +use crate::ops::*; +use crate::raw::Accessor; +use crate::raw::*; +use crate::*; +/// Add [prometheus](https://docs.rs/prometheus) for every operations. +/// +/// # Examples +/// +/// ``` +/// use log::debug; +/// use log::info; +/// use opendal::services; +/// use opendal::Operator; +/// use opendal::Result; +/// +/// use opendal::layers::PrometheusLayer; +/// use prometheus::Encoder; +/// +/// /// Visit [`opendal::services`] for more service related config. +/// /// Visit [`opendal::Object`] for more object level APIs. +/// #[tokio::main] +/// async fn main() -> Result<()> { +/// // Pick a builder and configure it. +/// let builder = services::Memory::default(); +/// +/// let op = Operator::new(builder) +/// .expect("must init") +/// .layer(PrometheusLayer) +/// .finish(); +/// debug!("operator: {op:?}"); +/// +/// // Write data into object test. +/// op.write("test", "Hello, World!").await?; +/// // Read data from object. +/// let bs = op.read("test").await?; +/// info!("content: {}", String::from_utf8_lossy(&bs)); +/// +/// // Get object metadata. +/// let meta = op.stat("test").await?; +/// info!("meta: {:?}", meta); +/// +/// // Export prometheus metrics. +/// let mut buffer = Vec::<u8>::new(); +/// let encoder = prometheus::TextEncoder::new(); +/// encoder.encode(&prometheus::gather(), &mut buffer).unwrap(); +/// println!("## Prometheus Metrics"); +/// println!("{}", String::from_utf8(buffer.clone()).unwrap()); +/// Ok(()) +/// } +/// ``` +#[derive(Debug, Copy, Clone)] +pub struct PrometheusLayer; + +impl<A: Accessor> Layer<A> for PrometheusLayer { + type LayeredAccessor = PrometheusMetricsAccessor<A>; + + fn layer(&self, inner: A) -> Self::LayeredAccessor { + let registry = prometheus::default_registry(); + + PrometheusMetricsAccessor { + inner, + stats: Arc::new(PrometheusMetrics::new(registry.clone())), + } + } +} +/// [`PrometheusMetrics`] provide the performance and IO metrics. +#[derive(Debug)] +pub struct PrometheusMetrics { + // metadata + pub metadata_request_counts: GenericCounter<AtomicU64>, + pub metadata_request_latency: Histogram, + + // create + pub create_request_counts: GenericCounter<AtomicU64>, + pub create_request_latency: Histogram, + + /// read + pub read_request_counts: GenericCounter<AtomicU64>, + pub read_request_latency: Histogram, + pub read_size: Histogram, + + // write + pub write_request_counts: GenericCounter<AtomicU64>, + pub write_request_latency: Histogram, + pub write_size: Histogram, + + // stat + pub stat_request_counts: GenericCounter<AtomicU64>, + pub stat_request_latency: Histogram, + + // delete + pub delete_request_counts: GenericCounter<AtomicU64>, + pub delete_request_latency: Histogram, + + // list + pub list_request_counts: GenericCounter<AtomicU64>, + pub list_request_latency: Histogram, + + // scan + pub scan_request_counts: GenericCounter<AtomicU64>, + pub scan_request_latency: Histogram, + + // presign + pub presign_request_counts: GenericCounter<AtomicU64>, + pub presign_request_latency: Histogram, + + // batch + pub batch_request_counts: GenericCounter<AtomicU64>, + pub batch_request_latency: Histogram, + + // blocking create + pub blocking_create_request_counts: GenericCounter<AtomicU64>, + pub blocking_create_request_latency: Histogram, + + // blocking read + pub blocking_read_request_counts: GenericCounter<AtomicU64>, + pub blocking_read_request_latency: Histogram, + pub blocking_read_size: Histogram, + + // blocking write + pub blocking_write_request_counts: GenericCounter<AtomicU64>, + pub blocking_write_request_latency: Histogram, + pub blocking_write_size: Histogram, + + // blocking stat + pub blocking_stat_request_counts: GenericCounter<AtomicU64>, + pub blocking_stat_request_latency: Histogram, + + // blocking delete + pub blocking_delete_request_counts: GenericCounter<AtomicU64>, + pub blocking_delete_request_latency: Histogram, + + // blocking list + pub blocking_list_request_counts: GenericCounter<AtomicU64>, + pub blocking_list_request_latency: Histogram, + + // blocking scan + pub blocking_scan_request_counts: GenericCounter<AtomicU64>, + pub blocking_scan_request_latency: Histogram, +} + +impl PrometheusMetrics { + /// new with prometheus register. + pub fn new(registry: Registry) -> Self { + // metadata + let metadata_request_counts = register_int_counter_with_registry!( + "metadata_request_counts", Review Comment: >does prometheus have the silimar concept of label? Sure, [an example](https://github.com/risingwavelabs/risingwave/blob/ee6d44f3dfb3a6a36de77033f2f7147bf77d1018/src/stream/src/executor/monitor/streaming_stats.rs#L104). Will refactor it soon. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
