This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new c0cb1737d feat(prometheus): add Clouseau, DB count, IOQ and Smoosh
metrics
c0cb1737d is described below
commit c0cb1737dca40a4d070e00b0378c87785f06e5e8
Author: Sam Smith <[email protected]>
AuthorDate: Mon Jul 13 14:02:10 2026 +0100
feat(prometheus): add Clouseau, DB count, IOQ and Smoosh metrics
Expose new Prometheus metrics for monitoring:
- Clouseau search connectivity status
- Total database count in cluster
- IOQ active channels and request counts
- Smoosh compaction job statistics (active/starting/waiting)
Improves observability of core CouchDB subsystems.
---
src/couch_prometheus/src/couch_prometheus.erl | 130 ++++++++++++++-
.../test/eunit/couch_prometheus_e2e_tests.erl | 12 +-
.../test/eunit/couch_prometheus_tests.erl | 185 +++++++++++++++++++++
src/ioq/src/ioq.erl | 5 +-
test/fixtures/allowed-xref.txt | 2 -
5 files changed, 328 insertions(+), 6 deletions(-)
diff --git a/src/couch_prometheus/src/couch_prometheus.erl
b/src/couch_prometheus/src/couch_prometheus.erl
index 7f3dc494d..bba7f8b57 100644
--- a/src/couch_prometheus/src/couch_prometheus.erl
+++ b/src/couch_prometheus/src/couch_prometheus.erl
@@ -26,12 +26,20 @@
-ifdef(TEST).
-export([
- get_internal_replication_jobs_stat/0
+ get_internal_replication_jobs_stat/0,
+ get_clouseau_status/0,
+ get_database_count/0,
+ get_ioq_stats/0,
+ get_smoosh_stats/0
]).
-endif.
-define(PROMETHEUS_VERSION, "2.0").
+%% Connection status constants
+-define(CLOUSEAU_CONNECTED, 1).
+-define(CLOUSEAU_DISCONNECTED, 0).
+
scrape() ->
CouchDB = get_couchdb_stats(),
System = couch_stats_httpd:to_ejson(get_system_stats()),
@@ -69,7 +77,11 @@ get_system_stats() ->
get_membership_stat(),
get_membership_nodes(),
get_distribution_stats(),
- get_bt_engine_cache_stats()
+ get_bt_engine_cache_stats(),
+ get_clouseau_status(),
+ get_database_count(),
+ get_ioq_stats(),
+ get_smoosh_stats()
]).
get_uptime_stat() ->
@@ -384,3 +396,117 @@ get_bt_engine_cache_stats() ->
to_prom(couchdb_bt_engine_cache_memory, gauge, "memory used by the
btree cache", Mem),
to_prom(couchdb_bt_engine_cache_size, gauge, "number of entries in the
btree cache", Size)
].
+
+get_clouseau_status() ->
+ Value =
+ case clouseau_rpc:connected() of
+ true -> ?CLOUSEAU_CONNECTED;
+ false -> ?CLOUSEAU_DISCONNECTED
+ end,
+ to_prom(clouseau_connected, gauge, "clouseau connectivity status", Value).
+
+get_database_count() ->
+ ShardsDb = config:get("mem3", "shards_db", "_dbs"),
+ case fabric:get_db_info(ShardsDb) of
+ {ok, Info} ->
+ DbCount = couch_util:get_value(doc_count, Info),
+ to_prom(database_count, gauge, "total database count", DbCount);
+ {error, Reason} ->
+ couch_log:warning("~p failed to get database count: ~p", [?MODULE,
Reason]),
+ []
+ end.
+
+get_ioq_stats() ->
+ IOQData = ioq:get_disk_queues(),
+ Interactive = couch_util:get_value(interactive, IOQData),
+ Background = couch_util:get_value(background, IOQData),
+ case {Interactive, Background} of
+ {I, B} when is_integer(I), is_integer(B) ->
+ [
+ to_prom(ioq_interactive_requests, gauge, "IOQ interactive
queue requests", I),
+ to_prom(ioq_background_requests, gauge, "IOQ background queue
requests", B),
+ to_prom(ioq_total_requests, gauge, "IOQ total active
requests", I + B)
+ ];
+ _ ->
+ Compaction = couch_util:get_value(compaction, IOQData, 0),
+ Low = couch_util:get_value(low, IOQData, 0),
+ Replication = couch_util:get_value(replication, IOQData, 0),
+ Channels =
+ case couch_util:get_value(channels, IOQData, []) of
+ {List} when is_list(List) -> List;
+ _ -> []
+ end,
+ ChannelRequests = count_channel_requests(Channels),
+ [
+ to_prom(ioq_active_channels, gauge, "IOQ active channels",
length(Channels)),
+ to_prom(
+ ioq_compaction_requests, gauge, "IOQ compaction queue
requests", Compaction
+ ),
+ to_prom(ioq_low_requests, gauge, "IOQ low priority queue
requests", Low),
+ to_prom(
+ ioq_replication_requests, gauge, "IOQ replication queue
requests", Replication
+ ),
+ to_prom(ioq_channel_requests, gauge, "IOQ channel requests",
ChannelRequests),
+ to_prom(
+ ioq_total_requests,
+ gauge,
+ "IOQ total active requests",
+ ChannelRequests + Compaction + Low + Replication
+ )
+ ]
+ end.
+
+count_channel_requests(Channels) ->
+ lists:foldl(
+ fun
+ ({_User, Vals}, Acc) when is_list(Vals) ->
+ Acc + lists:sum(Vals);
+ (_, Acc) ->
+ Acc
+ end,
+ 0,
+ Channels
+ ).
+
+get_smoosh_stats() ->
+ case smoosh:status() of
+ {ok, #{channels := Channels}} ->
+ ChannelsList = maps:to_list(Channels),
+ ChannelCount = length(ChannelsList),
+ InitAcc = #{active => 0, starting => 0, waiting => 0},
+ GlobalTotals = lists:foldl(
+ fun({_Channel, StatsMap}, Acc) ->
+ Active = maps:get(active, StatsMap, 0),
+ Starting = maps:get(starting, StatsMap, 0),
+ WaitingMap = maps:get(waiting, StatsMap, #{}),
+ WaitingSize = maps:get(size, WaitingMap, 0),
+ #{
+ active => Active + maps:get(active, Acc, 0),
+ starting => Starting + maps:get(starting, Acc, 0),
+ waiting => WaitingSize + maps:get(waiting, Acc, 0)
+ }
+ end,
+ InitAcc,
+ ChannelsList
+ ),
+ TotalActive = maps:get(active, GlobalTotals, 0),
+ TotalStarting = maps:get(starting, GlobalTotals, 0),
+ TotalWaiting = maps:get(waiting, GlobalTotals, 0),
+ [
+ to_prom(smoosh_channel_count, gauge, "total active smoosh
channels", ChannelCount),
+ to_prom(
+ smoosh_active_jobs, gauge, "global total active compaction
jobs", TotalActive
+ ),
+ to_prom(
+ smoosh_starting_jobs,
+ gauge,
+ "global total starting compaction jobs",
+ TotalStarting
+ ),
+ to_prom(
+ smoosh_waiting_jobs, gauge, "global total waiting
compaction jobs", TotalWaiting
+ )
+ ];
+ _ ->
+ []
+ end.
diff --git a/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl
b/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl
index dd8da5f79..913b80834 100644
--- a/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl
+++ b/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl
@@ -42,7 +42,8 @@ e2e_test_() ->
?TDEF_FE(t_metric_updated),
?TDEF_FE(t_no_duplicate_metrics),
?TDEF_FE(t_starts_with_couchdb),
- ?TDEF_FE(t_survives_mem3_sync_termination)
+ ?TDEF_FE(t_survives_mem3_sync_termination),
+ ?TDEF_FE(t_system_stats_metrics_present)
]
}
}
@@ -186,6 +187,15 @@ t_survives_mem3_sync_termination(_) ->
couch_prometheus:get_internal_replication_jobs_stat()
).
+t_system_stats_metrics_present(Port) ->
+ Url = node_local_url(Port),
+ Stats = get_stats(Url),
+ % Verify system stats metrics exist in the scrape output
+ ?assertNotEqual(not_found, metric_value(Stats,
"couchdb_clouseau_connected")),
+ ?assertNotEqual(not_found, metric_value(Stats, "couchdb_database_count")),
+ ?assertNotEqual(not_found, metric_value(Stats,
"couchdb_ioq_total_requests")),
+ ?assertNotEqual(not_found, metric_value(Stats,
"couchdb_smoosh_channel_count")).
+
node_local_url(Port) ->
Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
lists:concat(["http://";, Addr, ":", Port, "/_node/_local/_prometheus"]).
diff --git a/src/couch_prometheus/test/eunit/couch_prometheus_tests.erl
b/src/couch_prometheus/test/eunit/couch_prometheus_tests.erl
new file mode 100644
index 000000000..7f79cb5a5
--- /dev/null
+++ b/src/couch_prometheus/test/eunit/couch_prometheus_tests.erl
@@ -0,0 +1,185 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_prometheus_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+
+unit_test_() ->
+ {
+ "Unit tests for system stats metrics",
+ {
+ setup,
+ fun setup_unit/0,
+ fun teardown_unit/1,
+ with([
+ ?TDEF(t_clouseau_connected),
+ ?TDEF(t_clouseau_disconnected),
+ ?TDEF(t_database_count_success),
+ ?TDEF(t_database_count_error),
+ ?TDEF(t_ioq_stats_basic),
+ ?TDEF(t_ioq_stats_with_channels),
+ ?TDEF(t_ioq_stats_couchdb_ioq_format),
+ ?TDEF(t_ioq_stats_couchdb_ioq_format_empty),
+ ?TDEF(t_smoosh_stats_success),
+ ?TDEF(t_smoosh_stats_error)
+ ])
+ }
+ }.
+
+setup_unit() ->
+ Mods = [clouseau_rpc, fabric, ioq, smoosh, config],
+ lists:foreach(fun(M) -> meck:new(M, [passthrough]) end, Mods),
+ Mods.
+
+teardown_unit(Mods) ->
+ lists:foreach(fun meck:unload/1, Mods).
+
+t_clouseau_connected(_) ->
+ meck:expect(clouseau_rpc, connected, [], meck:val(true)),
+ Result = couch_prometheus:get_clouseau_status(),
+ ?assertMatch([_, _, <<"couchdb_clouseau_connected 1">>], Result).
+
+t_clouseau_disconnected(_) ->
+ meck:expect(clouseau_rpc, connected, [], meck:val(false)),
+ Result = couch_prometheus:get_clouseau_status(),
+ ?assertMatch([_, _, <<"couchdb_clouseau_connected 0">>], Result).
+
+t_database_count_success(_) ->
+ meck:expect(config, get, 3, meck:val("_dbs")),
+ meck:expect(fabric, get_db_info, 1, meck:val({ok, [{doc_count, 42}]})),
+ Result = couch_prometheus:get_database_count(),
+ ?assertMatch([_, _, <<"couchdb_database_count 42">>], Result).
+
+t_database_count_error(_) ->
+ meck:expect(config, get, 3, meck:val("_dbs")),
+ meck:expect(fabric, get_db_info, 1, meck:val({error, not_found})),
+ Result = couch_prometheus:get_database_count(),
+ ?assertEqual([], Result).
+
+t_ioq_stats_basic(_) ->
+ meck:expect(
+ ioq,
+ get_disk_queues,
+ [],
+ meck:val([
+ {compaction, 1},
+ {low, 2},
+ {replication, 3}
+ ])
+ ),
+ Result = couch_prometheus:get_ioq_stats(),
+ ?assert(is_list(Result)),
+ ?assertEqual(6, length(Result)),
+ % Verify total requests = 1 + 2 + 3 = 6
+ [_, _, TotalLine] = lists:last(Result),
+ ?assertMatch(<<"couchdb_ioq_total_requests 6">>, TotalLine).
+
+t_ioq_stats_with_channels(_) ->
+ meck:expect(
+ ioq,
+ get_disk_queues,
+ [],
+ meck:val([
+ {compaction, 1},
+ {low, 2},
+ {replication, 3},
+ {channels, {[{<<"user1">>, [1, 2, 3]}]}}
+ ])
+ ),
+ Result = couch_prometheus:get_ioq_stats(),
+ ?assert(is_list(Result)),
+ ?assertEqual(6, length(Result)),
+ % Verify channel requests = 1 + 2 + 3 = 6
+ [_, _, ChannelLine] = lists:nth(5, Result),
+ ?assertMatch(<<"couchdb_ioq_channel_requests 6">>, ChannelLine),
+ % Verify total requests = 1 + 2 + 3 + 6 = 12
+ [_, _, TotalLine] = lists:last(Result),
+ ?assertMatch(<<"couchdb_ioq_total_requests 12">>, TotalLine).
+
+t_ioq_stats_couchdb_ioq_format(_) ->
+ meck:expect(
+ ioq,
+ get_disk_queues,
+ [],
+ meck:val([
+ {interactive, 10},
+ {background, 5}
+ ])
+ ),
+ Result = couch_prometheus:get_ioq_stats(),
+ ?assert(is_list(Result)),
+ ?assertEqual(3, length(Result)),
+ % Verify interactive requests
+ [_, _, InteractiveLine] = lists:nth(1, Result),
+ ?assertMatch(<<"couchdb_ioq_interactive_requests 10">>, InteractiveLine),
+ % Verify background requests
+ [_, _, BackgroundLine] = lists:nth(2, Result),
+ ?assertMatch(<<"couchdb_ioq_background_requests 5">>, BackgroundLine),
+ % Verify total requests = 10 + 5 = 15
+ [_, _, TotalLine] = lists:nth(3, Result),
+ ?assertMatch(<<"couchdb_ioq_total_requests 15">>, TotalLine).
+
+t_ioq_stats_couchdb_ioq_format_empty(_) ->
+ meck:expect(
+ ioq,
+ get_disk_queues,
+ [],
+ meck:val([
+ {interactive, 0},
+ {background, 0}
+ ])
+ ),
+ Result = couch_prometheus:get_ioq_stats(),
+ ?assert(is_list(Result)),
+ ?assertEqual(3, length(Result)),
+ % Verify all metrics are 0
+ [_, _, InteractiveLine] = lists:nth(1, Result),
+ ?assertMatch(<<"couchdb_ioq_interactive_requests 0">>, InteractiveLine),
+ [_, _, BackgroundLine] = lists:nth(2, Result),
+ ?assertMatch(<<"couchdb_ioq_background_requests 0">>, BackgroundLine),
+ [_, _, TotalLine] = lists:nth(3, Result),
+ ?assertMatch(<<"couchdb_ioq_total_requests 0">>, TotalLine).
+
+t_smoosh_stats_success(_) ->
+ meck:expect(
+ smoosh,
+ status,
+ [],
+ meck:val(
+ {ok, #{
+ channels => #{
+ <<"ratio_dbs">> => #{
+ active => 2,
+ starting => 1,
+ waiting => #{size => 5}
+ }
+ }
+ }}
+ )
+ ),
+ Result = couch_prometheus:get_smoosh_stats(),
+ ?assert(is_list(Result)),
+ ?assertEqual(4, length(Result)),
+ [_, _, ChannelLine] = lists:nth(1, Result),
+ ?assertMatch(<<"couchdb_smoosh_channel_count 1">>, ChannelLine),
+ [_, _, ActiveLine] = lists:nth(2, Result),
+ ?assertMatch(<<"couchdb_smoosh_active_jobs 2">>, ActiveLine),
+ [_, _, StartingLine] = lists:nth(3, Result),
+ ?assertMatch(<<"couchdb_smoosh_starting_jobs 1">>, StartingLine),
+ [_, _, WaitingLine] = lists:nth(4, Result),
+ ?assertMatch(<<"couchdb_smoosh_waiting_jobs 5">>, WaitingLine).
+
+t_smoosh_stats_error(_) ->
+ meck:expect(smoosh, status, [], meck:val({error, down})),
+ Result = couch_prometheus:get_smoosh_stats(),
+ ?assertEqual([], Result).
diff --git a/src/ioq/src/ioq.erl b/src/ioq/src/ioq.erl
index 031626cfe..c458b99ed 100644
--- a/src/ioq/src/ioq.erl
+++ b/src/ioq/src/ioq.erl
@@ -15,7 +15,7 @@
-behaviour(config_listener).
-export([start_link/0, call/3, call_search/3]).
--export([get_queue_lengths/0]).
+-export([get_disk_queues/0, get_queue_lengths/0]).
-export([get_io_priority/0, set_io_priority/1, maybe_set_io_priority/1]).
-export([bypass/2]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2]).
@@ -70,6 +70,9 @@ call(Fd, Msg, Metadata) ->
get_queue_lengths() ->
gen_server:call(?MODULE, get_queue_lengths).
+get_disk_queues() ->
+ maps:to_list(get_queue_lengths()).
+
bypass(Msg, Metadata) ->
Priority = io_class(Msg, Metadata),
case Priority of
diff --git a/test/fixtures/allowed-xref.txt b/test/fixtures/allowed-xref.txt
index c630fc109..e69de29bb 100644
--- a/test/fixtures/allowed-xref.txt
+++ b/test/fixtures/allowed-xref.txt
@@ -1,2 +0,0 @@
-src/ioq.erl: Warning: ioq:get_disk_queues/0 is undefined function (Xref)
-src/weatherreport_check_ioq.erl:{95,1}: Warning:
weatherreport_check_ioq:check_legacy_int/1 calls undefined function
ioq:get_disk_queues/0 (Xref)
