This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new ab2cf16a6 Add dbname to mango exec stats (#4990)
ab2cf16a6 is described below
commit ab2cf16a60f4cf1cdde7b3f8c2b8e9828ca46fe5
Author: Russell Branca <chewbra...@apache.org>
AuthorDate: Tue Mar 5 14:44:07 2024 -0800
Add dbname to mango exec stats (#4990)
* WIP: include dbname in mango exec stats
* Don't output dbname in exec stats http response
* Make sure mango nouveau logs reports
* Test that mango reports get the dbname in the stats
* With compliant formatting
* Switch to adding exec stats dbname during create
* Add stats to explain cursor
* Add stats to mango cursor special
* Fix mango special cursor test
---
src/mango/src/mango_cursor.erl | 4 +-
src/mango/src/mango_cursor.hrl | 2 +-
src/mango/src/mango_cursor_nouveau.erl | 7 ++-
src/mango/src/mango_cursor_special.erl | 22 ++++++++--
src/mango/src/mango_cursor_text.erl | 25 +++++++----
src/mango/src/mango_cursor_view.erl | 76 ++++++++++++++++++++++++++-------
src/mango/src/mango_execution_stats.erl | 26 +++++++++--
src/mango/src/mango_execution_stats.hrl | 3 +-
8 files changed, 132 insertions(+), 33 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 3eedc7383..deb5d9144 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -424,6 +424,7 @@ create_cursor(Db, {[], Trace0}, Selector, Opts) ->
Skip = couch_util:get_value(skip, Opts, 0),
Fields = couch_util:get_value(fields, Opts, all_fields),
Bookmark = couch_util:get_value(bookmark, Opts),
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
{ok, #cursor{
db = Db,
index = none,
@@ -433,7 +434,8 @@ create_cursor(Db, {[], Trace0}, Selector, Opts) ->
limit = Limit,
skip = Skip,
fields = Fields,
- bookmark = Bookmark
+ bookmark = Bookmark,
+ execution_stats = Stats
}};
create_cursor(Db, {Indexes, Trace0}, Selector, Opts) ->
Trace1 = maps:merge(Trace0, #{filtered_indexes =>
sets:from_list(Indexes)}),
diff --git a/src/mango/src/mango_cursor.hrl b/src/mango/src/mango_cursor.hrl
index 6e5ffd6d9..cf02492a3 100644
--- a/src/mango/src/mango_cursor.hrl
+++ b/src/mango/src/mango_cursor.hrl
@@ -25,7 +25,7 @@
fields = undefined,
user_fun,
user_acc,
- execution_stats = #execution_stats{},
+ execution_stats,
bookmark,
bookmark_docid,
bookmark_key
diff --git a/src/mango/src/mango_cursor_nouveau.erl
b/src/mango/src/mango_cursor_nouveau.erl
index 5471ea46a..71beb053f 100644
--- a/src/mango/src/mango_cursor_nouveau.erl
+++ b/src/mango/src/mango_cursor_nouveau.erl
@@ -53,6 +53,8 @@ create(Db, {Indexes, Trace}, Selector, Opts) ->
Skip = couch_util:get_value(skip, Opts, 0),
Fields = couch_util:get_value(fields, Opts, all_fields),
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
+
{ok, #cursor{
db = Db,
index = Index,
@@ -62,7 +64,8 @@ create(Db, {Indexes, Trace}, Selector, Opts) ->
opts = Opts,
limit = Limit,
skip = Skip,
- fields = Fields
+ fields = Fields,
+ execution_stats = Stats
}}.
explain(Cursor) ->
@@ -128,6 +131,8 @@ execute(Cursor, UserFun, UserAcc) ->
{FinalUserAcc0, Stats1} = mango_execution_stats:maybe_add_stats(
Opts, UserFun, Stats0, FinalUserAcc
),
+ %% This needs Stats1 as log_end is called in maybe_add_stats
+ mango_execution_stats:log_stats(Stats1),
FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor,
Stats1, FinalUserAcc0),
{ok, FinalUserAcc1}
end.
diff --git a/src/mango/src/mango_cursor_special.erl
b/src/mango/src/mango_cursor_special.erl
index 1891ba6d0..c466b5db1 100644
--- a/src/mango/src/mango_cursor_special.erl
+++ b/src/mango/src/mango_cursor_special.erl
@@ -45,6 +45,7 @@ create(Db, {Indexes, Trace0}, Selector, Opts) ->
Skip = couch_util:get_value(skip, Opts, 0),
Fields = couch_util:get_value(fields, Opts, all_fields),
Bookmark = couch_util:get_value(bookmark, Opts),
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
IndexRanges1 = mango_cursor:maybe_noop_range(Selector, IndexRanges),
Trace = maps:merge(Trace0, #{sorted_index_ranges => SortedIndexRanges}),
@@ -59,7 +60,8 @@ create(Db, {Indexes, Trace0}, Selector, Opts) ->
limit = Limit,
skip = Skip,
fields = Fields,
- bookmark = Bookmark
+ bookmark = Bookmark,
+ execution_stats = Stats
}}.
explain(Cursor) ->
@@ -74,13 +76,26 @@ handle_message(Msg, Cursor) ->
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
-create_test() ->
+create_test_() ->
+ {
+ foreach,
+ fun() ->
+ meck:expect(couch_db, name, fun(A) when is_atom(A) ->
atom_to_binary(A) end)
+ end,
+ fun(_) -> meck:unload() end,
+ [
+ ?TDEF_FE(t_create)
+ ]
+ }.
+
+t_create(_) ->
Index = #idx{type = <<"special">>, def = all_docs},
Indexes = [Index],
Ranges = [{'$gt', null, '$lt', mango_json_max}],
Trace = #{},
Selector = {[]},
Options = [{limit, limit}, {skip, skip}, {fields, fields}, {bookmark,
bookmark}],
+ Stats = mango_execution_stats:stats_init(couch_db:name(db)),
Cursor =
#cursor{
db = db,
@@ -92,7 +107,8 @@ create_test() ->
skip = skip,
fields = fields,
bookmark = bookmark,
- trace = #{sorted_index_ranges => [{Index, Ranges, 0}]}
+ trace = #{sorted_index_ranges => [{Index, Ranges, 0}]},
+ execution_stats = Stats
},
?assertEqual({ok, Cursor}, create(db, {Indexes, Trace}, Selector,
Options)).
diff --git a/src/mango/src/mango_cursor_text.erl
b/src/mango/src/mango_cursor_text.erl
index ee1d962f7..6d2204070 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -50,7 +50,9 @@ create(Db, {Indexes, Trace}, Selector, Opts0) ->
?MANGO_ERROR(multiple_text_indexes)
end,
- Opts = unpack_bookmark(couch_db:name(Db), Opts0),
+ DbName = couch_db:name(Db),
+ Opts = unpack_bookmark(DbName, Opts0),
+ Stats = mango_execution_stats:stats_init(DbName),
DreyfusLimit = get_dreyfus_limit(),
Limit = erlang:min(DreyfusLimit, couch_util:get_value(limit, Opts,
mango_opts:default_limit())),
@@ -66,7 +68,8 @@ create(Db, {Indexes, Trace}, Selector, Opts0) ->
opts = Opts,
limit = Limit,
skip = Skip,
- fields = Fields
+ fields = Fields,
+ execution_stats = Stats
}}.
explain(Cursor) ->
@@ -389,8 +392,10 @@ t_create_regular(_) ->
Limit = 10,
Options = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark,
bookmark}],
Options1 = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark,
unpacked_bookmark}],
+ Db = db,
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
Cursor = #cursor{
- db = db,
+ db = Db,
index = Index,
ranges = null,
trace = Trace,
@@ -398,17 +403,20 @@ t_create_regular(_) ->
opts = Options1,
limit = Limit,
skip = skip,
- fields = fields
+ fields = fields,
+ execution_stats = Stats
},
meck:expect(dreyfus_bookmark, unpack, [db_name, bookmark],
meck:val(unpacked_bookmark)),
- ?assertEqual({ok, Cursor}, create(db, {Indexes, Trace}, selector,
Options)).
+ ?assertEqual({ok, Cursor}, create(Db, {Indexes, Trace}, selector,
Options)).
t_create_no_bookmark(_) ->
Limit = 99,
Options = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark,
nil}],
Options1 = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark,
[]}],
+ Db = db,
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
Cursor = #cursor{
- db = db,
+ db = Db,
index = index,
ranges = null,
trace = trace,
@@ -416,9 +424,10 @@ t_create_no_bookmark(_) ->
opts = Options1,
limit = Limit,
skip = skip,
- fields = fields
+ fields = fields,
+ execution_stats = Stats
},
- ?assertEqual({ok, Cursor}, create(db, {[index], trace}, selector,
Options)).
+ ?assertEqual({ok, Cursor}, create(Db, {[index], trace}, selector,
Options)).
t_create_invalid_bookmark(_) ->
Options = [{bookmark, invalid}],
diff --git a/src/mango/src/mango_cursor_view.erl
b/src/mango/src/mango_cursor_view.erl
index b103d869d..ac8714aa7 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -105,6 +105,8 @@ create(Db, {Indexes, Trace0}, Selector, Opts) ->
IndexRanges1 = mango_cursor:maybe_noop_range(Selector, IndexRanges),
Trace = maps:merge(Trace0, #{sorted_index_ranges => SortedIndexRanges}),
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
+
{ok, #cursor{
db = Db,
index = Index,
@@ -115,7 +117,8 @@ create(Db, {Indexes, Trace0}, Selector, Opts) ->
limit = Limit,
skip = Skip,
fields = Fields,
- bookmark = Bookmark
+ bookmark = Bookmark,
+ execution_stats = Stats
}}.
-spec required_fields(#cursor{}) -> fields().
@@ -906,16 +909,30 @@ composite_indexes_test() ->
],
?assertEqual(Result, composite_indexes(Indexes, Ranges)).
-create_test() ->
+create_test_() ->
+ {
+ foreach,
+ fun() ->
+ meck:expect(couch_db, name, fun(A) when is_atom(A) ->
atom_to_binary(A) end)
+ end,
+ fun(_) -> meck:unload() end,
+ [
+ ?TDEF_FE(t_create_ok)
+ ]
+ }.
+
+t_create_ok(_) ->
Index = #idx{type = <<"json">>, def = {[{<<"fields">>, {[]}}]}},
Indexes = [Index],
Trace = #{},
Ranges = [],
Selector = {[]},
Options = [{limit, limit}, {skip, skip}, {fields, fields}, {bookmark,
bookmark}],
+ Db = db,
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
Cursor =
#cursor{
- db = db,
+ db = Db,
index = Index,
ranges = Ranges,
selector = Selector,
@@ -924,9 +941,10 @@ create_test() ->
skip = skip,
fields = fields,
bookmark = bookmark,
- trace = #{sorted_index_ranges => [{Index, [], 0}]}
+ trace = #{sorted_index_ranges => [{Index, [], 0}]},
+ execution_stats = Stats
},
- ?assertEqual({ok, Cursor}, create(db, {Indexes, Trace}, Selector,
Options)).
+ ?assertEqual({ok, Cursor}, create(Db, {Indexes, Trace}, Selector,
Options)).
to_selector(Map) ->
test_util:as_selector(Map).
@@ -993,7 +1011,8 @@ execute_test_() ->
fun() ->
meck:new(foo, [non_strict]),
meck:new(fabric),
- meck:new(chttpd_stats)
+ meck:new(chttpd_stats),
+ meck:expect(couch_db, name, fun(A) when is_atom(A) ->
atom_to_binary(A) end)
end,
fun(_) -> meck:unload() end,
[
@@ -1007,7 +1026,9 @@ execute_test_() ->
}.
t_execute_empty(_) ->
- Cursor = #cursor{ranges = [empty]},
+ Db = db,
+ Stats = mango_execution_stats:stats_init(couch_db:name(Db)),
+ Cursor = #cursor{ranges = [empty], execution_stats = Stats},
meck:expect(fabric, all_docs, ['_', '_', '_', '_', '_'], meck:val(error)),
meck:expect(fabric, query_view, ['_', '_', '_', '_', '_', '_'],
meck:val(error)),
?assertEqual({ok, accumulator}, execute(Cursor, undefined, accumulator)),
@@ -1023,6 +1044,7 @@ t_execute_ok_all_docs(_) ->
Index = #idx{type = <<"json">>, def = all_docs},
Selector = {[]},
Fields = all_fields,
+ Stats = mango_execution_stats:stats_init(db),
Cursor =
#cursor{
index = Index,
@@ -1031,7 +1053,8 @@ t_execute_ok_all_docs(_) ->
fields = Fields,
ranges = [{'$gte', start_key, '$lte', end_key}],
opts = [{user_ctx, user_ctx}],
- bookmark = nil
+ bookmark = nil,
+ execution_stats = Stats
},
Cursor1 =
Cursor#cursor{
@@ -1050,7 +1073,8 @@ t_execute_ok_all_docs(_) ->
#execution_stats{
totalKeysExamined = TotalKeysExamined,
totalDocsExamined = TotalDocsExamined,
- executionStartTime = {0, 0, 0}
+ executionStartTime = {0, 0, 0},
+ dbname = db
}
},
Extra =
@@ -1089,6 +1113,7 @@ t_execute_ok_query_view(_) ->
IndexScanWarning =
<<"The number of documents examined is high in proportion to the
number of results returned. Consider adding a more specific index to improve
this.">>,
UserFnParameters2 = [{add_key, warning, IndexScanWarning}, accumulator2],
+ Stats = mango_execution_stats:stats_init(db),
meck:expect(
foo,
bar,
@@ -1114,7 +1139,8 @@ t_execute_ok_query_view(_) ->
fields = Fields,
ranges = [{'$gte', start_key, '$lte', end_key}],
opts = [{user_ctx, user_ctx}],
- bookmark = nil
+ bookmark = nil,
+ execution_stats = Stats
},
Cursor1 =
Cursor#cursor{
@@ -1131,6 +1157,7 @@ t_execute_ok_query_view(_) ->
bookmark_key = undefined,
execution_stats =
#execution_stats{
+ dbname = db,
totalKeysExamined = TotalKeysExamined,
totalDocsExamined = TotalDocsExamined,
executionStartTime = {0, 0, 0}
@@ -1174,18 +1201,22 @@ t_execute_ok_query_view(_) ->
t_execute_ok_all_docs_with_execution_stats(_) ->
Bookmark = bookmark,
+ DbName = db,
TotalKeysExamined = 33,
TotalDocsExamined = 12,
TotalQuorumDocsExamined = 0,
ResultsReturned = 20,
+ InitStats = mango_execution_stats:stats_init(couch_db:name(DbName)),
ExecutionStats =
#execution_stats{
totalKeysExamined = TotalKeysExamined,
totalDocsExamined = TotalDocsExamined,
totalQuorumDocsExamined = TotalQuorumDocsExamined,
resultsReturned = ResultsReturned,
- executionStartTime = {0, 0, 0}
+ executionStartTime = {0, 0, 0},
+ dbname = DbName
},
+ %% Stats doesn't include dbname as it will have already been stripped out
Stats =
{[
{total_keys_examined, TotalKeysExamined},
@@ -1209,12 +1240,13 @@ t_execute_ok_all_docs_with_execution_stats(_) ->
Cursor =
#cursor{
index = Index,
- db = db,
+ db = DbName,
selector = Selector,
fields = Fields,
ranges = [{'$gte', start_key, '$lte', end_key}],
opts = [{user_ctx, user_ctx}, {execution_stats, true}],
- bookmark = nil
+ bookmark = nil,
+ execution_stats = InitStats
},
Cursor1 =
Cursor#cursor{
@@ -1253,13 +1285,24 @@ t_execute_ok_all_docs_with_execution_stats(_) ->
Parameters = [
db, [{user_ctx, user_ctx}], fun
mango_cursor_view:handle_all_docs_message/2, Cursor1, Args
],
+ meck:new(couch_log, [passthrough]),
+ %% Pattern matching on DbName in the fun head results in a shadowed
variable
+ %% warning. Creating a new variable and testing in the guard works around
this.
+ meck:expect(
+ couch_log,
+ report,
+ fun("mango-stats", #{dbname := DbName1}) when DbName1 =:= DbName ->
true end
+ ),
+ meck:expect(chttpd_stats, incr_rows, [TotalKeysExamined], meck:val(ok)),
meck:expect(chttpd_stats, incr_rows, [TotalKeysExamined], meck:val(ok)),
meck:expect(chttpd_stats, incr_reads, [TotalDocsExamined], meck:val(ok)),
meck:expect(fabric, all_docs, Parameters, meck:val({ok, Cursor2})),
?assertEqual({ok, updated_accumulator2}, execute(Cursor, fun foo:bar/2,
accumulator)),
+ ?assert(meck:called(couch_log, report, '_')),
?assert(meck:called(fabric, all_docs, '_')).
t_execute_error_1(_) ->
+ Stats = mango_execution_stats:stats_init(db),
Cursor =
#cursor{
index = #idx{type = <<"json">>, ddoc = <<"_design/ghibli">>, name
= index_name},
@@ -1268,7 +1311,8 @@ t_execute_error_1(_) ->
fields = all_fields,
ranges = [{'$gte', start_key, '$lte', end_key}],
opts = [{user_ctx, user_ctx}],
- bookmark = nil
+ bookmark = nil,
+ execution_stats = Stats
},
Parameters = [
db, '_', <<"ghibli">>, index_name, fun
mango_cursor_view:handle_message/2, '_', '_'
@@ -1279,6 +1323,7 @@ t_execute_error_1(_) ->
?assertNot(meck:called(chttpd_stats, incr_reads, '_')).
t_execute_error_2(_) ->
+ Stats = mango_execution_stats:stats_init(db),
Cursor =
#cursor{
index = #idx{type = <<"json">>, ddoc = <<"_design/ghibli">>, name
= index_name},
@@ -1287,7 +1332,8 @@ t_execute_error_2(_) ->
fields = all_fields,
ranges = [{'$gte', start_key, '$lte', end_key}],
opts = [{user_ctx, user_ctx}],
- bookmark = nil
+ bookmark = nil,
+ execution_stats = Stats
},
Parameters = [
db, '_', <<"ghibli">>, index_name, fun
mango_cursor_view:handle_message/2, '_', '_'
diff --git a/src/mango/src/mango_execution_stats.erl
b/src/mango/src/mango_execution_stats.erl
index dce7c087b..67a1edd0a 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -13,7 +13,7 @@
-module(mango_execution_stats).
-export([
- to_json/1,
+ to_json/1, to_json/2,
to_map/1,
incr_keys_examined/1,
incr_keys_examined/2,
@@ -28,7 +28,8 @@
shard_init/0,
shard_incr_keys_examined/0,
shard_incr_docs_examined/0,
- shard_get_stats/0
+ shard_get_stats/0,
+ stats_init/0, stats_init/1
]).
-include("mango.hrl").
@@ -36,13 +37,31 @@
-define(SHARD_STATS_KEY, mango_shard_execution_stats).
+stats_init() ->
+ #execution_stats{}.
+
+stats_init(DbName) ->
+ Stats = stats_init(),
+ Stats#execution_stats{dbname = DbName}.
+
to_json(Stats) ->
+ to_json(Stats, true).
+
+to_json(Stats, IncludeDbName) ->
+ Base =
+ case IncludeDbName of
+ true ->
+ [{dbname, Stats#execution_stats.dbname}];
+ false ->
+ []
+ end,
{[
{total_keys_examined, Stats#execution_stats.totalKeysExamined},
{total_docs_examined, Stats#execution_stats.totalDocsExamined},
{total_quorum_docs_examined,
Stats#execution_stats.totalQuorumDocsExamined},
{results_returned, Stats#execution_stats.resultsReturned},
{execution_time_ms, Stats#execution_stats.executionTimeMs}
+ | Base
]}.
to_map(Stats) ->
@@ -51,6 +70,7 @@ to_map(Stats) ->
total_docs_examined => Stats#execution_stats.totalDocsExamined,
total_quorum_docs_examined =>
Stats#execution_stats.totalQuorumDocsExamined,
results_returned => Stats#execution_stats.resultsReturned,
+ dbname => Stats#execution_stats.dbname,
execution_time_ms => Stats#execution_stats.executionTimeMs
}.
@@ -103,7 +123,7 @@ maybe_add_stats(Opts, UserFun, Stats0, UserAcc) ->
FinalAcc =
case couch_util:get_value(execution_stats, Opts) of
true ->
- JSONValue = to_json(Stats1),
+ JSONValue = to_json(Stats1, false),
Arg = {add_key, execution_stats, JSONValue},
{_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
FinalUserAcc;
diff --git a/src/mango/src/mango_execution_stats.hrl
b/src/mango/src/mango_execution_stats.hrl
index ea5ed5ee8..302460c26 100644
--- a/src/mango/src/mango_execution_stats.hrl
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -16,5 +16,6 @@
totalQuorumDocsExamined = 0,
resultsReturned = 0,
executionStartTime,
- executionTimeMs
+ executionTimeMs,
+ dbname
}).
