This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new fbfd9109b More db delete 500 race fixes
fbfd9109b is described below
commit fbfd9109bd44e7156a7b6ad0eb8578ca4aff9435
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Oct 7 15:00:57 2026 -0400
More db delete 500 race fixes
Continuation of https://github.com/apache/couchdb/pull/6145. When we ran
internal tests we noticed a few 500 errors still leaked when db is opened
with
a call to get_security() first. Then, instead of getting
`database_does_not_exist` we caught a `throw({error, missing_target})` from
`get_or_create_db/2`.
Issue: #6142
---
src/chttpd/test/eunit/chttpd_db_test.erl | 42 +++++++++++++++++++++-----------
src/fabric/src/fabric_util.erl | 32 +++++++++++++++++++-----
2 files changed, 54 insertions(+), 20 deletions(-)
diff --git a/src/chttpd/test/eunit/chttpd_db_test.erl
b/src/chttpd/test/eunit/chttpd_db_test.erl
index 85bec025c..73b81406f 100644
--- a/src/chttpd/test/eunit/chttpd_db_test.erl
+++ b/src/chttpd/test/eunit/chttpd_db_test.erl
@@ -58,6 +58,7 @@ all_test_() ->
?TDEF_FE(t_return_ok_true_on_ensure_full_commit, ?TIMEOUT),
?TDEF_FE(t_return_404_for_ensure_full_commit_on_no_db,
?TIMEOUT),
?TDEF_FE(t_return_404_for_delete_db_with_stale_shard_cache, ?TIMEOUT),
+
?TDEF_FE(t_return_404_for_get_security_with_stale_shard_cache, ?TIMEOUT),
?TDEF_FE(t_accept_live_as_an_alias_for_continuous,
?TIMEOUT),
?TDEF_FE(t_return_headers_after_starting_continuous,
?TIMEOUT),
?TDEF_FE(t_return_404_for_delete_att_on_notadoc, ?TIMEOUT),
@@ -147,24 +148,17 @@ t_return_404_for_ensure_full_commit_on_no_db(Db) ->
% then can reach nodes after the dbs doc is gone and so it should answer with a
% 404.
t_return_404_for_delete_db_with_stale_shard_cache(_Db) ->
- Db = ?tempdb(),
- create_db(Db),
- Shards = lists:sort(mem3:shards(Db)),
- delete_db(Db),
- test_util:wait(fun() ->
- try mem3:shards(Db) of
- _ -> wait
- catch
- error:database_does_not_exist -> ok
- end
- end),
- % simulate the re-insert
- true = ets:insert(mem3_shards, Shards),
- ?assertEqual(Shards, lists:sort(mem3:shards(Db))),
+ Db = stale_shard_cache_db(),
{Status, Response} = req(delete, url(Db)),
?assertEqual(404, Status),
?assertMatch(#{<<"error">> := <<"not_found">>}, Response).
+t_return_404_for_get_security_with_stale_shard_cache(_Db) ->
+ Db = stale_shard_cache_db(),
+ {Status, Response} = req(get, url(Db, "_security")),
+ ?assertEqual(404, Status),
+ ?assertMatch(#{<<"error">> := <<"not_found">>}, Response).
+
t_accept_live_as_an_alias_for_continuous(Db) ->
GetLastSeq =
fun(Chunks) ->
@@ -337,6 +331,26 @@ helper_queries_with_keys_limit_skip(Db, Path,
ExpectedOffset, ExpectedRows1, Exp
?assertEqual(ExpectedRows2, length(Rows2)).
%%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%%
+
+% Create and delete a db, then put its shard map back into the shard cache to
+% simulate a read racing the delete. Returns the db name.
+stale_shard_cache_db() ->
+ Db = ?tempdb(),
+ create_db(Db),
+ Shards = lists:sort(mem3:shards(Db)),
+ delete_db(Db),
+ test_util:wait(fun() ->
+ try mem3:shards(Db) of
+ _ -> wait
+ catch
+ error:database_does_not_exist -> ok
+ end
+ end),
+ % simulate the re-insert
+ true = ets:insert(mem3_shards, Shards),
+ ?assertEqual(Shards, lists:sort(mem3:shards(Db))),
+ Db.
+
url(Db) ->
Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
Port = mochiweb_socket_server:get(chttpd, port),
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index e8b998a19..5437f0c14 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -139,11 +139,21 @@ get_db(DbName, Options) ->
MinTimeout = config:get_integer("fabric", "shard_timeout_min_msec", 100),
MaxTimeout = request_timeout(),
Timeout = get_db_timeout(length(Live), Factor, MinTimeout, MaxTimeout),
- get_shard(Live, Options, Timeout, Factor).
+ case get_shard(Live, Options, Timeout, Factor, []) of
+ {ok, Db} ->
+ {ok, Db};
+ {error, Failures} ->
+ % missing_target means the shard doc is missing. If every worker
+ % returns that the shard map from our cache is stale
+ case all_missing_target(Failures) of
+ true -> error(database_does_not_exist, [DbName]);
+ false -> error({internal_server_error, "No DB shards could be
opened."})
+ end
+ end.
-get_shard([], _Opts, _Timeout, _Factor) ->
- error({internal_server_error, "No DB shards could be opened."});
-get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) ->
+get_shard([], _Opts, _Timeout, _Factor, Failures) ->
+ {error, Failures};
+get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor,
Failures) ->
Mon = rexi_monitor:start([rexi_utils:server_pid(Node)]),
MFA = {fabric_rpc, open_shard, [Name, [{timeout, Timeout} | Opts]]},
Ref = rexi:cast(Node, self(), MFA, [sync]),
@@ -157,15 +167,25 @@ get_shard([#shard{node = Node, name = Name} | Rest],
Opts, Timeout, Factor) ->
throw(Error);
{Ref, Reason} ->
couch_log:debug("Failed to open shard ~p because: ~p", [Name,
Reason]),
- get_shard(Rest, Opts, Timeout, Factor)
+ get_shard(Rest, Opts, Timeout, Factor, [Reason | Failures])
after Timeout ->
couch_log:debug("Failed to open shard ~p after: ~p", [Name,
Timeout]),
- get_shard(Rest, Opts, Factor * Timeout, Factor)
+ get_shard(Rest, Opts, Factor * Timeout, Factor, [timeout |
Failures])
end
after
rexi_monitor:stop(Mon)
end.
+all_missing_target([]) ->
+ false;
+all_missing_target(Failures) ->
+ lists:all(fun is_missing_target/1, Failures).
+
+is_missing_target({'rexi_EXIT', {{error, missing_target}, _}}) ->
+ true;
+is_missing_target(_) ->
+ false.
+
get_db_timeout(N, Factor, MinTimeout, infinity) ->
% MaxTimeout may be infinity so we just use the largest Erlang small int to
% avoid blowing up the arithmetic