This is an automated email from the ASF dual-hosted git repository.

nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/main by this push:
     new fbfd9109b More db delete 500 race fixes
fbfd9109b is described below

commit fbfd9109bd44e7156a7b6ad0eb8578ca4aff9435
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Oct 7 15:00:57 2026 -0400

    More db delete 500 race fixes
    
    Continuation of https://github.com/apache/couchdb/pull/6145. When we ran
    internal tests we noticed a few 500 errors still leaked when db is opened 
with
    a call to get_security() first. Then, instead of getting
    `database_does_not_exist` we caught a `throw({error, missing_target})` from
    `get_or_create_db/2`.
    
    Issue: #6142
---
 src/chttpd/test/eunit/chttpd_db_test.erl | 42 +++++++++++++++++++++-----------
 src/fabric/src/fabric_util.erl           | 32 +++++++++++++++++++-----
 2 files changed, 54 insertions(+), 20 deletions(-)

diff --git a/src/chttpd/test/eunit/chttpd_db_test.erl 
b/src/chttpd/test/eunit/chttpd_db_test.erl
index 85bec025c..73b81406f 100644
--- a/src/chttpd/test/eunit/chttpd_db_test.erl
+++ b/src/chttpd/test/eunit/chttpd_db_test.erl
@@ -58,6 +58,7 @@ all_test_() ->
                     ?TDEF_FE(t_return_ok_true_on_ensure_full_commit, ?TIMEOUT),
                     ?TDEF_FE(t_return_404_for_ensure_full_commit_on_no_db, 
?TIMEOUT),
                     
?TDEF_FE(t_return_404_for_delete_db_with_stale_shard_cache, ?TIMEOUT),
+                    
?TDEF_FE(t_return_404_for_get_security_with_stale_shard_cache, ?TIMEOUT),
                     ?TDEF_FE(t_accept_live_as_an_alias_for_continuous, 
?TIMEOUT),
                     ?TDEF_FE(t_return_headers_after_starting_continuous, 
?TIMEOUT),
                     ?TDEF_FE(t_return_404_for_delete_att_on_notadoc, ?TIMEOUT),
@@ -147,24 +148,17 @@ t_return_404_for_ensure_full_commit_on_no_db(Db) ->
 % then can reach nodes after the dbs doc is gone and so it should answer with a
 % 404.
 t_return_404_for_delete_db_with_stale_shard_cache(_Db) ->
-    Db = ?tempdb(),
-    create_db(Db),
-    Shards = lists:sort(mem3:shards(Db)),
-    delete_db(Db),
-    test_util:wait(fun() ->
-        try mem3:shards(Db) of
-            _ -> wait
-        catch
-            error:database_does_not_exist -> ok
-        end
-    end),
-    % simulate the re-insert
-    true = ets:insert(mem3_shards, Shards),
-    ?assertEqual(Shards, lists:sort(mem3:shards(Db))),
+    Db = stale_shard_cache_db(),
     {Status, Response} = req(delete, url(Db)),
     ?assertEqual(404, Status),
     ?assertMatch(#{<<"error">> := <<"not_found">>}, Response).
 
+t_return_404_for_get_security_with_stale_shard_cache(_Db) ->
+    Db = stale_shard_cache_db(),
+    {Status, Response} = req(get, url(Db, "_security")),
+    ?assertEqual(404, Status),
+    ?assertMatch(#{<<"error">> := <<"not_found">>}, Response).
+
 t_accept_live_as_an_alias_for_continuous(Db) ->
     GetLastSeq =
         fun(Chunks) ->
@@ -337,6 +331,26 @@ helper_queries_with_keys_limit_skip(Db, Path, 
ExpectedOffset, ExpectedRows1, Exp
     ?assertEqual(ExpectedRows2, length(Rows2)).
 
 %%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%%
+
+% Create and delete a db, then put its shard map back into the shard cache to
+% simulate a read racing the delete. Returns the db name.
+stale_shard_cache_db() ->
+    Db = ?tempdb(),
+    create_db(Db),
+    Shards = lists:sort(mem3:shards(Db)),
+    delete_db(Db),
+    test_util:wait(fun() ->
+        try mem3:shards(Db) of
+            _ -> wait
+        catch
+            error:database_does_not_exist -> ok
+        end
+    end),
+    % simulate the re-insert
+    true = ets:insert(mem3_shards, Shards),
+    ?assertEqual(Shards, lists:sort(mem3:shards(Db))),
+    Db.
+
 url(Db) ->
     Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
     Port = mochiweb_socket_server:get(chttpd, port),
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index e8b998a19..5437f0c14 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -139,11 +139,21 @@ get_db(DbName, Options) ->
     MinTimeout = config:get_integer("fabric", "shard_timeout_min_msec", 100),
     MaxTimeout = request_timeout(),
     Timeout = get_db_timeout(length(Live), Factor, MinTimeout, MaxTimeout),
-    get_shard(Live, Options, Timeout, Factor).
+    case get_shard(Live, Options, Timeout, Factor, []) of
+        {ok, Db} ->
+            {ok, Db};
+        {error, Failures} ->
+            % missing_target means the shard doc is missing. If every worker
+            % returns that the shard map from our cache is stale
+            case all_missing_target(Failures) of
+                true -> error(database_does_not_exist, [DbName]);
+                false -> error({internal_server_error, "No DB shards could be 
opened."})
+            end
+    end.
 
-get_shard([], _Opts, _Timeout, _Factor) ->
-    error({internal_server_error, "No DB shards could be opened."});
-get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) ->
+get_shard([], _Opts, _Timeout, _Factor, Failures) ->
+    {error, Failures};
+get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor, 
Failures) ->
     Mon = rexi_monitor:start([rexi_utils:server_pid(Node)]),
     MFA = {fabric_rpc, open_shard, [Name, [{timeout, Timeout} | Opts]]},
     Ref = rexi:cast(Node, self(), MFA, [sync]),
@@ -157,15 +167,25 @@ get_shard([#shard{node = Node, name = Name} | Rest], 
Opts, Timeout, Factor) ->
                 throw(Error);
             {Ref, Reason} ->
                 couch_log:debug("Failed to open shard ~p because: ~p", [Name, 
Reason]),
-                get_shard(Rest, Opts, Timeout, Factor)
+                get_shard(Rest, Opts, Timeout, Factor, [Reason | Failures])
         after Timeout ->
             couch_log:debug("Failed to open shard ~p after: ~p", [Name, 
Timeout]),
-            get_shard(Rest, Opts, Factor * Timeout, Factor)
+            get_shard(Rest, Opts, Factor * Timeout, Factor, [timeout | 
Failures])
         end
     after
         rexi_monitor:stop(Mon)
     end.
 
+all_missing_target([]) ->
+    false;
+all_missing_target(Failures) ->
+    lists:all(fun is_missing_target/1, Failures).
+
+is_missing_target({'rexi_EXIT', {{error, missing_target}, _}}) ->
+    true;
+is_missing_target(_) ->
+    false.
+
 get_db_timeout(N, Factor, MinTimeout, infinity) ->
     % MaxTimeout may be infinity so we just use the largest Erlang small int to
     % avoid blowing up the arithmetic

Reply via email to