This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch mem3-rep-batch-preads in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit eea023610014331293eb0feb29eaccc75d655c17 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Nov 5 23:20:53 2026 -0500 Add a batched version of make_docs For a while now we were able to issue a batch of preads to the couch_file [1]. We can use that to batch read `#doc{}` records. With this we can speed up the internal replicator, especially when copying lots of small documents between shards. Tested with small 1GB of 1KB docs, blew away a shard copy then re-created with the internal replicator: main: 113-118 sec pr: 81-85 sec About 1.4x faster or so. [1] https://github.com/apache/couchdb/blob/eafed9939fe88de296d8875cf3b2ace5be3293cb/src/couch/src/couch_file.erl#L199 --- src/couch/src/couch_bt_engine.erl | 10 ++++ src/couch/src/couch_db.erl | 29 +++++++--- src/couch/src/couch_db_engine.erl | 10 ++++ src/couch/test/eunit/couch_db_doc_tests.erl | 88 ++++++++++++++++++++++++----- src/mem3/src/mem3_rep.erl | 16 ++---- 5 files changed, 123 insertions(+), 30 deletions(-) diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index 55dcfd409..da7e34bb4 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -56,6 +56,7 @@ open_docs/2, open_local_docs/2, read_doc_body/2, + read_doc_bodies/2, load_purge_infos/2, serialize_doc/2, @@ -404,6 +405,15 @@ read_doc_body(#st{} = St, #doc{} = Doc) -> atts = Atts }. +read_doc_bodies(#st{} = St, Docs) when is_list(Docs) -> + Ptrs = [Ptr || #doc{body = Ptr} <- Docs], + {ok, Terms} = couch_file:pread_terms(St#st.fd, Ptrs), + lists:zipwith( + fun(#doc{} = Doc, {Body, Atts}) -> Doc#doc{body = Body, atts = Atts} end, + Docs, + Terms + ). + load_purge_infos(St, UUIDs) -> Results = couch_btree:lookup(St#st.purge_tree, UUIDs), lists:map( diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index 131b58c96..438d4a3c5 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -138,6 +138,7 @@ validate_dbname/1, make_doc/5, + make_docs/2, new_revid/1 ]). @@ -2017,14 +2018,28 @@ make_doc(_Db, Id, Deleted, nil = _Bp, RevisionPath) -> atts = [], deleted = Deleted }; -make_doc(#db{} = Db, Id, Deleted, Bp, {Pos, Revs}) -> +make_doc(#db{} = Db, Id, Deleted, Bp, RevisionPath) -> + [Doc] = make_docs(Db, [{Id, Deleted, Bp, RevisionPath}]), + Doc. + +% Specs is [{Id, Deleted, BodyPtr, RevPath}], every spec with a body pointer +make_docs(#db{} = Db, Specs) when is_list(Specs) -> RevsLimit = get_revs_limit(Db), - Doc0 = couch_db_engine:read_doc_body(Db, #doc{ - id = Id, - revs = {Pos, lists:sublist(Revs, 1, RevsLimit)}, - body = Bp, - deleted = Deleted - }), + Docs0 = lists:map( + fun({Id, Del, Bp, {Pos, Revs}}) -> + #doc{ + id = Id, + revs = {Pos, lists:sublist(Revs, 1, RevsLimit)}, + body = Bp, + deleted = Del + } + end, + Specs + ), + Docs1 = couch_db_engine:read_doc_bodies(Db, Docs0), + [doc_atts_and_opts(Db, Doc) || Doc <- Docs1]. + +doc_atts_and_opts(#db{} = Db, #doc{} = Doc0) -> Doc1 = case Doc0#doc.atts of BinAtts when is_binary(BinAtts) -> diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl index d83e0ed9d..91f5a9859 100644 --- a/src/couch/src/couch_db_engine.erl +++ b/src/couch/src/couch_db_engine.erl @@ -345,6 +345,11 @@ -callback read_doc_body(DbHandle :: db_handle(), RawDoc :: doc()) -> doc(). +% Batch version of read_doc_body/2. Returns the list of #doc{} records +% in the same order as the RawDocs specs +-callback read_doc_bodies(DbHandle :: db_handle(), RawDocs :: [doc()]) -> + [doc()]. + % This function will be called from many contexts concurrently. % If the storage engine has a purge_info() record for any of the % provided UUIDs, those purge_info() records should be returned. The @@ -707,6 +712,7 @@ open_docs/2, open_local_docs/2, read_doc_body/2, + read_doc_bodies/2, load_purge_infos/2, serialize_doc/2, @@ -906,6 +912,10 @@ read_doc_body(#db{} = Db, RawDoc) -> #db{engine = {Engine, EngineState}} = Db, Engine:read_doc_body(EngineState, RawDoc). +read_doc_bodies(#db{} = Db, RawDocs) when is_list(RawDocs) -> + #db{engine = {Engine, EngineState}} = Db, + Engine:read_doc_bodies(EngineState, RawDocs). + load_purge_infos(#db{} = Db, UUIDs) -> #db{engine = {Engine, EngineState}} = Db, Engine:load_purge_infos(EngineState, UUIDs). diff --git a/src/couch/test/eunit/couch_db_doc_tests.erl b/src/couch/test/eunit/couch_db_doc_tests.erl index 75005df76..58b2910e7 100644 --- a/src/couch/test/eunit/couch_db_doc_tests.erl +++ b/src/couch/test/eunit/couch_db_doc_tests.erl @@ -40,24 +40,25 @@ couch_db_doc_test_() -> fun setup/0, fun teardown/1, [ - fun should_truncate_number_of_revisions/1, - fun should_raise_bad_request_on_invalid_rev/1, - fun should_allow_access_in_doc_keys_test/1 + ?TDEF_FE(t_truncate_number_of_revisions), + ?TDEF_FE(t_raise_bad_request_on_invalid_rev), + ?TDEF_FE(t_allow_access_in_doc_keys), + ?TDEF_FE(t_make_docs_in_batch) ] } } }. -should_truncate_number_of_revisions(DbName) -> +t_truncate_number_of_revisions(DbName) -> DocId = <<"foo">>, Db = open_db(DbName), couch_db:set_revs_limit(Db, 5), Rev = create_doc(Db, DocId), Rev10 = add_revisions(Db, DocId, Rev, 10), {ok, [{ok, #doc{revs = {11, Revs}}}]} = open_doc_rev(Db, DocId, Rev10), - ?_assertEqual(5, length(Revs)). + ?assertEqual(5, length(Revs)). -should_raise_bad_request_on_invalid_rev(DbName) -> +t_raise_bad_request_on_invalid_rev(DbName) -> DocId = <<"foo">>, InvalidRev1 = <<"foo">>, InvalidRev2 = <<"a-foo">>, @@ -65,25 +66,86 @@ should_raise_bad_request_on_invalid_rev(DbName) -> Expect = {bad_request, <<"Invalid rev format">>}, Db = open_db(DbName), create_doc(Db, DocId), - [ - {InvalidRev1, ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev1, 1))}, - {InvalidRev2, ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev2, 1))}, - {InvalidRev3, ?_assertThrow(Expect, add_revisions(Db, DocId, InvalidRev3, 1))} - ]. + ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev1, 1)), + ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev2, 1)), + ?assertThrow(Expect, add_revisions(Db, DocId, InvalidRev3, 1)). -should_allow_access_in_doc_keys_test(_DbName) -> +t_allow_access_in_doc_keys(_DbName) -> Json = <<"{\"_id\":\"foo\",\"_access\":[\"test\"]}">>, EJson = couch_util:json_decode(Json), Expected = {[{<<"_id">>, <<"foo">>}, {<<"_access">>, [<<"test">>]}]}, EJson = Expected, Doc = couch_doc:from_json_obj(EJson), NewEJson = couch_doc:to_json_obj(Doc, []), - ?_assertEqual(NewEJson, Expected). + ?assertEqual(NewEJson, Expected). + +t_make_docs_in_batch(DbName) -> + Db0 = open_db(DbName), + Rev = create_doc(Db0, ~"somedoc"), + add_revisions(Db0, ~"somedoc", Rev, 3), + del_doc(Db0, ~"deleted", create_doc(Db0, ~"deleted")), + create_att_doc(Db0, ~"att"), + {ok, Db} = couch_db:reopen(Db0), + Ids = [~"somedoc", ~"deleted", ~"att"], + Specs = doc_specs(Db, Ids), + ?assertEqual(3, length(Specs)), + + Expected = [ + couch_db:make_doc(Db, Id, Del, Ptr, Path) + || {Id, Del, Ptr, Path} <- Specs + ], + ?assertEqual(Expected, couch_db:make_docs(Db, Specs)), + ?assertEqual(lists:reverse(Expected), couch_db:make_docs(Db, lists:reverse(Specs))), + ?assertEqual([], couch_db:make_docs(Db, [])), + + % And the docs are the ones written + [#doc{revs = {4, RevIds}}] = couch_db:make_docs(Db, doc_specs(Db, [~"somedoc"])), + ?assertEqual(4, length(RevIds)), + [#doc{deleted = true}] = couch_db:make_docs(Db, doc_specs(Db, [~"deleted"])), + [#doc{atts = [Att]}] = couch_db:make_docs(Db, doc_specs(Db, [~"att"])), + ?assertEqual(~"a.txt", couch_att:fetch(name, Att)). open_db(DbName) -> {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), Db. +doc_specs(Db, Ids) -> + lists:flatmap( + fun(Id) -> + #full_doc_info{rev_tree = Tree} = couch_db:get_full_doc_info(Db, Id), + Leafs = couch_key_tree:get_all_leafs(Tree), + [{Id, Del, Ptr, Path} || {#leaf{deleted = Del, ptr = Ptr}, Path} <- Leafs] + end, + Ids + ). + +del_doc(Db0, DocId, Rev) -> + {ok, Db} = couch_db:reopen(Db0), + {Pos, RevId} = couch_doc:parse_rev(Rev), + Doc = #doc{id = DocId, revs = {Pos, [RevId]}, deleted = true, body = {[]}}, + {ok, _} = couch_db:update_doc(Db, Doc, []). + +create_att_doc(Db0, DocId) -> + {ok, Db} = couch_db:reopen(Db0), + Doc = couch_doc:from_json_obj( + {[ + {~"_id", DocId}, + { + ~"_attachments", + {[ + { + ~"a.txt", + {[ + {~"content_type", ~"text/plain"}, + {~"data", base64:encode(~"xkcd")} + ]} + } + ]} + } + ]} + ), + {ok, _} = couch_db:update_doc(Db, Doc, []). + create_doc(Db, DocId) -> add_revision(Db, DocId, undefined). diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl index 55ac938cd..659adf275 100644 --- a/src/mem3/src/mem3_rep.erl +++ b/src/mem3/src/mem3_rep.erl @@ -844,19 +844,15 @@ chunk_revs([{Id, R, A} | Revs], {Count, Chunk}, Chunks, Limit) -> open_docs(Db, Infos, Missing) -> InfoMap = #{Id => FDI || #full_doc_info{id = Id} = FDI <- Infos}, - lists:flatmap( + Specs = lists:flatmap( fun({Id, Revs, _}) -> - #full_doc_info{rev_tree = RevTree} = map_get(Id, InfoMap), - {FoundRevs, _} = couch_key_tree:get_key_leafs(RevTree, Revs), - lists:map( - fun({#leaf{deleted = IsDel, ptr = SummaryPtr}, FoundRevPath}) -> - couch_db:make_doc(Db, Id, IsDel, SummaryPtr, FoundRevPath) - end, - FoundRevs - ) + #full_doc_info{rev_tree = Tree} = map_get(Id, InfoMap), + {Revs1, _} = couch_key_tree:get_key_leafs(Tree, Revs), + [{Id, Del, Ptr, Path} || {#leaf{deleted = Del, ptr = Ptr}, Path} <- Revs1] end, Missing - ). + ), + couch_db:make_docs(Db, Specs). save_on_target(Node, Name, Docs, Timeout) -> mem3_rpc:update_docs(
