This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 14b319ef1 Replicator support for heartbeat_seq change feed parameter
14b319ef1 is described below
commit 14b319ef12ef05044b0d48c92a743900c6322133
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Oct 21 14:38:43 2026 -0400
Replicator support for heartbeat_seq change feed parameter
Implement heartbeat_seq handling in the replicator as a follow-up to #6165.
Take advantage of the fact that unknown parameters are ignored by
endpoints, so
we always pass `heartbeat_seq=true`, and if we get any heartbeat rows we'll
use
them for checkpointing, and if we don't (an old source endpoint) we'll
function
as before.
Issue: #6120
---
.../src/couch_replicator_api_wrap.erl | 42 ++++++++++++++---
.../src/couch_replicator_changes_reader.erl | 54 +++++++++++++++++++---
src/docs/src/replication/protocol.rst | 6 +++
3 files changed, 90 insertions(+), 12 deletions(-)
diff --git a/src/couch_replicator/src/couch_replicator_api_wrap.erl
b/src/couch_replicator/src/couch_replicator_api_wrap.erl
index c1ab912d4..57e15d2c4 100644
--- a/src/couch_replicator/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator/src/couch_replicator_api_wrap.erl
@@ -547,12 +547,7 @@ changes_since(
Timeout = max(1000, InactiveTimeout div 3),
EncodedSeq = couch_replicator_utils:seq_encode(StartSeq),
BaseQArgs =
- case get_value(continuous, Options, false) of
- false ->
- [{"feed", "normal"}];
- true ->
- [{"feed", "continuous"}]
- end ++
+ changes_feed_qargs(Options) ++
[
{"style", atom_to_list(Style)},
{"since", EncodedSeq},
@@ -669,6 +664,17 @@ maybe_add_changes_filter_q_args(BaseQS, Options) ->
]
end.
+changes_feed_qargs(Options) ->
+ Feed =
+ case get_value(continuous, Options, false) of
+ false -> "normal";
+ true -> "continuous"
+ end,
+ % If the source supports heartbeat_seq=true we'll get and use {seq: ...,
+ % id: null, changes: []} heartbeat rows for checkpointing. If the source
+ % doesn't support it, it will ignore the parameter.
+ [{"feed", Feed}, {"heartbeat_seq", "true"}].
+
parse_changes_feed(Options, UserFun, DataStreamFun) ->
case get_value(continuous, Options, false) of
true ->
@@ -961,6 +967,9 @@ json_to_doc_info({Props}) ->
case get_value(<<"changes">>, Props) of
undefined ->
{last_seq, get_value(<<"last_seq">>, Props)};
+ [] ->
+ % Got a heartbeat row from heartbeat_seq=true param
+ {heartbeat_seq, get_value(~"seq", Props)};
Changes ->
RevsInfo0 = lists:map(
fun({Change}) ->
@@ -1131,4 +1140,25 @@ normalize_http_db_test() ->
?assertEqual(Expected, normalize_db(HttpDb)),
?assertEqual(<<"local">>, normalize_db(<<"local">>)).
+json_to_doc_info_test() ->
+ Seq = ~"42-abc",
+ ?assertEqual({last_seq, Seq}, json_to_doc_info({[{~"last_seq", Seq}]})),
+ ?assertEqual({last_seq, undefined}, json_to_doc_info({[{~"pending", 0}]})),
+ HB = {[{~"seq", Seq}, {~"id", null}, {~"changes", []}]},
+ ?assertEqual({heartbeat_seq, Seq}, json_to_doc_info(HB)),
+ Row =
+ {[
+ {~"seq", Seq},
+ {~"id", ~"doc1"},
+ {~"changes", [{[{~"rev", ~"1-abc"}]}]}
+ ]},
+ ?assertEqual(
+ #doc_info{
+ id = ~"doc1",
+ high_seq = Seq,
+ revs = [#rev_info{rev = {1, ~"abc"}}]
+ },
+ json_to_doc_info(Row)
+ ).
+
-endif.
diff --git a/src/couch_replicator/src/couch_replicator_changes_reader.erl
b/src/couch_replicator/src/couch_replicator_changes_reader.erl
index bb6733608..7ee121ced 100644
--- a/src/couch_replicator/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator/src/couch_replicator_changes_reader.erl
@@ -132,18 +132,31 @@ process_change({last_seq, LS}, {_Parent, _, ChangesQueue,
true = _Continuous}) -
undefined -> get(last_seq);
_ -> LS
end,
- OldSeq = get(last_seq),
- if
- Seq == OldSeq -> ok;
- true -> ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq})
- end,
- put(last_seq, Seq),
+ queue_last_seq(ChangesQueue, Seq),
throw(recurse);
process_change({last_seq, _}, _) ->
% This clause is unreachable today, but let's plan ahead for the future
% where we checkpoint against last_seq instead of the sequence of the last
% change. The two can differ substantially in the case of a restrictive
% filter.
+ ok;
+process_change({heartbeat_seq, Seq}, {_Parent, _, ChangesQueue, _Continuous})
when
+ is_binary(Seq); is_integer(Seq)
+->
+ queue_last_seq(ChangesQueue, Seq);
+process_change({heartbeat_seq, _}, _) ->
+ % We're being defensive here since we're in the replicator, and have to
+ % handle other, possibly non-ASF CouchDB implementations, or strange
+ % proxies. Here, we'd catch heartbeat objects without a seq or seq set to
+ % null, for example and we ignore them instead of crashing everything.
+ ok.
+
+queue_last_seq(ChangesQueue, Seq) ->
+ case get(last_seq) of
+ Seq -> ok;
+ _ -> ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq})
+ end,
+ put(last_seq, Seq),
ok.
is_doc_id_too_long(IdLength) ->
@@ -154,3 +167,32 @@ is_doc_id_too_long(IdLength) ->
ConfigMax = list_to_integer(ConfigMaxStr),
ConfigMax > 0 andalso IdLength > ConfigMax
end.
+
+-ifdef(TEST).
+
+-include_lib("eunit/include/eunit.hrl").
+
+heartbeat_seq_test() ->
+ {ok, Q} = couch_work_queue:new([{max_size, 100}, {max_items, 100}]),
+ Ctx = {self(), #httpdb{}, Q, true},
+ put(last_seq, ~"1-a"),
+ % new sequence => last_seq
+ ?assertEqual(ok, process_change({heartbeat_seq, ~"2-b"}, Ctx)),
+ ?assertEqual(~"2-b", get(last_seq)),
+ ?assertEqual({ok, [{last_seq, ~"2-b"}]}, couch_work_queue:dequeue(Q)),
+ % dedup works
+ ?assertEqual(ok, process_change({heartbeat_seq, ~"2-b"}, Ctx)),
+ ?assertEqual(0, couch_work_queue:item_count(Q)),
+ % new seq is queued
+ ?assertEqual(ok, process_change({heartbeat_seq, ~"4-d"}, Ctx)),
+ ?assertEqual({ok, [{last_seq, ~"4-d"}]}, couch_work_queue:dequeue(Q)),
+ ?assertEqual(~"4-d", get(last_seq)),
+ % old replicators had integer sequences, we're maximally backwards
+ % compatible so test them too
+ ?assertEqual(ok, process_change({heartbeat_seq, 5}, Ctx)),
+ ?assertEqual({ok, [{last_seq, 5}]}, couch_work_queue:dequeue(Q)),
+ ?assertEqual(5, get(last_seq)),
+ erase(last_seq),
+ couch_work_queue:close(Q).
+
+-endif.
diff --git a/src/docs/src/replication/protocol.rst
b/src/docs/src/replication/protocol.rst
index f361bc334..902af38bc 100644
--- a/src/docs/src/replication/protocol.rst
+++ b/src/docs/src/replication/protocol.rst
@@ -766,6 +766,12 @@ request. This request MUST be made with the following
query parameters:
period in *milliseconds*. The RECOMMENDED value by default is ``10000``
(10 seconds).
+- The ``heartbeat_seq=true`` parameter MAY be used so that heartbeats will be
+ emitted as rows of the form ``{"seq": ..., "id": null, "changes": []}``.
+ Replicator SHOULD use its sequence to record a Checkpoint. This advances
+ Checkpoints when a filter rejects the Changes for a long time. A Source which
+ does not support the parameter SHOULD ignore it.
+
- If a startup Checkpoint was found during the Replication Logs comparison,
the ``since`` query parameter MUST be passed with this value.
In case of Full Replication it MAY be ``0`` (number zero) or