This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new e273f702b Handle cases when dreyfus checkpoint is out-of-sync with the
index
e273f702b is described below
commit e273f702bf96be540d2ef4b9fcd02a2d680c4007
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Mar 12 17:24:15 2026 -0400
Handle cases when dreyfus checkpoint is out-of-sync with the index
Currently, there are two places where the index purge seq is tracked: in the
index and in the db local doc checkpoints. Purge sequence folding should
never
start below the value in the checkpoint document as that could raise an
`invalid_start_purge_seq`. Normally both sequences should match, but if they
don't try to be explicit about what should happen:
* Index pseq > checkpoint pseq. Index somehow got ahead of the checkpoint.
Use
the checkpoint seq and re-process some purges through the index. This will
do
extra work but should be safe.
* Index pseq < checkpoint pseq. Index somehow got behind the checkpoint
and it
looks like it could have skipped purges. For views we reset the index, and
arguably that's the most correct solution. However, we never really had a
reset facility for clouseau, so instead choose to emit an error log and let
the user intervene manually but otherwise keep updating the index.
When updating the purge sequence in clouseau, save an rpc call if we're not
advancing clouseau's purge sequence. Clouseau as of recently already has a
check to return `ok` right away if new purge_seq is somehow less or equal to
the current one, but it's still nice not have to do an extra round-trip.
It was a bit surprising to discover that we had a bunch of nice dreyfus
eunit purge
tests around but they never actually ran. The test functions there were not
discoverably by EUnit. Switching them to be discoverable still wouldn't
work as
the test suite would need clouseau running during EUnit tests. Since we
don't
really have a framework for that, let's switch them to Elixir test and run
them
alongside other search tests.
---
src/dreyfus/src/dreyfus_index_updater.erl | 56 +-
src/dreyfus/test/eunit/dreyfus_purge_test.erl | 1119 -------------------------
test/elixir/test/config/search.elixir | 17 +
test/elixir/test/dreyfus_purge_test.exs | 474 +++++++++++
4 files changed, 537 insertions(+), 1129 deletions(-)
diff --git a/src/dreyfus/src/dreyfus_index_updater.erl
b/src/dreyfus/src/dreyfus_index_updater.erl
index 387ab09e2..54f79df82 100644
--- a/src/dreyfus/src/dreyfus_index_updater.erl
+++ b/src/dreyfus/src/dreyfus_index_updater.erl
@@ -30,8 +30,12 @@ update(IndexPid, Index) ->
erlang:put(io_priority, {search, DbName, IndexName}),
{ok, Db} = couch_db:open_int(DbName, []),
try
+ CheckpointPSeq = get_local_doc_purge_seq(Db, Index),
+ {ok, ClouseauPSeq} = clouseau_rpc:get_purge_seq(IndexPid),
+ IdxPurgeSeq = get_index_purge_seq(Db, CheckpointPSeq, ClouseauPSeq,
DDocId, IndexName),
+ DbPurgeSeq = couch_db:get_purge_seq(Db),
+ TotalPurgeChanges = DbPurgeSeq - IdxPurgeSeq,
TotalUpdateChanges = couch_db:count_changes_since(Db, CurSeq),
- TotalPurgeChanges = count_pending_purged_docs_since(Db, IndexPid),
TotalChanges = TotalUpdateChanges + TotalPurgeChanges,
couch_task_status:add_task([
@@ -49,7 +53,7 @@ update(IndexPid, Index) ->
%ExcludeIdRevs is [{Id1, Rev1}, {Id2, Rev2}, ...]
%The Rev is the final Rev, not purged Rev.
- {ok, ExcludeIdRevs} = purge_index(Db, IndexPid, Index),
+ {ok, ExcludeIdRevs} = purge_index(Db, IndexPid, Index, IdxPurgeSeq,
ClouseauPSeq),
%% compute on all docs modified since we last computed.
NewCurSeq = couch_db:get_update_seq(Db),
@@ -87,8 +91,7 @@ load_docs(FDI, {I, IndexPid, Db, Proc, Total, LastCommitTime,
ExcludeIdRevs} = A
{ok, setelement(1, Acc, I + 1)}
end.
-purge_index(Db, IndexPid, Index) ->
- {ok, IdxPurgeSeq} = clouseau_rpc:get_purge_seq(IndexPid),
+purge_index(Db, IndexPid, Index, IdxPurgeSeq, OldClouseauPSeq) ->
Proc = get_os_process(Index#index.def_lang),
try
true = proc_prompt(Proc, [<<"add_fun">>, Index#index.def]),
@@ -113,18 +116,19 @@ purge_index(Db, IndexPid, Index) ->
end,
{ok, ExcludeList} = couch_db:fold_purge_infos(Db, IdxPurgeSeq,
FoldFun, []),
NewPurgeSeq = couch_db:get_purge_seq(Db),
- ok = clouseau_rpc:set_purge_seq(IndexPid, NewPurgeSeq),
+ case NewPurgeSeq > OldClouseauPSeq of
+ true ->
+ ok = clouseau_rpc:set_purge_seq(IndexPid, NewPurgeSeq);
+ false ->
+ % Save an rpc call if we aren't advancing the purge sequence
+ ok
+ end,
update_local_doc(Db, Index, NewPurgeSeq),
{ok, ExcludeList}
after
ret_os_process(Proc)
end.
-count_pending_purged_docs_since(Db, IndexPid) ->
- DbPurgeSeq = couch_db:get_purge_seq(Db),
- {ok, IdxPurgeSeq} = clouseau_rpc:get_purge_seq(IndexPid),
- DbPurgeSeq - IdxPurgeSeq.
-
update_or_delete_index(IndexPid, Db, DI, Proc) ->
#doc_info{id = Id, revs = [#rev_info{deleted = Del} | _]} = DI,
case Del of
@@ -152,6 +156,38 @@ update_local_doc(Db, Index, PurgeSeq) ->
DocContent = dreyfus_util:get_local_purge_doc_body(Db, DocId, PurgeSeq,
Index),
couch_db:update_doc(Db, DocContent, []).
+get_local_doc_purge_seq(Db, Index) ->
+ DocId = dreyfus_util:get_local_purge_doc_id(Index#index.sig),
+ % We're implicitly asserting this purge checkpoint doc exists. It is
+ % created either on open or during compaction in on_compact handler.
+ {ok, #doc{body = {[_ | _] = Props}}} = couch_db:open_doc(Db, DocId),
+ couch_util:get_value(<<"purge_seq">>, Props).
+
+get_index_purge_seq(Db, CheckpointPSeq, ClouseauPSeq, DDocId, IndexName) when
+ is_integer(CheckpointPSeq), is_integer(ClouseauPSeq), CheckpointPSeq >= 0,
ClouseauPSeq >= 0
+->
+ if
+ CheckpointPSeq == ClouseauPSeq ->
+ % The default state is they both match
+ CheckpointPSeq;
+ CheckpointPSeq > ClouseauPSeq ->
+ % Somehow index fell behind. We should reset the index but don't
really
+ % have a facility for it, so log an error instead. We still can
only fold
+ % purges start from checkpoint sequence and higher (and not lower).
+ DbName = couch_db:name(Db),
+ Msg = "~p : index pseq:~p is behind the checkpoint pseq:~p db:~p
ddoc:~p index:~p",
+ couch_log:error(Msg, [?MODULE, ClouseauPSeq, CheckpointPSeq,
DbName, DDocId, IndexName]),
+ CheckpointPSeq;
+ CheckpointPSeq < ClouseauPSeq ->
+ % Somehow the checkpoint fell behind. Perhaps someone manually
+ % manipulated checkpoint docs, or the the system crashed right
+ % after the set_purge_seq was called but before the checkpoint doc
+ % was written. Choose to reprocess the changes from the
+ % checkpointed sequence, it may add extra work, but should still be
+ % correct.
+ CheckpointPSeq
+ end.
+
update_task(NumChanges) ->
[Changes, Total] = couch_task_status:get([changes_done, total_changes]),
Changes2 = Changes + NumChanges,
diff --git a/src/dreyfus/test/eunit/dreyfus_purge_test.erl
b/src/dreyfus/test/eunit/dreyfus_purge_test.erl
deleted file mode 100644
index a7c0068e0..000000000
--- a/src/dreyfus/test/eunit/dreyfus_purge_test.erl
+++ /dev/null
@@ -1,1119 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(dreyfus_purge_test).
-
--export([
- test_purge_single/0,
- test_purge_multiple/0,
- test_purge_multiple2/0,
- test_purge_conflict/0,
- test_purge_conflict2/0,
- test_purge_conflict3/0,
- test_purge_conflict4/0,
- test_purge_update/0,
- test_purge_update2/0,
- test_delete/0,
- test_delete_purge_conflict/0,
- test_delete_conflict/0,
- test_all/0
-]).
-
--export([
- test_verify_index_exists1/0,
- test_verify_index_exists2/0,
- test_verify_index_exists_failed/0,
- test_local_doc/0,
- test_delete_local_doc/0,
- test_purge_search/0
-]).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("couch/include/couch_db.hrl").
--include_lib("dreyfus/include/dreyfus.hrl").
--include_lib("couch/include/couch_eunit.hrl").
--include_lib("mem3/include/mem3.hrl").
-
-test_all() ->
- test_purge_single(),
- test_purge_multiple(),
- test_purge_multiple2(),
- test_purge_conflict(),
- test_purge_conflict2(),
- test_purge_conflict3(),
- test_purge_conflict4(),
- test_purge_update(),
- test_purge_update2(),
- test_delete(),
- test_delete_purge_conflict(),
- test_delete_conflict(),
- test_verify_index_exists1(),
- test_verify_index_exists2(),
- test_verify_index_exists_failed(),
- test_delete_local_doc(),
- test_local_doc(),
- test_purge_search(),
- ok.
-
-test_purge_single() ->
- DbName = db_name(),
- create_db_docs(DbName),
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount1, 1),
- purge_docs(DbName, [<<"apple">>]),
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount2, 0),
- delete_db(DbName),
- ok.
-
-test_purge_multiple() ->
- Query = <<"color:red">>,
-
- %create the db and docs
- DbName = db_name(),
- create_db_docs(DbName),
-
- %first search request
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, Query),
-
- ?assertEqual(HitCount1, 5),
-
- %purge 5 docs
- purge_docs(DbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %second search request
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, Query),
-
- ?assertEqual(HitCount2, 0),
-
- %delete the db
- delete_db(DbName),
- ok.
-
-test_purge_multiple2() ->
- %create the db and docs
- DbName = db_name(),
- create_db_docs(DbName),
-
- Query = <<"color:red">>,
-
- %first search request
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, Query),
-
- ?assertEqual(HitCount1, 5),
-
- %purge 2 docs
- purge_docs(DbName, [<<"apple">>, <<"tomato">>]),
-
- %second search request
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, Query),
-
- ?assertEqual(HitCount2, 3),
-
- %purge 2 docs
- purge_docs(DbName, [<<"cherry">>, <<"haw">>]),
-
- %third search request
- {ok, _, HitCount3, _, _, _} = dreyfus_search(DbName, Query),
-
- ?assertEqual(HitCount3, 1),
-
- %delete the db
- delete_db(DbName),
- ok.
-
-test_purge_conflict() ->
- %create dbs and docs
- SourceDbName = db_name(),
- timer:sleep(2000),
- TargetDbName = db_name(),
-
- create_db_docs(SourceDbName),
- create_db_docs(TargetDbName, <<"green">>),
-
- %first search
- {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount1 + GreenHitCount1),
-
- %do replicate and make conflicted docs
- {ok, _} = fabric:update_doc(
- <<"_replicator">>,
- make_replicate_doc(
- SourceDbName, TargetDbName
- ),
- [?ADMIN_CTX]
- ),
-
- %%check doc version
- wait_for_replicate(
- TargetDbName,
- [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ],
- 2,
- 5
- ),
-
- %second search
- {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount2 + GreenHitCount2),
-
- purge_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %third search
- {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search(
- TargetDbName,
- <<"color:red">>
- ),
- {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search(
- TargetDbName,
- <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount3 + GreenHitCount3),
- ?assertEqual(RedHitCount2, GreenHitCount3),
- ?assertEqual(GreenHitCount2, RedHitCount3),
-
- delete_db(SourceDbName),
- delete_db(TargetDbName),
- ok.
-
-test_purge_conflict2() ->
- %create dbs and docs
- SourceDbName = db_name(),
- timer:sleep(2000),
- TargetDbName = db_name(),
-
- create_db_docs(SourceDbName),
- create_db_docs(TargetDbName, <<"green">>),
-
- %first search
- {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search(
- TargetDbName,
- <<"color:red">>
- ),
- {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search(
- TargetDbName,
- <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount1 + GreenHitCount1),
-
- %do replicate and make conflicted docs
- {ok, _} = fabric:update_doc(
- <<"_replicator">>,
- make_replicate_doc(
- SourceDbName, TargetDbName
- ),
- [?ADMIN_CTX]
- ),
-
- wait_for_replicate(
- TargetDbName,
- [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ],
- 2,
- 5
- ),
-
- %second search
- {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount2 + GreenHitCount2),
-
- purge_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
- purge_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %third search
- {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(0, RedHitCount3 + GreenHitCount3),
-
- delete_db(SourceDbName),
- delete_db(TargetDbName),
- ok.
-
-test_purge_conflict3() ->
- %create dbs and docs
- SourceDbName = db_name(),
- timer:sleep(2000),
- TargetDbName = db_name(),
-
- create_db_docs(SourceDbName),
- create_db_docs(TargetDbName, <<"green">>),
-
- %first search
- {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount1 + GreenHitCount1),
-
- %do replicate and make conflicted docs
- {ok, _} = fabric:update_doc(
- <<"_replicator">>,
- make_replicate_doc(
- SourceDbName, TargetDbName
- ),
- [?ADMIN_CTX]
- ),
-
- %%check doc version
- wait_for_replicate(
- TargetDbName,
- [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ],
- 2,
- 5
- ),
-
- %second search
- {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount2 + GreenHitCount2),
-
- purge_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %third search
- {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount3 + GreenHitCount3),
- ?assertEqual(RedHitCount2, GreenHitCount3),
- ?assertEqual(GreenHitCount2, RedHitCount3),
-
- purge_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
- {ok, _, RedHitCount4, _, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount4, _, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(0, RedHitCount4 + GreenHitCount4),
-
- delete_db(SourceDbName),
- delete_db(TargetDbName),
- ok.
-
-test_purge_conflict4() ->
- %create dbs and docs
- SourceDbName = db_name(),
- timer:sleep(2000),
- TargetDbName = db_name(),
-
- create_db_docs(SourceDbName, <<"green">>),
- create_db_docs(TargetDbName, <<"red">>),
-
- %first search
- {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount1 + GreenHitCount1),
-
- %do replicate and make conflicted docs
- {ok, _} = fabric:update_doc(
- <<"_replicator">>,
- make_replicate_doc(
- SourceDbName, TargetDbName
- ),
- [?ADMIN_CTX]
- ),
-
- %%check doc version
- wait_for_replicate(
- TargetDbName,
- [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ],
- 2,
- 5
- ),
-
- %second search
- {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount2 + GreenHitCount2),
-
- purge_docs_with_all_revs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %third search
- {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(0, RedHitCount3 + GreenHitCount3),
-
- delete_db(SourceDbName),
- delete_db(TargetDbName),
- ok.
-
-test_purge_update() ->
- %create the db and docs
- DbName = db_name(),
- create_db_docs(DbName),
-
- QueryRed = <<"color:red">>,
- QueryGreen = <<"color:green">>,
-
- %first search request
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, QueryRed),
-
- ?assertEqual(HitCount1, 5),
-
- %update doc
- Rev = get_rev(DbName, <<"apple">>),
- Doc = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"apple">>},
- {<<"_rev">>, couch_doc:rev_to_str(Rev)},
- {<<"color">>, <<"green">>},
- {<<"size">>, 8}
- ]}
- ),
- {ok, _} = fabric:update_docs(DbName, [Doc], [?ADMIN_CTX]),
-
- %second search request
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, QueryRed),
- {ok, _, HitCount3, _, _, _} = dreyfus_search(DbName, QueryGreen),
-
- % 4 red and 1 green
- ?assertEqual(HitCount2, 4),
- ?assertEqual(HitCount3, 1),
-
- % purge 2 docs, 1 red and 1 green
- purge_docs(DbName, [<<"apple">>, <<"tomato">>]),
-
- % third search request
- {ok, _, HitCount4, _, _, _} = dreyfus_search(DbName, QueryRed),
- {ok, _, HitCount5, _, _, _} = dreyfus_search(DbName, QueryGreen),
-
- % 3 red and 0 green
- ?assertEqual(HitCount4, 3),
- ?assertEqual(HitCount5, 0),
-
- delete_db(DbName),
- ok.
-
-test_purge_update2() ->
- %create the db and docs
- DbName = db_name(),
- create_db_docs(DbName),
-
- Query1 = <<"size:1">>,
- Query1000 = <<"size:1000">>,
-
- %first search request
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, Query1),
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, Query1000),
-
- ?assertEqual(HitCount1, 5),
- ?assertEqual(HitCount2, 0),
-
- %update doc 999 times, it will take about 30 seconds.
- update_doc(DbName, <<"apple">>, 999),
-
- %second search request
- {ok, _, HitCount3, _, _, _} = dreyfus_search(DbName, Query1),
- {ok, _, HitCount4, _, _, _} = dreyfus_search(DbName, Query1000),
-
- % 4 value(1) and 1 value(1000)
- ?assertEqual(HitCount3, 4),
- ?assertEqual(HitCount4, 1),
-
- % purge doc
- purge_docs(DbName, [<<"apple">>]),
-
- % third search request
- {ok, _, HitCount5, _, _, _} = dreyfus_search(DbName, Query1),
- {ok, _, HitCount6, _, _, _} = dreyfus_search(DbName, Query1000),
-
- % 4 value(1) and 0 value(1000)
- ?assertEqual(HitCount5, 4),
- ?assertEqual(HitCount6, 0),
-
- delete_db(DbName),
- ok.
-
-test_delete() ->
- DbName = db_name(),
- create_db_docs(DbName),
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount1, 1),
- ok = delete_docs(DbName, [<<"apple">>]),
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount2, 0),
- delete_db(DbName),
- ok.
-
-test_delete_conflict() ->
- %create dbs and docs
- SourceDbName = db_name(),
- timer:sleep(2000),
- TargetDbName = db_name(),
-
- create_db_docs(SourceDbName),
- create_db_docs(TargetDbName, <<"green">>),
-
- %first search
- {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount1 + GreenHitCount1),
-
- %do replicate and make conflicted docs
- {ok, _} = fabric:update_doc(
- <<"_replicator">>,
- make_replicate_doc(
- SourceDbName, TargetDbName
- ),
- [?ADMIN_CTX]
- ),
-
- wait_for_replicate(
- TargetDbName,
- [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ],
- 2,
- 5
- ),
-
- %second search
- {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount2 + GreenHitCount2),
-
- %delete docs
- delete_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %third search
- {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount3 + GreenHitCount3),
- ?assertEqual(RedHitCount2, GreenHitCount3),
- ?assertEqual(GreenHitCount2, RedHitCount3),
-
- delete_db(SourceDbName),
- delete_db(TargetDbName),
- ok.
-
-test_delete_purge_conflict() ->
- %create dbs and docs
- SourceDbName = db_name(),
- timer:sleep(2000),
- TargetDbName = db_name(),
-
- create_db_docs(SourceDbName),
- create_db_docs(TargetDbName, <<"green">>),
-
- %first search
- {ok, _, RedHitCount1, _RedHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount1, _GreenHits1, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount1 + GreenHitCount1),
-
- %do replicate and make conflicted docs
- {ok, _} = fabric:update_doc(
- <<"_replicator">>,
- make_replicate_doc(
- SourceDbName, TargetDbName
- ),
- [?ADMIN_CTX]
- ),
-
- wait_for_replicate(
- TargetDbName,
- [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ],
- 2,
- 5
- ),
-
- %second search
- {ok, _, RedHitCount2, _RedHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount2, _GreenHits2, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(5, RedHitCount2 + GreenHitCount2),
-
- %purge docs
- purge_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %delete docs
- delete_docs(TargetDbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"haw">>,
- <<"strawberry">>
- ]),
-
- %third search
- {ok, _, RedHitCount3, _RedHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:red">>
- ),
- {ok, _, GreenHitCount3, _GreenHits3, _, _} = dreyfus_search(
- TargetDbName, <<"color:green">>
- ),
-
- ?assertEqual(RedHitCount3, 0),
- ?assertEqual(GreenHitCount3, 0),
- ?assertEqual(GreenHitCount3, 0),
- ?assertEqual(RedHitCount3, 0),
-
- delete_db(SourceDbName),
- delete_db(TargetDbName),
- ok.
-
-test_local_doc() ->
- DbName = db_name(),
- create_db_docs(DbName),
-
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount1, 1),
- purge_docs(DbName, [
- <<"apple">>,
- <<"tomato">>,
- <<"cherry">>,
- <<"strawberry">>
- ]),
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount2, 0),
-
- %get local doc
- [Sig | _] = get_sigs(DbName),
- LocalId = dreyfus_util:get_local_purge_doc_id(Sig),
- LocalShards = mem3:local_shards(DbName),
- PurgeSeqs = lists:map(
- fun(Shard) ->
- {ok, Db} = couch_db:open_int(Shard#shard.name, [?ADMIN_CTX]),
- {ok, LDoc} = couch_db:open_doc(Db, LocalId, []),
- {Props} = couch_doc:to_json_obj(LDoc, []),
- dreyfus_util:get_value_from_options(<<"updated_on">>, Props),
- PurgeSeq = dreyfus_util:get_value_from_options(<<"purge_seq">>,
Props),
- Type = dreyfus_util:get_value_from_options(<<"type">>, Props),
- ?assertEqual(<<"dreyfus">>, Type),
- couch_db:close(Db),
- PurgeSeq
- end,
- LocalShards
- ),
- ?assertEqual(lists:sum(PurgeSeqs), 4),
-
- delete_db(DbName),
- ok.
-
-test_verify_index_exists1() ->
- DbName = db_name(),
- create_db_docs(DbName),
-
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount1, 1),
-
- ok = purge_docs(DbName, [<<"apple">>]),
-
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount2, 0),
-
- ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)],
- [ShardDbName | _Rest] = ShardNames,
- {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]),
- {ok, LDoc} = couch_db:open_doc(
- Db,
- dreyfus_util:get_local_purge_doc_id(
- <<"49e82c2a910b1046b55cc45ad058a7ee">>
- ),
- []
- ),
- #doc{body = {Props}} = LDoc,
- ?assertEqual(true, dreyfus_util:verify_index_exists(ShardDbName, Props)),
- delete_db(DbName),
- ok.
-
-test_verify_index_exists2() ->
- DbName = db_name(),
- create_db_docs(DbName),
-
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount1, 1),
-
- ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)],
- [ShardDbName | _Rest] = ShardNames,
- {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]),
- {ok, LDoc} = couch_db:open_doc(
- Db,
- dreyfus_util:get_local_purge_doc_id(
- <<"49e82c2a910b1046b55cc45ad058a7ee">>
- ),
- []
- ),
- #doc{body = {Props}} = LDoc,
- ?assertEqual(true, dreyfus_util:verify_index_exists(ShardDbName, Props)),
-
- delete_db(DbName),
- ok.
-
-test_verify_index_exists_failed() ->
- DbName = db_name(),
- create_db_docs(DbName),
-
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount1, 1),
-
- ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)],
- [ShardDbName | _Rest] = ShardNames,
- {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]),
- {ok, LDoc} = couch_db:open_doc(
- Db,
- dreyfus_util:get_local_purge_doc_id(
- <<"49e82c2a910b1046b55cc45ad058a7ee">>
- ),
- []
- ),
- #doc{body = {Options}} = LDoc,
- OptionsDbErr = [
- {<<"indexname">>, dreyfus_util:get_value_from_options(<<"indexname">>,
Options)},
- {<<"ddoc_id">>, dreyfus_util:get_value_from_options(<<"ddoc_id">>,
Options)},
- {<<"signature">>, dreyfus_util:get_value_from_options(<<"signature">>,
Options)}
- ],
- ?assertEqual(
- false,
- dreyfus_util:verify_index_exists(
- ShardDbName, OptionsDbErr
- )
- ),
-
- OptionsIdxErr = [
- {<<"indexname">>, <<"someindex">>},
- {<<"ddoc_id">>, dreyfus_util:get_value_from_options(<<"ddoc_id">>,
Options)},
- {<<"signature">>, dreyfus_util:get_value_from_options(<<"signature">>,
Options)}
- ],
- ?assertEqual(
- false,
- dreyfus_util:verify_index_exists(
- ShardDbName, OptionsIdxErr
- )
- ),
-
- OptionsDDocErr = [
- {<<"indexname">>, dreyfus_util:get_value_from_options(<<"indexname">>,
Options)},
- {<<"ddoc_id">>, <<"somedesigndoc">>},
- {<<"signature">>, dreyfus_util:get_value_from_options(<<"signature">>,
Options)}
- ],
- ?assertEqual(
- false,
- dreyfus_util:verify_index_exists(
- ShardDbName, OptionsDDocErr
- )
- ),
-
- OptionsSigErr = [
- {<<"indexname">>, dreyfus_util:get_value_from_options(<<"indexname">>,
Options)},
- {<<"ddoc_id">>, dreyfus_util:get_value_from_options(<<"ddoc_id">>,
Options)},
- {<<"signature">>, <<"12345678901234567890123456789012">>}
- ],
- ?assertEqual(
- false,
- dreyfus_util:verify_index_exists(
- ShardDbName, OptionsSigErr
- )
- ),
-
- delete_db(DbName),
- ok.
-
-test_delete_local_doc() ->
- DbName = db_name(),
- create_db_docs(DbName),
-
- {ok, _, HitCount1, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount1, 1),
-
- ok = purge_docs(DbName, [<<"apple">>]),
-
- {ok, _, HitCount2, _, _, _} = dreyfus_search(DbName, <<"apple">>),
- ?assertEqual(HitCount2, 0),
-
- LDocId = dreyfus_util:get_local_purge_doc_id(
- <<"49e82c2a910b1046b55cc45ad058a7ee">>
- ),
- ShardNames = [Sh || #shard{name = Sh} <- mem3:local_shards(DbName)],
- [ShardDbName | _Rest] = ShardNames,
- {ok, Db} = couch_db:open(ShardDbName, [?ADMIN_CTX]),
- {ok, _} = couch_db:open_doc(Db, LDocId, []),
-
- delete_docs(DbName, [<<"_design/search">>]),
- io:format("DbName ~p~n", [DbName]),
- ?debugFmt("Converting ... ~n~p~n", [DbName]),
-
- dreyfus_fabric_cleanup:go(DbName),
- {ok, Db2} = couch_db:open(ShardDbName, [?ADMIN_CTX]),
- {not_found, _} = couch_db:open_doc(Db2, LDocId, []),
-
- delete_db(DbName),
- ok.
-
-test_purge_search() ->
- DbName = db_name(),
- create_db_docs(DbName),
- purge_docs(DbName, [<<"apple">>, <<"tomato">>, <<"haw">>]),
- {ok, _, HitCount, _, _, _} = dreyfus_search(DbName, <<"color:red">>),
- ?assertEqual(HitCount, 2),
- delete_db(DbName),
- ok.
-
-%private API
-db_name() ->
- iolist_to_binary([
- "dreyfus-test-db-",
- [
- integer_to_list(I)
- || I <- [
- erlang:unique_integer([positive]),
- rand:uniform(10000)
- ]
- ]
- ]).
-
-purge_docs(DBName, DocIds) ->
- IdsRevs = [{DocId, [get_rev(DBName, DocId)]} || DocId <- DocIds],
- {ok, _} = fabric:purge_docs(DBName, IdsRevs, []),
- ok.
-
-purge_docs_with_all_revs(DBName, DocIds) ->
- IdsRevs = [{DocId, get_revs(DBName, DocId)} || DocId <- DocIds],
- {ok, _} = fabric:purge_docs(DBName, IdsRevs, []),
- ok.
-
-dreyfus_search(DbName, KeyWord) ->
- QueryArgs = #index_query_args{q = KeyWord},
- {ok, DDoc} = fabric:open_doc(DbName, <<"_design/search">>, []),
- dreyfus_fabric_search:go(DbName, DDoc, <<"index">>, QueryArgs).
-
-create_db_docs(DbName) ->
- create_db(DbName),
- create_docs(DbName, 5, <<"red">>).
-
-create_db_docs(DbName, Color) ->
- create_db(DbName),
- create_docs(DbName, 5, Color).
-
-create_docs(DbName, Count, Color) ->
- {ok, _} = fabric:update_docs(DbName, make_docs(Count, Color),
[?ADMIN_CTX]),
- {ok, _} = fabric:update_doc(DbName, make_design_doc(dreyfus),
[?ADMIN_CTX]).
-
-create_db(DbName) ->
- ok = fabric:create_db(DbName, [?ADMIN_CTX, {q, 1}]).
-
-delete_db(DbName) ->
- ok = fabric:delete_db(DbName, [?ADMIN_CTX]).
-
-make_docs(Count, Color) ->
- [make_doc(I, Color) || I <- lists:seq(1, Count)].
-
-make_doc(Id, Color) ->
- couch_doc:from_json_obj(
- {[
- {<<"_id">>, get_value(Id)},
- {<<"color">>, Color},
- {<<"size">>, 1}
- ]}
- ).
-
-get_value(Key) ->
- case Key of
- 1 -> <<"apple">>;
- 2 -> <<"tomato">>;
- 3 -> <<"cherry">>;
- 4 -> <<"strawberry">>;
- 5 -> <<"haw">>;
- 6 -> <<"carrot">>;
- 7 -> <<"pitaya">>;
- 8 -> <<"grape">>;
- 9 -> <<"date">>;
- 10 -> <<"watermelon">>
- end.
-
-make_design_doc(dreyfus) ->
- couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"_design/search">>},
- {<<"language">>, <<"javascript">>},
- {<<"indexes">>,
- {[
- {<<"index">>,
- {[
- {<<"analyzer">>, <<"standard">>},
- {<<"index">>, <<
- "function (doc) { \n"
- " index(\"default\", doc._id);\n"
- " if(doc.color) {\n"
- " index(\"color\", doc.color);\n"
- " }\n"
- " if(doc.size) {\n"
- " index(\"size\", doc.size);\n"
- " }\n"
- "}"
- >>}
- ]}}
- ]}}
- ]}
- ).
-
-make_replicate_doc(SourceDbName, TargetDbName) ->
- couch_doc:from_json_obj(
- {[
- {<<"_id">>,
- list_to_binary(
- "replicate_fm_" ++
- binary_to_list(SourceDbName) ++ "_to_" ++
binary_to_list(TargetDbName)
- )},
- {<<"source">>, list_to_binary("http://localhost:15984/"; ++
SourceDbName)},
- {<<"target">>, list_to_binary("http://localhost:15984/"; ++
TargetDbName)}
- ]}
- ).
-
-get_rev(DbName, DocId) ->
- FDI = fabric:get_full_doc_info(DbName, DocId, []),
- #doc_info{revs = [#rev_info{} = PrevRev | _]} = couch_doc:to_doc_info(FDI),
- PrevRev#rev_info.rev.
-
-get_revs(DbName, DocId) ->
- FDI = fabric:get_full_doc_info(DbName, DocId, []),
- #doc_info{revs = Revs} = couch_doc:to_doc_info(FDI),
- [Rev#rev_info.rev || Rev <- Revs].
-
-update_doc(_, _, 0) ->
- ok;
-update_doc(DbName, DocId, Times) ->
- Rev = get_rev(DbName, DocId),
- Doc = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"apple">>},
- {<<"_rev">>, couch_doc:rev_to_str(Rev)},
- {<<"size">>, 1001 - Times}
- ]}
- ),
- {ok, _} = fabric:update_docs(DbName, [Doc], [?ADMIN_CTX]),
- update_doc(DbName, DocId, Times - 1).
-
-delete_docs(DbName, DocIds) ->
- lists:foreach(
- fun(DocId) -> ok = delete_doc(DbName, DocId) end,
- DocIds
- ).
-
-delete_doc(DbName, DocId) ->
- Rev = get_rev(DbName, DocId),
- DDoc = couch_doc:from_json_obj(
- {[
- {<<"_id">>, DocId},
- {<<"_rev">>, couch_doc:rev_to_str(Rev)},
- {<<"_deleted">>, true}
- ]}
- ),
- {ok, _} = fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]),
- ok.
-
-wait_for_replicate(_, _, _, 0) ->
- couch_log:notice("[~p] wait time out", [?MODULE]),
- ok;
-wait_for_replicate(DbName, DocIds, ExpectRevCount, TimeOut) when
- is_list(DocIds)
-->
- [wait_for_replicate(DbName, DocId, ExpectRevCount, TimeOut) || DocId <-
DocIds];
-wait_for_replicate(DbName, DocId, ExpectRevCount, TimeOut) ->
- FDI = fabric:get_full_doc_info(DbName, DocId, []),
- #doc_info{revs = Revs} = couch_doc:to_doc_info(FDI),
- case length(Revs) of
- ExpectRevCount ->
- couch_log:notice(
- "[~p] wait end by expect, time used:~p, DocId:~p",
- [?MODULE, 5 - TimeOut, DocId]
- ),
- ok;
- true ->
- timer:sleep(1000),
- wait_for_replicate(DbName, DocId, ExpectRevCount, TimeOut - 1)
- end,
- ok.
-
-get_sigs(DbName) ->
- {ok, DesignDocs} = fabric:design_docs(DbName),
- lists:usort(
- lists:flatmap(
- fun(Doc) -> active_sigs(DbName, Doc) end,
- [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
- )
- ).
-
-active_sigs(DbName, #doc{body = {Fields}} = Doc) ->
- {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}),
- {IndexNames, _} = lists:unzip(RawIndexes),
- [
- begin
- {ok, Index} = dreyfus_index:design_doc_to_index(DbName, Doc,
IndexName),
- Index#index.sig
- end
- || IndexName <- IndexNames
- ].
diff --git a/test/elixir/test/config/search.elixir
b/test/elixir/test/config/search.elixir
index 9e998cd29..eeeb7d7ae 100644
--- a/test/elixir/test/config/search.elixir
+++ b/test/elixir/test/config/search.elixir
@@ -119,5 +119,22 @@
"PartitionMangoTest": [
"explain options (text)",
"explain works with bookmarks (text)"
+ ],
+ "DreyfusPurgeTest": [
+ "purge single document",
+ "purge multiple documents",
+ "purge multiple documents in batches",
+ "purge conflict",
+ "purge conflict twice removes all",
+ "purge conflict then purge remaining",
+ "purge conflict with all revs",
+ "purge after update",
+ "purge after many updates",
+ "delete document",
+ "delete purge conflict",
+ "delete conflict",
+ "purge then search",
+ "purge updates local checkpoint doc",
+ "delete design doc cleans up local purge doc"
]
}
diff --git a/test/elixir/test/dreyfus_purge_test.exs
b/test/elixir/test/dreyfus_purge_test.exs
new file mode 100644
index 000000000..309d5634f
--- /dev/null
+++ b/test/elixir/test/dreyfus_purge_test.exs
@@ -0,0 +1,474 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+defmodule DreyfusPurgeTest do
+ use CouchTestCase
+
+ @moduletag :search
+
+ @doc_ids ["apple", "tomato", "cherry", "strawberry", "haw"]
+
+ @search_ddoc %{
+ _id: "_design/search",
+ language: "javascript",
+ indexes: %{
+ index: %{
+ analyzer: "standard",
+ index: ~S"""
+ function (doc) {
+ index("default", doc._id);
+ if(doc.color) {
+ index("color", doc.color);
+ }
+ if(doc.size) {
+ index("size", doc.size);
+ }
+ }
+ """
+ }
+ }
+ }
+
+ @tag :with_db
+ test "purge single document", context do
+ db_name = context[:db_name]
+
+ create_db_docs(db_name)
+ wait_search(db_name, "apple", 1)
+
+ purge_docs(db_name, ["apple"])
+ wait_search(db_name, "apple", 0)
+ end
+
+ @tag :with_db
+ test "purge multiple documents", context do
+ db_name = context[:db_name]
+
+ create_db_docs(db_name)
+ wait_search(db_name, "color:red", 5)
+
+ purge_docs(db_name, @doc_ids)
+ wait_search(db_name, "color:red", 0)
+ end
+
+ @tag :with_db
+ test "purge multiple documents in batches", context do
+ db_name = context[:db_name]
+
+ create_db_docs(db_name)
+ wait_search(db_name, "color:red", 5)
+
+ purge_docs(db_name, ["apple", "tomato"])
+ wait_search(db_name, "color:red", 3)
+
+ purge_docs(db_name, ["cherry", "haw"])
+ wait_search(db_name, "color:red", 1)
+ end
+
+ @tag :with_db
+ test "purge conflict", context do
+ db_name_target = context[:db_name]
+ {red_count1, green_count1} = setup_conflict(db_name_target)
+
+ purge_docs(db_name_target, @doc_ids)
+
+ # After purging winning revs, losers become winners (colors swap)
+ retry_until(
+ fn ->
+ red_count2 = dreyfus_search(db_name_target, "color:red")
+ green_count2 = dreyfus_search(db_name_target, "color:green")
+ assert red_count2 + green_count2 == 5
+ assert red_count1 == green_count2
+ assert green_count1 == red_count2
+ end,
+ 200,
+ 60_000
+ )
+ end
+
+ @tag :with_db
+ test "purge conflict twice removes all", context do
+ db_name_target = context[:db_name]
+ setup_conflict(db_name_target)
+
+ purge_docs(db_name_target, @doc_ids)
+ purge_docs(db_name_target, @doc_ids)
+
+ wait_search(db_name_target, "color:red", 0)
+ wait_search(db_name_target, "color:green", 0)
+ end
+
+ @tag :with_db
+ test "purge conflict then purge remaining", context do
+ db_name_target = context[:db_name]
+ {red_count1, green_count1} = setup_conflict(db_name_target)
+
+ purge_docs(db_name_target, @doc_ids)
+
+ # after purging, losers become winners (colors swap)
+ retry_until(
+ fn ->
+ red_count2 = dreyfus_search(db_name_target, "color:red")
+ green_count2 = dreyfus_search(db_name_target, "color:green")
+ assert red_count2 + green_count2 == 5
+ assert red_count1 == green_count2
+ assert green_count1 == red_count2
+ end,
+ 200,
+ 60_000
+ )
+
+ purge_docs(db_name_target, @doc_ids)
+
+ wait_search(db_name_target, "color:red", 0)
+ wait_search(db_name_target, "color:green", 0)
+ end
+
+ @tag :with_db
+ test "purge conflict with all revs", context do
+ db_name_target = context[:db_name]
+ setup_conflict(db_name_target, "green", "red")
+
+ purge_docs_with_all_revs(db_name_target, @doc_ids)
+
+ wait_search(db_name_target, "color:red", 0)
+ wait_search(db_name_target, "color:green", 0)
+ end
+
+ @tag :with_db
+ test "purge after update", context do
+ db_name = context[:db_name]
+ create_db_docs(db_name)
+
+ wait_search(db_name, "color:red", 5)
+
+ # update apple from red to green
+ rev = get_rev(db_name, "apple")
+
+ resp =
+ Couch.put("/#{db_name}/apple",
+ body: %{"_rev" => rev, "color" => "green", "size" => 8}
+ )
+
+ assert resp.status_code in [201, 202]
+
+ retry_until(
+ fn ->
+ red_count = dreyfus_search(db_name, "color:red")
+ green_count = dreyfus_search(db_name, "color:green")
+ assert red_count == 4
+ assert green_count == 1
+ end,
+ 200,
+ 60_000
+ )
+
+ # purge 1 red and 1 green
+ purge_docs(db_name, ["apple", "tomato"])
+
+ retry_until(
+ fn ->
+ red_count = dreyfus_search(db_name, "color:red")
+ green_count = dreyfus_search(db_name, "color:green")
+ assert red_count == 3
+ assert green_count == 0
+ end,
+ 200,
+ 60_000
+ )
+ end
+
+ @tag :with_db
+ test "purge after many updates", context do
+ db_name = context[:db_name]
+ create_db_docs(db_name)
+
+ wait_search(db_name, "size:1", 5)
+
+ # update apple doc 999 times, final size = 1 + 999 = 1000
+ Enum.reduce(1..999, get_rev(db_name, "apple"), fn i, rev ->
+ resp =
+ Couch.put("/#{db_name}/apple",
+ body: %{"_rev" => rev, "size" => i + 1}
+ )
+
+ assert resp.status_code in [201, 202]
+ resp.body["rev"]
+ end)
+
+ retry_until(
+ fn ->
+ count_1 = dreyfus_search(db_name, "size:1")
+ count_1000 = dreyfus_search(db_name, "size:1000")
+ assert count_1 == 4
+ assert count_1000 == 1
+ end,
+ 200,
+ 60_000
+ )
+
+ purge_docs(db_name, ["apple"])
+
+ retry_until(
+ fn ->
+ count_1 = dreyfus_search(db_name, "size:1")
+ count_1000 = dreyfus_search(db_name, "size:1000")
+ assert count_1 == 4
+ assert count_1000 == 0
+ end,
+ 200,
+ 60_000
+ )
+ end
+
+ @tag :with_db
+ test "delete document", context do
+ db_name = context[:db_name]
+
+ create_db_docs(db_name)
+ wait_search(db_name, "apple", 1)
+
+ delete_docs(db_name, ["apple"])
+ wait_search(db_name, "apple", 0)
+ end
+
+ @tag :with_db
+ test "delete purge conflict", context do
+ db_name_target = context[:db_name]
+ setup_conflict(db_name_target)
+
+ purge_docs(db_name_target, @doc_ids)
+ delete_docs(db_name_target, @doc_ids)
+
+ wait_search(db_name_target, "color:red", 0)
+ wait_search(db_name_target, "color:green", 0)
+ end
+
+ @tag :with_db
+ test "delete conflict", context do
+ db_name_target = context[:db_name]
+ {red_count1, green_count1} = setup_conflict(db_name_target)
+
+ delete_docs(db_name_target, @doc_ids)
+
+ # after deleting winning revs, losers become winner (colors swap)
+ retry_until(
+ fn ->
+ red_count2 = dreyfus_search(db_name_target, "color:red")
+ green_count2 = dreyfus_search(db_name_target, "color:green")
+ assert red_count2 + green_count2 == 5
+ assert red_count1 == green_count2
+ assert green_count1 == red_count2
+ end,
+ 200,
+ 60_000
+ )
+ end
+
+ @tag :with_db
+ test "purge then search", context do
+ db_name = context[:db_name]
+ create_db_docs(db_name)
+ purge_docs(db_name, ["apple", "tomato", "haw"])
+ wait_search(db_name, "color:red", 2)
+ end
+
+ @tag :with_db
+ test "purge updates local checkpoint doc", context do
+ db_name = context[:db_name]
+ create_db_docs(db_name)
+ wait_search(db_name, "apple", 1)
+
+ # purge checkpoints exist before, purge_seq == 0
+ checkpoints_before = get_checkpoints(db_name)
+ assert checkpoints_before != []
+
+ Enum.each(checkpoints_before, fn doc ->
+ assert doc["type"] == "dreyfus"
+ assert doc["ddoc_id"] == "_design/search"
+ assert doc["indexname"] == "index"
+ assert doc["purge_seq"] == 0
+ end)
+
+ purge_docs(db_name, ["apple", "tomato", "cherry", "strawberry"])
+ wait_search(db_name, "apple", 0)
+
+ # purge checkpoints updated after, purge_seq > 0
+ checkpoints_after = get_checkpoints(db_name)
+ assert length(checkpoints_after) == length(checkpoints_before)
+
+ total_purge_seq =
+ Enum.reduce(checkpoints_after, 0, fn doc, acc ->
+ assert doc["type"] == "dreyfus"
+ assert doc["ddoc_id"] == "_design/search"
+ assert doc["purge_seq"] > 0
+ acc + doc["purge_seq"]
+ end)
+
+ # assert total purge seq count
+ assert total_purge_seq == 4
+ end
+
+ @tag :with_db
+ test "delete design doc cleans up local purge doc", context do
+ db_name = context[:db_name]
+ create_db_docs(db_name)
+ wait_search(db_name, "apple", 1)
+ purge_docs(db_name, ["apple"])
+ wait_search(db_name, "apple", 0)
+
+ # purge checkpoints exist before
+ assert get_checkpoints(db_name) != []
+
+ delete_docs(db_name, ["_design/search"])
+ resp = Couch.post("/#{db_name}/_search_cleanup")
+ assert resp.status_code in [201, 202]
+
+ # after cleanup checkpoints are gone
+ retry_until(
+ fn ->
+ assert get_checkpoints(db_name) == []
+ end,
+ 200,
+ 60_000
+ )
+ end
+
+ defp create_db_docs(db_name, color \\ "red", opts \\ []) do
+ docs =
+ Enum.map(@doc_ids, fn id ->
+ %{"_id" => id, "color" => color, "size" => 1}
+ end)
+ resp = Couch.post("/#{db_name}/_bulk_docs", body: %{docs: docs})
+ assert resp.status_code in [201, 202]
+ unless opts[:skip_ddoc] do
+ resp = Couch.post("/#{db_name}", body: @search_ddoc)
+ assert resp.status_code in [201, 202]
+ end
+ end
+
+ defp get_checkpoints(db_name) do
+ resp =
+ Couch.get("/#{db_name}/_local_docs",
+ query: %{include_docs: true, include_system: true}
+ )
+ assert resp.status_code == 200
+ resp.body["rows"]
+ |> Enum.filter(fn row ->
+ String.starts_with?(row["id"], "_local/purge-dreyfus-")
+ end)
+ |> Enum.map(fn row -> row["doc"] end)
+ end
+
+ defp dreyfus_search(db_name, query) do
+ url = "/#{db_name}/_design/search/_search/index"
+ resp = Couch.get(url, query: %{q: query}, timeout: 60_000)
+ assert resp.status_code == 200
+ resp.body["total_rows"]
+ end
+
+ defp wait_search(db_name, query, expected_count) do
+ retry_fun = fn -> assert expected_count == dreyfus_search(db_name, query)
end
+ retry_until(retry_fun, 200, 60_000)
+ end
+
+ defp get_rev(db_name, doc_id) do
+ resp = Couch.get("/#{db_name}/#{doc_id}")
+ assert resp.status_code == 200
+ resp.body["_rev"]
+ end
+
+ defp get_revs(db_name, doc_id) do
+ resp = Couch.get("/#{db_name}/#{doc_id}", query: %{conflicts: true,
revs_info: true})
+ assert resp.status_code == 200
+ # we include the winning rev + the rest of the conflicts in the response
+ case resp.body["_conflicts"] do
+ nil -> [resp.body["_rev"]]
+ conflicts -> [resp.body["_rev"] | conflicts]
+ end
+ end
+
+ defp purge_docs(db_name, doc_ids) do
+ purge_body =
+ Enum.into(doc_ids, %{}, fn doc_id ->
+ rev = get_rev(db_name, doc_id)
+ {doc_id, [rev]}
+ end)
+ resp = Couch.post("/#{db_name}/_purge", body: purge_body)
+ assert resp.status_code in [201, 202]
+ end
+
+ defp purge_docs_with_all_revs(db_name, doc_ids) do
+ purge_body =
+ Enum.into(doc_ids, %{}, fn doc_id ->
+ revs = get_revs(db_name, doc_id)
+ {doc_id, revs}
+ end)
+ resp = Couch.post("/#{db_name}/_purge", body: purge_body)
+ assert resp.status_code in [201, 202]
+ end
+
+ defp delete_docs(db_name, doc_ids) do
+ Enum.each(doc_ids, fn doc_id ->
+ rev = get_rev(db_name, doc_id)
+ resp = Couch.delete("/#{db_name}/#{doc_id}", query: %{rev: rev})
+ assert resp.status_code in [200, 202]
+ end)
+ end
+
+ # This is kind of a silly way to induce conflicts, but we are just copying it
+ # from the erlang dreyfus_purge_test module. In the end return the red and
+ # green count tuple.
+ #
+ defp setup_conflict(db_name_target, source_color \\ "red", target_color \\
"green") do
+ db_name_source = random_db_name()
+ create_db(db_name_source)
+ on_exit(fn -> delete_db(db_name_source) end)
+
+ create_db_docs(db_name_source, source_color, skip_ddoc: true)
+ create_db_docs(db_name_target, target_color)
+
+ retry_until(
+ fn ->
+ red = dreyfus_search(db_name_target, "color:red")
+ green = dreyfus_search(db_name_target, "color:green")
+ assert red + green == 5
+ end,
+ 200,
+ 60_000
+ )
+
+ replicate(db_name_source, db_name_target)
+
+ # Assert conflicts were created on each doc
+ Enum.each(@doc_ids, fn doc_id ->
+ revs = get_revs(db_name_target, doc_id)
+ assert length(revs) == 2, "expected 2 revs (conflict) for #{doc_id}, got
#{length(revs)}"
+ end)
+
+ # After replicating, search should still see 5 docs total
+ retry_until(
+ fn ->
+ red = dreyfus_search(db_name_target, "color:red")
+ green = dreyfus_search(db_name_target, "color:green")
+ assert red + green == 5
+ end,
+ 200,
+ 60_000
+ )
+
+ red_count = dreyfus_search(db_name_target, "color:red")
+ green_count = dreyfus_search(db_name_target, "color:green")
+ {red_count, green_count}
+ end
+
+end
