[couchdb] branch COUCHDB-3326-clustered-purge-davisp-refactor-2 updated (1af4acb -> 337515c)

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


 discard 1af4acb  Fix UUID test
 new 337515c  Fix UUID test

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (1af4acb)
\
 N -- N -- N   
refs/heads/COUCHDB-3326-clustered-purge-davisp-refactor-2 (337515c)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 src/couch/src/test_engine_purge_docs.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 01/01: Fix UUID test

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 337515cae5f941b2fdd68a9e9231baad52fe9cc3
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 15:03:06 2018 -0500

Fix UUID test
---
 src/couch/src/test_engine_purge_docs.erl | 53 
 1 file changed, 27 insertions(+), 26 deletions(-)

diff --git a/src/couch/src/test_engine_purge_docs.erl 
b/src/couch/src/test_engine_purge_docs.erl
index abc9138..8e6dbc0 100644
--- a/src/couch/src/test_engine_purge_docs.erl
+++ b/src/couch/src/test_engine_purge_docs.erl
@@ -16,7 +16,6 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("couch/include/couch_db.hrl").
--include("couch_bt_engine.hrl").
 
 
 cet_purge_simple() ->
@@ -54,43 +53,41 @@ cet_purge_simple() ->
 
 
 cet_purge_UUID() ->
-{ok, Engine, St1} = test_engine_util:init_engine(),
+{ok, Db1} = test_engine_util:create_db(),
 
 Actions1 = [
-{create, {<<"foo">>, [{<<"vsn">>, 1}]}}
+{create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}
 ],
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
-{ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []),
+{ok, Db2} = test_engine_util:apply_actions(Db1, Actions1),
+{ok, PIdRevs2} = couch_db_engine:fold_purge_infos(
+Db2, 0, fun fold_fun/2, [], []),
 
-?assertEqual(1, Engine:get_doc_count(St2)),
-?assertEqual(0, Engine:get_del_doc_count(St2)),
-?assertEqual(1, Engine:get_update_seq(St2)),
-?assertEqual(0, Engine:get_purge_seq(St2)),
+?assertEqual(1, couch_db_engine:get_doc_count(Db2)),
+?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)),
+?assertEqual(1, couch_db_engine:get_update_seq(Db2)),
+?assertEqual(0, couch_db_engine:get_purge_seq(Db2)),
 ?assertEqual([], PIdRevs2),
 
-[FDI] = Engine:open_docs(St2, [<<"foo">>]),
+[FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]),
 PrevRev = test_engine_util:prev_rev(FDI),
 Rev = PrevRev#rev_info.rev,
 
 Actions2 = [
 {purge, {<<"foo">>, Rev}}
 ],
-{ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2),
-{ok, _PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []),
-
-?assertEqual(0, Engine:get_doc_count(St3)),
-?assertEqual(0, Engine:get_del_doc_count(St3)),
-?assertEqual(2, Engine:get_update_seq(St3)),
-?assertEqual(1, Engine:get_purge_seq(St3)),
-
-PurgeSeqTree = St3#st.purge_seq_tree,
-
-Fun = fun({PurgeSeq, UUID, _, _}, _Reds, _Acc) ->
-{stop, {PurgeSeq, UUID}}
- end,
-{ok, _, {_, UUID}} = couch_btree:fold(
-PurgeSeqTree, Fun, 0, [{dir, rev}]
-),
+{ok, Db3} = test_engine_util:apply_actions(Db2, Actions2),
+{ok, PIdRevs3} = couch_db_engine:fold_purge_infos(
+Db3, 0, fun fold_fun/2, [], []),
+
+?assertEqual(0, couch_db_engine:get_doc_count(Db3)),
+?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)),
+?assertEqual(2, couch_db_engine:get_update_seq(Db3)),
+?assertEqual(1, couch_db_engine:get_purge_seq(Db3)),
+?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3),
+
+{ok, {PSeq, UUID}} = couch_db_engine:fold_purge_infos(
+Db3, 0, fun first_uuid/2, [], []),
+?assertEqual(1, PSeq),
 ?assert(is_binary(UUID)).
 
 
@@ -220,3 +217,7 @@ cet_add_two_purge_one() ->
 
 fold_fun({_Pseq, _UUID, Id, Revs}, Acc) ->
 {ok, [{Id, Revs} | Acc]}.
+
+
+first_uuid({PSeq, UUID, _, _}, _) ->
+{stop, {PSeq, UUID}}.

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 01/01: Fix UUID test

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 1af4acbaa2a9096c3797cead712ee87e094b5bf2
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 15:03:06 2018 -0500

Fix UUID test
---
 src/couch/src/test_engine_purge_docs.erl | 53 
 1 file changed, 27 insertions(+), 26 deletions(-)

diff --git a/src/couch/src/test_engine_purge_docs.erl 
b/src/couch/src/test_engine_purge_docs.erl
index abc9138..7b30ad9 100644
--- a/src/couch/src/test_engine_purge_docs.erl
+++ b/src/couch/src/test_engine_purge_docs.erl
@@ -16,7 +16,6 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("couch/include/couch_db.hrl").
--include("couch_bt_engine.hrl").
 
 
 cet_purge_simple() ->
@@ -54,43 +53,41 @@ cet_purge_simple() ->
 
 
 cet_purge_UUID() ->
-{ok, Engine, St1} = test_engine_util:init_engine(),
+{ok, Db1} = test_engine_util:create_db(),
 
 Actions1 = [
-{create, {<<"foo">>, [{<<"vsn">>, 1}]}}
+{create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}
 ],
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
-{ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []),
+{ok, Db2} = test_engine_util:apply_actions(Db1, Actions1),
+{ok, PIdRevs2} = couch_db_engine:fold_purge_infos(
+Db2, 0, fun fold_fun/2, [], []),
 
-?assertEqual(1, Engine:get_doc_count(St2)),
-?assertEqual(0, Engine:get_del_doc_count(St2)),
-?assertEqual(1, Engine:get_update_seq(St2)),
-?assertEqual(0, Engine:get_purge_seq(St2)),
+?assertEqual(1, couch_db_engine:get_doc_count(Db2)),
+?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)),
+?assertEqual(1, couch_db_engine:get_update_seq(Db2)),
+?assertEqual(0, couch_db_engine:get_purge_seq(Db2)),
 ?assertEqual([], PIdRevs2),
 
-[FDI] = Engine:open_docs(St2, [<<"foo">>]),
+[FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]),
 PrevRev = test_engine_util:prev_rev(FDI),
 Rev = PrevRev#rev_info.rev,
 
 Actions2 = [
 {purge, {<<"foo">>, Rev}}
 ],
-{ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2),
-{ok, _PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []),
-
-?assertEqual(0, Engine:get_doc_count(St3)),
-?assertEqual(0, Engine:get_del_doc_count(St3)),
-?assertEqual(2, Engine:get_update_seq(St3)),
-?assertEqual(1, Engine:get_purge_seq(St3)),
-
-PurgeSeqTree = St3#st.purge_seq_tree,
-
-Fun = fun({PurgeSeq, UUID, _, _}, _Reds, _Acc) ->
-{stop, {PurgeSeq, UUID}}
- end,
-{ok, _, {_, UUID}} = couch_btree:fold(
-PurgeSeqTree, Fun, 0, [{dir, rev}]
-),
+{ok, Db3} = test_engine_util:apply_actions(Db2, Actions2),
+{ok, PIdRevs3} = couch_db_engine:fold_purge_infos(
+Db3, 0, fun fold_fun/2, [], []),
+
+?assertEqual(0, couch_db_engine:get_doc_count(Db3)),
+?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)),
+?assertEqual(2, couch_db_engine:get_update_seq(Db3)),
+?assertEqual(1, couch_db_engine:get_purge_seq(Db3)),
+?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3),
+
+{ok, {PSeq, UUID}} = couch_db_engine:fold_purge_infso(
+Db3, 0, fun first_uuid/2, [], []),
+?assertEqual(1, PSeq),
 ?assert(is_binary(UUID)).
 
 
@@ -220,3 +217,7 @@ cet_add_two_purge_one() ->
 
 fold_fun({_Pseq, _UUID, Id, Revs}, Acc) ->
 {ok, [{Id, Revs} | Acc]}.
+
+
+first_uuid({PSeq, UUID, _, _}, _) ->
+{stop, {PSeq, UUID}}.

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] branch COUCHDB-3326-clustered-purge-davisp-refactor-2 updated (c941892 -> 1af4acb)

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


 discard c941892  Fix UUID test
 new 1af4acb  Fix UUID test

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (c941892)
\
 N -- N -- N   
refs/heads/COUCHDB-3326-clustered-purge-davisp-refactor-2 (1af4acb)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 src/couch/src/test_engine_purge_docs.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 01/03: Add more test cases

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 871cdd091502ed99d850e22c537966098a090804
Author: jiangphcn 
AuthorDate: Wed Apr 25 16:16:52 2018 +0800

Add more test cases

COUCHDB-3326
---
 src/couch/src/test_engine_purge_docs.erl  |  42 +
 src/couch/test/couch_db_purge_docs_tests.erl  | 105 --
 src/couch/test/fixtures/db_with_1_purge_req.couch | Bin 0 -> 8354 bytes
 3 files changed, 141 insertions(+), 6 deletions(-)

diff --git a/src/couch/src/test_engine_purge_docs.erl 
b/src/couch/src/test_engine_purge_docs.erl
index f70cf58..abc9138 100644
--- a/src/couch/src/test_engine_purge_docs.erl
+++ b/src/couch/src/test_engine_purge_docs.erl
@@ -16,6 +16,7 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("couch/include/couch_db.hrl").
+-include("couch_bt_engine.hrl").
 
 
 cet_purge_simple() ->
@@ -52,6 +53,47 @@ cet_purge_simple() ->
 ?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3).
 
 
+cet_purge_UUID() ->
+{ok, Engine, St1} = test_engine_util:init_engine(),
+
+Actions1 = [
+{create, {<<"foo">>, [{<<"vsn">>, 1}]}}
+],
+{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
+{ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []),
+
+?assertEqual(1, Engine:get_doc_count(St2)),
+?assertEqual(0, Engine:get_del_doc_count(St2)),
+?assertEqual(1, Engine:get_update_seq(St2)),
+?assertEqual(0, Engine:get_purge_seq(St2)),
+?assertEqual([], PIdRevs2),
+
+[FDI] = Engine:open_docs(St2, [<<"foo">>]),
+PrevRev = test_engine_util:prev_rev(FDI),
+Rev = PrevRev#rev_info.rev,
+
+Actions2 = [
+{purge, {<<"foo">>, Rev}}
+],
+{ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2),
+{ok, _PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []),
+
+?assertEqual(0, Engine:get_doc_count(St3)),
+?assertEqual(0, Engine:get_del_doc_count(St3)),
+?assertEqual(2, Engine:get_update_seq(St3)),
+?assertEqual(1, Engine:get_purge_seq(St3)),
+
+PurgeSeqTree = St3#st.purge_seq_tree,
+
+Fun = fun({PurgeSeq, UUID, _, _}, _Reds, _Acc) ->
+{stop, {PurgeSeq, UUID}}
+ end,
+{ok, _, {_, UUID}} = couch_btree:fold(
+PurgeSeqTree, Fun, 0, [{dir, rev}]
+),
+?assert(is_binary(UUID)).
+
+
 cet_purge_conflicts() ->
 {ok, Db1} = test_engine_util:create_db(),
 
diff --git a/src/couch/test/couch_db_purge_docs_tests.erl 
b/src/couch/test/couch_db_purge_docs_tests.erl
index ed42f2b..4e46dfb 100644
--- a/src/couch/test/couch_db_purge_docs_tests.erl
+++ b/src/couch/test/couch_db_purge_docs_tests.erl
@@ -45,6 +45,7 @@ couch_db_purge_docs() ->
foreach,
 fun setup/0, fun teardown/1,
 [
+fun test_purge_2_to_purge_3/1,
 fun test_purge_all/1,
 fun test_purge_some/1,
 fun test_purge_none/1,
@@ -53,6 +54,8 @@ couch_db_purge_docs() ->
 %fun test_purge_repeated_rev/1, % improving
 fun test_purge_partial/1,
 fun test_all_removal_purges/1,
+fun test_purge_invalid_rev/1,
+fun test_purge_duplicate_UUID/1,
 fun purge_id_not_exist/1,
 fun purge_non_leaf_rev/1,
 fun purge_deep_tree/1
@@ -60,6 +63,27 @@ couch_db_purge_docs() ->
 }.
 
 
+test_purge_2_to_purge_3(DbName) ->
+?_test(
+begin
+{ok, Db} = couch_db:open_int(DbName, []),
+Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1.1}]},
+{ok, Rev} = save_doc(Db, Doc1),
+couch_db:ensure_full_commit(Db),
+{ok, Db2} = couch_db:reopen(Db),
+UUID = couch_uuids:new(),
+{ok, [{ok, PRevs}]} = couch_db:purge_docs(
+Db2, [{UUID, <<"foo1">>, [Rev]}]
+),
+?assertEqual([Rev], PRevs),
+{ok, Db3} = couch_db:reopen(Db2),
+{ok, _PIdsRevs} = couch_db:fold_purge_infos(
+Db3, 0, fun fold_fun/2, [], []),
+?assertEqual(0, couch_db_engine:get_doc_count(Db3)),
+?assertEqual(1, couch_db_engine:get_purge_seq(Db3))
+end).
+
+
 test_purge_all(DbName) ->
 ?_test(
 begin
@@ -305,7 +329,9 @@ purge_non_leaf_rev(DbName) ->
 ?assertEqual([], PRevs),
 
 {ok, Db4} = couch_db:reopen(Db3),
-{ok, PIdsRevs} = couch_db:fold_purge_infos(Db4, 0, fun fold_fun/2, 
[], []),
+{ok, PIdsRevs} = couch_db:fold_purge_infos(
+Db4, 0, fun fold_fun/2, [], []
+),
 ?assertEqual(1, couch_db_engine:get_doc_count(Db4)),
 ?assertEqual(2, couch_db_engine:get_update_seq(Db4)),
 ?assertEqual(0, 

[couchdb] 03/03: Fix UUID test

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit c94189248def06aec080987d7edfa8916440a174
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 15:03:06 2018 -0500

Fix UUID test
---
 src/couch/src/test_engine_purge_docs.erl | 51 
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/src/couch/src/test_engine_purge_docs.erl 
b/src/couch/src/test_engine_purge_docs.erl
index abc9138..d216616 100644
--- a/src/couch/src/test_engine_purge_docs.erl
+++ b/src/couch/src/test_engine_purge_docs.erl
@@ -16,7 +16,6 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("couch/include/couch_db.hrl").
--include("couch_bt_engine.hrl").
 
 
 cet_purge_simple() ->
@@ -54,43 +53,41 @@ cet_purge_simple() ->
 
 
 cet_purge_UUID() ->
-{ok, Engine, St1} = test_engine_util:init_engine(),
+{ok, Db1} = test_engine_util:create_db(),
 
 Actions1 = [
 {create, {<<"foo">>, [{<<"vsn">>, 1}]}}
 ],
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
-{ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []),
+{ok, Db2} = test_engine_util:apply_actions(Db1, Actions1),
+{ok, PIdRevs2} = couch_db_engine:fold_purge_infos(
+Db2, 0, fun fold_fun/2, [], []),
 
-?assertEqual(1, Engine:get_doc_count(St2)),
-?assertEqual(0, Engine:get_del_doc_count(St2)),
-?assertEqual(1, Engine:get_update_seq(St2)),
-?assertEqual(0, Engine:get_purge_seq(St2)),
+?assertEqual(1, couch_db_engine:get_doc_count(Db2)),
+?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)),
+?assertEqual(1, couch_db_engine:get_update_seq(Db2)),
+?assertEqual(0, couch_db_engine:get_purge_seq(Db2)),
 ?assertEqual([], PIdRevs2),
 
-[FDI] = Engine:open_docs(St2, [<<"foo">>]),
+[FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]),
 PrevRev = test_engine_util:prev_rev(FDI),
 Rev = PrevRev#rev_info.rev,
 
 Actions2 = [
 {purge, {<<"foo">>, Rev}}
 ],
-{ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2),
-{ok, _PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []),
-
-?assertEqual(0, Engine:get_doc_count(St3)),
-?assertEqual(0, Engine:get_del_doc_count(St3)),
-?assertEqual(2, Engine:get_update_seq(St3)),
-?assertEqual(1, Engine:get_purge_seq(St3)),
-
-PurgeSeqTree = St3#st.purge_seq_tree,
-
-Fun = fun({PurgeSeq, UUID, _, _}, _Reds, _Acc) ->
-{stop, {PurgeSeq, UUID}}
- end,
-{ok, _, {_, UUID}} = couch_btree:fold(
-PurgeSeqTree, Fun, 0, [{dir, rev}]
-),
+{ok, Db3} = test_engine_util:apply_actions(Db2, Actions2),
+{ok, PIdRevs3} = couch_db_engine:fold_purge_infos(
+Db3, 0, fun fold_fun/2, [], []),
+
+?assertEqual(0, couch_db_engine:get_doc_count(Db3)),
+?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)),
+?assertEqual(2, couch_db_engine:get_update_seq(Db3)),
+?assertEqual(1, couch_db_engine:get_purge_seq(Db3)),
+?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3),
+
+{ok, {PSeq, UUID}} = couch_db_engine:fold_purge_infso(
+Db3, 0, fun first_uuid/2, [], []),
+?assertEqual(1, PSeq),
 ?assert(is_binary(UUID)).
 
 
@@ -220,3 +217,7 @@ cet_add_two_purge_one() ->
 
 fold_fun({_Pseq, _UUID, Id, Revs}, Acc) ->
 {ok, [{Id, Revs} | Acc]}.
+
+
+first_uuid({PSeq, UUID, _, _}, _) ->
+{stop, {PSeq, UUID}}.

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 02/03: Add test compact with broken purge checkpoint doc

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 441ea3282d11b2706a15b419fc8aa8794531a420
Author: jiangphcn 
AuthorDate: Thu Apr 26 17:46:06 2018 +0800

Add test compact with broken purge checkpoint doc

COUCHDB-3326
---
 src/couch/src/couch_db.erl |   6 +-
 src/couch/test/couch_db_purge_checkpoint_tests.erl | 237 +
 2 files changed, 241 insertions(+), 2 deletions(-)

diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 8592193..07feab0 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -80,6 +80,8 @@
 get_purge_infos/2,
 
 get_minimum_purge_seq/1,
+purge_client_exists/3,
+get_purge_client_fun/2,
 
 update_doc/3,
 update_doc/4,
@@ -481,7 +483,7 @@ get_purge_client_fun(DocId, Props) ->
 M = try
 binary_to_existing_atom(M0, latin1)
 catch error:badarg ->
-Fmt1 = "Missing index module '~s' for purge checkpoint '~s'",
+Fmt1 = "Missing index module '~p' for purge checkpoint '~p'",
 couch_log:error(Fmt1, [M0, DocId]),
 throw(failed)
 end,
@@ -491,7 +493,7 @@ get_purge_client_fun(DocId, Props) ->
 F = binary_to_existing_atom(F0, latin1),
 fun M:F/2
 catch error:badarg ->
-Fmt2 = "Missing function '~s' in '~s' for purge checkpoint '~s'",
+Fmt2 = "Missing function '~p' in '~p' for purge checkpoint '~p'",
 couch_log:error(Fmt2, [F0, M0, DocId]),
 throw(failed)
 end.
diff --git a/src/couch/test/couch_db_purge_checkpoint_tests.erl 
b/src/couch/test/couch_db_purge_checkpoint_tests.erl
new file mode 100644
index 000..1634e61
--- /dev/null
+++ b/src/couch/test/couch_db_purge_checkpoint_tests.erl
@@ -0,0 +1,237 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_db_purge_checkpoint_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+
+setup() ->
+DbName = ?tempdb(),
+{ok, _Db} = create_db(DbName),
+DbName.
+
+teardown(DbName) ->
+delete_db(DbName),
+ok.
+
+couch_db_purge_checkpoint_test_() ->
+{
+"Couch_db purge_checkpoint",
+[
+{
+setup,
+fun test_util:start_couch/0, fun test_util:stop_couch/1,
+[couch_db_purge_checkpoint()]
+}
+]
+
+}.
+
+
+couch_db_purge_checkpoint() ->
+{
+   foreach,
+fun setup/0, fun teardown/1,
+[
+fun test_purge_cp_bad_purgeseq/1,
+fun test_purge_cp_bad_verify_mod/1,
+fun test_purge_cp_bad_verify_fun/1,
+fun test_purge_cp_verify_fun_with_throw/1,
+fun test_purge_cp_verify_fun_without_boolean_rc/1
+]
+}.
+
+
+test_purge_cp_bad_purgeseq(DbName) ->
+?_test(
+begin
+{ok, Db} = couch_db:open_int(DbName, []),
+update_local_purge_doc(
+Db,
+"<>",
+bad_purgeseq,
+<<"couch_db_purge_checkpoint_tests">>,
+<<"normal_verify_fun">>
+),
+{ok, Db2} = couch_db:reopen(Db),
+Result = try
+couch_db:get_minimum_purge_seq(Db2)
+catch _:_ ->
+failed
+end,
+?assertEqual(0, Result)
+end).
+
+
+test_purge_cp_bad_verify_mod(DbName) ->
+?_test(
+begin
+{ok, Db} = couch_db:open_int(DbName, []),
+update_local_purge_doc(
+Db,
+"<>",
+1,
+[invalid_module],
+<<"valid_verify_fun">>
+
+),
+{ok, Db2} = couch_db:reopen(Db),
+FoldFun = fun(#doc{id = DocId, body = {Props}}, SeqAcc) ->
+case DocId of
+<<"_local/purge-", _/binary>> ->
+try
+couch_db:get_purge_client_fun(DocId, Props)
+catch failed ->
+{ok, badarg}
+end;
+_ ->
+{stop, SeqAcc}
+end
+end,
+Opts = [
+

[couchdb] branch COUCHDB-3326-clustered-purge-davisp-refactor-2 updated (0f369cf -> c941892)

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


from 0f369cf  Update tests for new PSE test suite
 new 871cdd0  Add more test cases
 new 441ea32  Add test compact with broken purge checkpoint doc
 new c941892  Fix UUID test

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 src/couch/src/couch_db.erl |   6 +-
 src/couch/src/test_engine_purge_docs.erl   |  43 
 src/couch/test/couch_db_purge_checkpoint_tests.erl | 237 +
 src/couch/test/couch_db_purge_docs_tests.erl   | 105 -
 src/couch/test/fixtures/db_with_1_purge_req.couch  | Bin 0 -> 8354 bytes
 5 files changed, 383 insertions(+), 8 deletions(-)
 create mode 100644 src/couch/test/couch_db_purge_checkpoint_tests.erl
 create mode 100644 src/couch/test/fixtures/db_with_1_purge_req.couch

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 01/01: Update tests for new PSE test suite

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 0f369cf3fd508ceb7c073fa56f0fb65a5b418f24
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 13:57:52 2018 -0500

Update tests for new PSE test suite
---
 src/couch/src/test_engine_compaction.erl   | 181 +
 src/couch/src/test_engine_fold_purge_infos.erl |  74 +-
 src/couch/src/test_engine_get_set_props.erl|   1 -
 src/couch/src/test_engine_purge_docs.erl   |   4 +-
 src/couch/src/test_engine_util.erl |   9 +-
 5 files changed, 113 insertions(+), 156 deletions(-)

diff --git a/src/couch/src/test_engine_compaction.erl 
b/src/couch/src/test_engine_compaction.erl
index e49167a..26714a1 100644
--- a/src/couch/src/test_engine_compaction.erl
+++ b/src/couch/src/test_engine_compaction.erl
@@ -104,7 +104,7 @@ cet_compact_with_everything() ->
 {<<"foo">>, [FooRev#rev_info.rev]}
 ],
 
-{ok, PIdRevs4} = couch_db_enigne:fold_purge_infos(
+{ok, PIdRevs4} = couch_db_engine:fold_purge_infos(
 Db4, 0, fun fold_fun/2, [], []),
 ?assertEqual(PurgedIdRevs, PIdRevs4),
 
@@ -177,180 +177,131 @@ cet_recompact_updates() ->
 
 
 cet_purge_during_compact() ->
-{ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath),
+{ok, Db1} = test_engine_util:create_db(),
 
-Actions1 = [
-{create, {<<"foo">>, []}},
-{create, {<<"bar">>, []}},
-{conflict, {<<"bar">>, [{<<"vsn">>, 2}]}},
-{create, {<<"baz">>, []}}
+Actions1 = lists:map(fun(Seq) ->
+{create, {docid(Seq), {[{<<"int">>, Seq}]}}}
+end, lists:seq(1, 1000)),
+Actions2 = [
+{create, {<<"foo">>, {[]}}},
+{create, {<<"bar">>, {[]}}},
+{create, {<<"baz">>, {[]}}}
 ],
+{ok, Db2} = test_engine_util:apply_batch(Db1, Actions1 ++ Actions2),
+Actions3 = [
+{conflict, {<<"bar">>, {[{<<"vsn">>, 2}]}}}
+],
+{ok, Db3} = test_engine_util:apply_actions(Db2, Actions3),
 
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
-{ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path),
+{ok, Pid} = couch_db:start_compact(Db3),
+catch erlang:suspend_process(Pid),
 
-[BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]),
+[BarFDI, BazFDI] = couch_db_engine:open_docs(Db3, [<<"bar">>, <<"baz">>]),
 BarRev = test_engine_util:prev_rev(BarFDI),
 BazRev = test_engine_util:prev_rev(BazFDI),
-Actions2 = [
+Actions4 = [
 {purge, {<<"bar">>, BarRev#rev_info.rev}},
 {purge, {<<"baz">>, BazRev#rev_info.rev}}
 ],
-{ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2),
-Db1 = test_engine_util:db_as_term(Engine, St4),
 
-{ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term),
+{ok, Db4} = test_engine_util:apply_actions(Db3, Actions4),
+Term1 = test_engine_util:db_as_term(Db4),
 
-?assertEqual(true, is_pid(NewPid)),
-Ref = erlang:monitor(process, NewPid),
+catch erlang:resume_process(Pid),
+test_engine_util:compact(Db4),
 
-NewTerm = receive
-{'$gen_cast', {compact_done, Engine, Term0}} ->
-Term0;
-{'DOWN', Ref, _, _, Reason} ->
-erlang:error({compactor_died, Reason})
-after 1 ->
-erlang:error(compactor_timed_out)
-end,
+{ok, Db5} = couch_db:reopen(Db4),
+Term2 = test_engine_util:db_as_term(Db5),
 
-{ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm),
-Db2 = test_engine_util:db_as_term(Engine, St6),
-Diff = test_engine_util:term_diff(Db1, Db2),
+Diff = test_engine_util:term_diff(Term1, Term2),
 ?assertEqual(nodiff, Diff).
 
 
 cet_multiple_purge_during_compact() ->
-{ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath),
-
-Actions1 = [
-{create, {<<"foo">>, []}},
-{create, {<<"bar">>, []}},
-{conflict, {<<"bar">>, [{<<"vsn">>, 2}]}},
-{create, {<<"baz">>, []}}
-],
-
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
-{ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path),
+{ok, Db1} = test_engine_util:create_db(),
 
-[BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]),
-BarRev = test_engine_util:prev_rev(BarFDI),
+Actions1 = lists:map(fun(Seq) ->
+{create, {docid(Seq), {[{<<"int">>, Seq}]}}}
+end, lists:seq(1, 1000)),
 Actions2 = [
-{purge, {<<"bar">>, BarRev#rev_info.rev}}
+{create, {<<"foo">>, {[]}}},
+{create, {<<"bar">>, {[]}}},
+{create, {<<"baz">>, {[]}}}
 ],
-{ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2),
+{ok, Db2} = test_engine_util:apply_batch(Db1, Actions1 ++ Actions2),
 
-

[couchdb] branch COUCHDB-3326-clustered-purge-davisp-refactor-2 updated (f0b7ddb -> 0f369cf)

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


 discard f0b7ddb  Update tests for new PSE test suite
 discard 53356ba  TMP - avoid couch_db:open_int/2 in PSE callbacks
 new 0f369cf  Update tests for new PSE test suite

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (f0b7ddb)
\
 N -- N -- N   
refs/heads/COUCHDB-3326-clustered-purge-davisp-refactor-2 (0f369cf)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 src/couch/src/couch_bt_engine_compactor.erl | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 17/19: Update HTTP JS tests for clustered purge

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit b7dccd5a7646ae3c7b63fcdee84fe4d86801f95b
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:28:24 2018 -0500

Update HTTP JS tests for clustered purge
---
 test/javascript/tests/erlang_views.js |  5 +++--
 test/javascript/tests/purge.js| 27 ---
 2 files changed, 11 insertions(+), 21 deletions(-)

diff --git a/test/javascript/tests/erlang_views.js 
b/test/javascript/tests/erlang_views.js
index ec78e65..9b15e10 100644
--- a/test/javascript/tests/erlang_views.js
+++ b/test/javascript/tests/erlang_views.js
@@ -56,7 +56,7 @@ couchTests.erlang_views = function(debug) {
 '  {Info} = couch_util:get_value(<<"info">>, Req, {[]}), ' +
 '  Purged = couch_util:get_value(<<"purge_seq">>, Info, -1), ' +
 '  Verb = couch_util:get_value(<<"method">>, Req, <<"not_get">>), 
' +
-'  R = list_to_binary(io_lib:format("~b - ~s", [Purged, Verb])), ' 
+
+'  R = list_to_binary(io_lib:format("~s - ~s", [Purged, Verb])), ' 
+
 '  {[{<<"code">>, 200}, {<<"headers">>, {[]}}, {<<"body">>, R}]} ' 
+
 'end.'
 },
@@ -85,7 +85,8 @@ couchTests.erlang_views = function(debug) {
   var url = "/" + db_name + "/_design/erlview/_show/simple/1";
   var xhr = CouchDB.request("GET", url);
   T(xhr.status == 200, "standard get should be 200");
-  T(xhr.responseText == "0 - GET");
+  T(/0-/.test(xhr.responseText));
+  T(/- GET/.test(xhr.responseText));
 
   var url = "/" + db_name + 
"/_design/erlview/_list/simple_list/simple_view";
   var xhr = CouchDB.request("GET", url);
diff --git a/test/javascript/tests/purge.js b/test/javascript/tests/purge.js
index 38eca8d..6bfba02 100644
--- a/test/javascript/tests/purge.js
+++ b/test/javascript/tests/purge.js
@@ -11,7 +11,6 @@
 // the License.
 
 couchTests.purge = function(debug) {
-  return console.log('TODO: this feature is not yet implemented');
   var db_name = get_random_db_name();
   var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"});
   db.createDb();
@@ -53,21 +52,13 @@ couchTests.purge = function(debug) {
   var xhr = CouchDB.request("POST", "/" + db_name + "/_purge", {
 body: JSON.stringify({"1":[doc1._rev], "2":[doc2._rev]})
   });
-  console.log(xhr.status);
-  console.log(xhr.responseText);
-  T(xhr.status == 200);
+  T(xhr.status == 201);
 
   var result = JSON.parse(xhr.responseText);
   var newInfo = db.info();
-  
-  // purging increments the update sequence
-  T(info.update_seq+1 == newInfo.update_seq);
-  // and it increments the purge_seq
-  T(info.purge_seq+1 == newInfo.purge_seq);
-  T(result.purge_seq == newInfo.purge_seq);
 
-  T(result.purged["1"][0] == doc1._rev);
-  T(result.purged["2"][0] == doc2._rev);
+  T(result.purged["1"].purged[0] == doc1._rev);
+  T(result.purged["2"].purged[0] == doc2._rev);
 
   T(db.open("1") == null);
   T(db.open("2") == null);
@@ -85,7 +76,6 @@ couchTests.purge = function(debug) {
   // compaction isn't instantaneous, loop until done
   while (db.info().compact_running) {};
   var compactInfo = db.info();
-  T(compactInfo.purge_seq == newInfo.purge_seq);
 
   // purge documents twice in a row without loading views
   // (causes full view rebuilds)
@@ -97,15 +87,14 @@ couchTests.purge = function(debug) {
 body: JSON.stringify({"3":[doc3._rev]})
   });
 
-  T(xhr.status == 200);
+  T(xhr.status == 201);
 
   xhr = CouchDB.request("POST", "/" + db_name + "/_purge", {
 body: JSON.stringify({"4":[doc4._rev]})
   });
 
-  T(xhr.status == 200);
+  T(xhr.status == 201);
   result = JSON.parse(xhr.responseText);
-  T(result.purge_seq == db.info().purge_seq);
 
   var rows = db.view("test/all_docs_twice").rows;
   for (var i = 4; i < numDocs; i++) {
@@ -129,7 +118,7 @@ couchTests.purge = function(debug) {
   var xhr = CouchDB.request("POST", "/" + dbB.name + "/_purge", {
 body: JSON.stringify({"test":[docA._rev]})
   });
-  TEquals(200, xhr.status, "single rev purge after replication succeeds");
+  TEquals(201, xhr.status, "single rev purge after replication succeeds");
 
   var xhr = CouchDB.request("GET", "/" + dbB.name + "/test?rev=" + docA._rev);
   TEquals(404, xhr.status, "single rev purge removes revision");
@@ -137,14 +126,14 @@ couchTests.purge = function(debug) {
   var xhr = CouchDB.request("POST", "/" + dbB.name + "/_purge", {
 body: JSON.stringify({"test":[docB._rev]})
   });
-  TEquals(200, xhr.status, "single rev purge after replication succeeds");
+  TEquals(201, xhr.status, "single rev purge after replication succeeds");
   var xhr = CouchDB.request("GET", "/" + dbB.name + "/test?rev=" + docB._rev);
   TEquals(404, xhr.status, "single rev purge removes revision");
 
   var xhr = CouchDB.request("POST", "/" + 

[couchdb] 11/19: Add EPI hook for creating purge docs on compaction

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 2a741eb407c6e83c3a62432e015f01de9253ea55
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 15:56:39 2018 -0500

Add EPI hook for creating purge docs on compaction
---
 src/couch/src/couch_bt_engine_compactor.erl | 18 ++
 src/couch/src/couch_db_plugin.erl   |  8 +++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/couch/src/couch_bt_engine_compactor.erl 
b/src/couch/src/couch_bt_engine_compactor.erl
index 6bb981e..370cae2 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -44,6 +44,12 @@ start(#st{} = St, DbName, Options, Parent) ->
 } = St,
 couch_log:debug("Compaction process spawned for db \"~s\"", [DbName]),
 
+{ok, DDocs} = design_docs(DbName),
+lists:map(fun(DDoc) ->
+JsonDDoc = couch_doc:from_json_obj(DDoc),
+couch_db_plugin:maybe_init_index_purge_state(DbName, JsonDDoc)
+end, DDocs),
+
 {ok, NewSt, DName, DFd, MFd, Retry} =
 open_compaction_files(Header, FilePath, Options),
 erlang:monitor(process, MFd),
@@ -582,3 +588,15 @@ update_compact_task(NumChanges) ->
 end,
 couch_task_status:update([{changes_done, Changes2}, {progress, Progress}]).
 
+
+design_docs(DbName) ->
+try
+case fabric:design_docs(mem3:dbname(DbName)) of
+{error, {maintenance_mode, _, _Node}} ->
+{ok, []};
+Else ->
+Else
+end
+catch error:database_does_not_exist ->
+{ok, []}
+end.
diff --git a/src/couch/src/couch_db_plugin.erl 
b/src/couch/src/couch_db_plugin.erl
index 740b812..0107403 100644
--- a/src/couch/src/couch_db_plugin.erl
+++ b/src/couch/src/couch_db_plugin.erl
@@ -18,7 +18,8 @@
 after_doc_read/2,
 validate_docid/1,
 check_is_admin/1,
-on_delete/2
+on_delete/2,
+maybe_init_index_purge_state/2
 ]).
 
 -define(SERVICE_ID, couch_db).
@@ -60,6 +61,11 @@ on_delete(DbName, Options) ->
 Handle = couch_epi:get_handle(?SERVICE_ID),
 couch_epi:apply(Handle, ?SERVICE_ID, on_delete, [DbName, Options], []).
 
+maybe_init_index_purge_state(DbName, DDoc) ->
+Handle = couch_epi:get_handle(?SERVICE_ID),
+couch_epi:apply(Handle, ?SERVICE_ID, maybe_init_index_purge_state,
+[DbName, DDoc], []).
+
 %% --
 %% Internal Function Definitions
 %% --

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 02/19: Fix bug during purge

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 21747f2429577ae34e6c167a3077e10cee7fe11a
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 11:36:49 2018 -0500

Fix bug during purge
---
 src/couch/src/couch_db_updater.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/couch/src/couch_db_updater.erl 
b/src/couch/src/couch_db_updater.erl
index a2de3bc..6a30d65 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -128,7 +128,7 @@ handle_call({purge_docs, IdRevs}, _From, Db) ->
 % If we purged every #leaf{} in the doc record
 % then we're removing it completely from the
 % database.
-FDIAcc;
+{FDIAcc, SeqAcc0};
 _ ->
 % Its possible to purge the #leaf{} that contains
 % the update_seq where this doc sits in the update_seq

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 08/19: Implement new node local purge APIs

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit d16071de595270b401100a2c395148896c9459b7
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:24:10 2018 -0500

Implement new node local purge APIs

Rewrite purge logic to use the new couch_db_engine purge APIs. This work
will allow for the new purge behaviors to enable clustered purge.
---
 src/couch/priv/stats_descriptions.cfg |  12 +++
 src/couch/src/couch_db.erl| 153 +--
 src/couch/src/couch_db_updater.erl| 164 +-
 src/couch/src/couch_httpd_db.erl  |  23 +++--
 4 files changed, 258 insertions(+), 94 deletions(-)

diff --git a/src/couch/priv/stats_descriptions.cfg 
b/src/couch/priv/stats_descriptions.cfg
index f091978..bceb0ce 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -34,6 +34,10 @@
 {type, counter},
 {desc, <<"number of times a document was read from a database">>}
 ]}.
+{[couchdb, database_purges], [
+{type, counter},
+{desc, <<"number of times a database was purged">>}
+]}.
 {[couchdb, db_open_time], [
 {type, histogram},
 {desc, <<"milliseconds required to open a database">>}
@@ -46,6 +50,10 @@
 {type, counter},
 {desc, <<"number of document write operations">>}
 ]}.
+{[couchdb, document_purges], [
+{type, counter},
+{desc, <<"number of document purge operations">>}
+]}.
 {[couchdb, local_document_writes], [
 {type, counter},
 {desc, <<"number of _local document write operations">>}
@@ -74,6 +82,10 @@
 {type, counter},
 {desc, <<"number of clients for continuous _changes">>}
 ]}.
+{[couchdb, httpd, purge_requests], [
+{type, counter},
+{desc, <<"number of purge requests">>}
+]}.
 {[couchdb, httpd_request_methods, 'COPY'], [
 {type, counter},
 {desc, <<"number of HTTP COPY requests">>}
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 3449274..8592193 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -43,7 +43,6 @@
 get_epochs/1,
 get_filepath/1,
 get_instance_start_time/1,
-get_last_purged/1,
 get_pid/1,
 get_revs_limit/1,
 get_security/1,
@@ -51,12 +50,15 @@
 get_user_ctx/1,
 get_uuid/1,
 get_purge_seq/1,
+get_oldest_purge_seq/1,
+get_purge_infos_limit/1,
 
 is_db/1,
 is_system_db/1,
 is_clustered/1,
 
 set_revs_limit/2,
+set_purge_infos_limit/2,
 set_security/2,
 set_user_ctx/2,
 
@@ -75,6 +77,9 @@
 get_full_doc_infos/2,
 get_missing_revs/2,
 get_design_docs/1,
+get_purge_infos/2,
+
+get_minimum_purge_seq/1,
 
 update_doc/3,
 update_doc/4,
@@ -84,6 +89,7 @@
 delete_doc/3,
 
 purge_docs/2,
+purge_docs/3,
 
 with_stream/3,
 open_write_stream/2,
@@ -97,6 +103,8 @@
 fold_changes/4,
 fold_changes/5,
 count_changes_since/2,
+fold_purge_infos/4,
+fold_purge_infos/5,
 
 calculate_start_seq/3,
 owner_of/2,
@@ -369,8 +377,132 @@ get_full_doc_info(Db, Id) ->
 get_full_doc_infos(Db, Ids) ->
 couch_db_engine:open_docs(Db, Ids).
 
-purge_docs(#db{main_pid=Pid}, IdsRevs) ->
-gen_server:call(Pid, {purge_docs, IdsRevs}).
+purge_docs(Db, IdRevs) ->
+purge_docs(Db, IdRevs, []).
+
+-spec purge_docs(#db{}, [{UUId, Id, [Rev]}], [PurgeOption]) ->
+{ok, [Reply]} when
+UUId :: binary(),
+Id :: binary(),
+Rev :: {non_neg_integer(), binary()},
+PurgeOption :: interactive_edit | replicated_changes,
+Reply :: {ok, []} | {ok, [Rev]}.
+purge_docs(#db{main_pid = Pid} = Db, UUIdsIdsRevs, Options) ->
+increment_stat(Db, [couchdb, database_purges]),
+gen_server:call(Pid, {purge_docs, UUIdsIdsRevs, Options}).
+
+-spec get_purge_infos(#db{}, [UUId]) -> [PurgeInfo] when
+UUId :: binary(),
+PurgeInfo :: {PurgeSeq, UUId, Id, [Rev]} | not_found,
+PurgeSeq :: non_neg_integer(),
+Id :: binary(),
+Rev :: {non_neg_integer(), binary()}.
+get_purge_infos(Db, UUIDs) ->
+couch_db_engine:load_purge_infos(Db, UUIDs).
+
+
+get_minimum_purge_seq(#db{} = Db) ->
+PurgeSeq = couch_db_engine:get_purge_seq(Db),
+OldestPurgeSeq = couch_db_engine:get_oldest_purge_seq(Db),
+PurgeInfosLimit = couch_db_engine:get_purge_infos_limit(Db),
+
+FoldFun = fun(#doc{id = DocId, body = {Props}}, SeqAcc) ->
+case DocId of
+<<"_local/purge-", _/binary>> ->
+ClientSeq = couch_util:get_value(<<"purge_seq">>, Props),
+case ClientSeq of
+CS when is_integer(CS), CS >= PurgeSeq - PurgeInfosLimit ->
+{ok, SeqAcc};
+CS when is_integer(CS) ->
+case purge_client_exists(Db, DocId, Props) of
+   

[couchdb] 14/19: Add clustered purge API to fabric

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit cee6c218c45ff27e53aec5334b40755049d9d432
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:27:43 2018 -0500

Add clustered purge API to fabric
---
 src/fabric/rebar.config |   4 +-
 src/fabric/src/fabric.erl   |  27 ++-
 src/fabric/src/fabric_db_info.erl   |  29 +--
 src/fabric/src/fabric_db_meta.erl   |  26 ++-
 src/fabric/src/fabric_doc_open.erl  |  42 ++--
 src/fabric/src/fabric_doc_purge.erl | 413 
 src/fabric/src/fabric_rpc.erl   |  98 -
 7 files changed, 600 insertions(+), 39 deletions(-)

diff --git a/src/fabric/rebar.config b/src/fabric/rebar.config
index 362c878..3f51af3 100644
--- a/src/fabric/rebar.config
+++ b/src/fabric/rebar.config
@@ -10,5 +10,5 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
-{cover_enabled, true}.
-{cover_print_enabled, true}.
+%{cover_enabled, true}.
+%{cover_print_enabled, true}.
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 4a07271..b2e71dc 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -21,12 +21,13 @@
 delete_db/2, get_db_info/1, get_doc_count/1, set_revs_limit/3,
 set_security/2, set_security/3, get_revs_limit/1, get_security/1,
 get_security/2, get_all_security/1, get_all_security/2,
+get_purge_infos_limit/1, set_purge_infos_limit/3,
 compact/1, compact/2]).
 
 % Documents
 -export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
 get_missing_revs/2, get_missing_revs/3, update_doc/3, update_docs/3,
-purge_docs/2, att_receiver/2]).
+purge_docs/3, att_receiver/2]).
 
 % Views
 -export([all_docs/4, all_docs/5, changes/4, query_view/3, query_view/4,
@@ -137,6 +138,18 @@ set_security(DbName, SecObj) ->
 set_security(DbName, SecObj, Options) ->
 fabric_db_meta:set_security(dbname(DbName), SecObj, opts(Options)).
 
+%% @doc sets the upper bound for the number of stored purge requests
+-spec set_purge_infos_limit(dbname(), pos_integer(), [option()]) -> ok.
+set_purge_infos_limit(DbName, Limit, Options)
+when is_integer(Limit), Limit > 0 ->
+fabric_db_meta:set_purge_infos_limit(dbname(DbName), Limit, opts(Options)).
+
+%% @doc retrieves the upper bound for the number of stored purge requests
+-spec get_purge_infos_limit(dbname()) -> pos_integer() | no_return().
+get_purge_infos_limit(DbName) ->
+{ok, Db} = fabric_util:get_db(dbname(DbName), [?ADMIN_CTX]),
+try couch_db:get_purge_infos_limit(Db) after catch couch_db:close(Db) end.
+
 get_security(DbName) ->
 get_security(DbName, [?ADMIN_CTX]).
 
@@ -267,8 +280,16 @@ update_docs(DbName, Docs, Options) ->
 {aborted, PreCommitFailures}
 end.
 
-purge_docs(_DbName, _IdsRevs) ->
-not_implemented.
+
+%% @doc purge revisions for a list '{Id, Revs}'
+%%  returns {ok, {PurgeSeq, Results}}
+-spec purge_docs(dbname(), [{docid(), [revision()]}], [option()]) ->
+{ok, [{Health, [revision()]}] | {error, any()}} when
+Health :: ok | accepted.
+purge_docs(DbName, IdsRevs, Options) when is_list(IdsRevs) ->
+IdsRevs2 = [idrevs(IdRs) || IdRs <- IdsRevs],
+fabric_doc_purge:go(dbname(DbName), IdsRevs2, opts(Options)).
+
 
 %% @doc spawns a process to upload attachment data and
 %%  returns a function that shards can use to communicate
diff --git a/src/fabric/src/fabric_db_info.erl 
b/src/fabric/src/fabric_db_info.erl
index 98e8e52..97a31c2 100644
--- a/src/fabric/src/fabric_db_info.erl
+++ b/src/fabric/src/fabric_db_info.erl
@@ -23,10 +23,12 @@ go(DbName) ->
 RexiMon = fabric_util:create_monitors(Shards),
 Fun = fun handle_message/3,
 {ok, ClusterInfo} = get_cluster_info(Shards),
-Acc0 = {fabric_dict:init(Workers, nil), [{cluster, ClusterInfo}]},
+Acc0 = {fabric_dict:init(Workers, nil), [], [{cluster, ClusterInfo}]},
 try
 case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of
-{ok, Acc} -> {ok, Acc};
+
+{ok, Acc} ->
+{ok, Acc};
 {timeout, {WorkersDict, _}} ->
 DefunctWorkers = fabric_util:remove_done_workers(
 WorkersDict,
@@ -37,44 +39,49 @@ go(DbName) ->
 "get_db_info"
 ),
 {error, timeout};
-{error, Error} -> throw(Error)
+{error, Error} ->
+throw(Error)
 end
 after
 rexi_monitor:stop(RexiMon)
 end.
 
-handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {Counters, Acc}) ->
+handle_message({rexi_DOWN,
+_, {_,NodeRef},_}, _Shard, {Counters, PseqAcc, Acc}) ->
 case fabric_util:remove_down_workers(Counters, NodeRef) of
 {ok, NewCounters} ->
-{ok, 

[couchdb] 09/19: Implement new purge APIs for couch_bt_enigne

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 57a8dc2aa6211a892f4b585ed61ead0763645700
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:25:43 2018 -0500

Implement new purge APIs for couch_bt_enigne
---
 src/couch/src/couch_bt_engine.erl   | 240 
 src/couch/src/couch_bt_engine.hrl   |   4 +-
 src/couch/src/couch_bt_engine_compactor.erl | 122 --
 src/couch/src/couch_bt_engine_header.erl|  35 ++--
 4 files changed, 334 insertions(+), 67 deletions(-)

diff --git a/src/couch/src/couch_bt_engine.erl 
b/src/couch/src/couch_bt_engine.erl
index 2583f10..f4d37aa 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -35,8 +35,9 @@
 get_disk_version/1,
 get_doc_count/1,
 get_epochs/1,
-get_last_purged/1,
 get_purge_seq/1,
+get_oldest_purge_seq/1,
+get_purge_infos_limit/1,
 get_revs_limit/1,
 get_security/1,
 get_size_info/1,
@@ -44,15 +45,18 @@
 get_uuid/1,
 
 set_revs_limit/2,
+set_purge_infos_limit/2,
 set_security/2,
 
 open_docs/2,
 open_local_docs/2,
 read_doc_body/2,
+load_purge_infos/2,
 
 serialize_doc/2,
 write_doc_body/2,
-write_doc_infos/4,
+write_doc_infos/3,
+purge_docs/3,
 
 commit_data/1,
 
@@ -63,6 +67,7 @@
 fold_docs/4,
 fold_local_docs/4,
 fold_changes/5,
+fold_purge_infos/5,
 count_changes_since/2,
 
 start_compaction/4,
@@ -85,7 +90,13 @@
 seq_tree_reduce/2,
 
 local_tree_split/1,
-local_tree_join/2
+local_tree_join/2,
+
+purge_tree_split/1,
+purge_tree_join/2,
+purge_tree_reduce/2,
+purge_seq_tree_split/1,
+purge_seq_tree_join/2
 ]).
 
 
@@ -217,18 +228,24 @@ get_epochs(#st{header = Header}) ->
 couch_bt_engine_header:get(Header, epochs).
 
 
-get_last_purged(#st{header = Header} = St) ->
-case couch_bt_engine_header:get(Header, purged_docs) of
-nil ->
-[];
-Pointer ->
-{ok, PurgeInfo} = couch_file:pread_term(St#st.fd, Pointer),
-PurgeInfo
-end.
+get_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) ->
+Fun = fun({PurgeSeq, _, _, _}, _Reds, _Acc) ->
+{stop, PurgeSeq}
+end,
+{ok, _, PurgeSeq} = couch_btree:fold(PurgeSeqTree, Fun, 0, [{dir, rev}]),
+PurgeSeq.
+
+
+get_oldest_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) ->
+Fun = fun({PurgeSeq, _, _, _}, _Reds, _Acc) ->
+{stop, PurgeSeq}
+end,
+{ok, _, PurgeSeq} = couch_btree:fold(PurgeSeqTree, Fun, 0, []),
+PurgeSeq.
 
 
-get_purge_seq(#st{header = Header}) ->
-couch_bt_engine_header:get(Header, purge_seq).
+get_purge_infos_limit(#st{header = Header}) ->
+couch_bt_engine_header:get(Header, purge_infos_limit).
 
 
 get_revs_limit(#st{header = Header}) ->
@@ -284,6 +301,16 @@ set_revs_limit(#st{header = Header} = St, RevsLimit) ->
 {ok, increment_update_seq(NewSt)}.
 
 
+set_purge_infos_limit(#st{header = Header} = St, PurgeInfosLimit) ->
+NewSt = St#st{
+header = couch_bt_engine_header:set(Header, [
+{purge_infos_limit, PurgeInfosLimit}
+]),
+needs_commit = true
+},
+{ok, increment_update_seq(NewSt)}.
+
+
 set_security(#st{header = Header} = St, NewSecurity) ->
 Options = [{compression, St#st.compression}],
 {ok, Ptr, _} = couch_file:append_term(St#st.fd, NewSecurity, Options),
@@ -320,6 +347,14 @@ read_doc_body(#st{} = St, #doc{} = Doc) ->
 }.
 
 
+load_purge_infos(St, UUIDs) ->
+Results = couch_btree:lookup(St#st.purge_tree, UUIDs),
+lists:map(fun
+({ok, Info}) -> Info;
+(not_found) -> not_found
+end, Results).
+
+
 serialize_doc(#st{} = St, #doc{} = Doc) ->
 Compress = fun(Term) ->
 case couch_compress:is_compressed(Term, St#st.compression) of
@@ -351,7 +386,7 @@ write_doc_body(St, #doc{} = Doc) ->
 {ok, Doc#doc{body = Ptr}, Written}.
 
 
-write_doc_infos(#st{} = St, Pairs, LocalDocs, PurgedIdRevs) ->
+write_doc_infos(#st{} = St, Pairs, LocalDocs) ->
 #st{
 id_tree = IdTree,
 seq_tree = SeqTree,
@@ -391,23 +426,9 @@ write_doc_infos(#st{} = St, Pairs, LocalDocs, 
PurgedIdRevs) ->
 erlang:max(Seq, Acc)
 end, get_update_seq(St), Add),
 
-NewHeader = case PurgedIdRevs of
-[] ->
-couch_bt_engine_header:set(St#st.header, [
-{update_seq, NewUpdateSeq}
-]);
-_ ->
-{ok, Ptr, _} = couch_file:append_term(St#st.fd, PurgedIdRevs),
-OldPurgeSeq = couch_bt_engine_header:get(St#st.header, purge_seq),
-% We bump NewUpdateSeq because we have to ensure that
-% indexers see that they need to process the new purge
-% information.
-

[couchdb] 12/19: Update the view engine to use the new purge APIs

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit f118eaee5ac239428cfc0aa670ee1828b28bc4ee
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:26:42 2018 -0500

Update the view engine to use the new purge APIs
---
 src/couch_index/src/couch_index_epi.erl|   5 +-
 ...dex_epi.erl => couch_index_plugin_couch_db.erl} |  39 +--
 src/couch_index/src/couch_index_updater.erl|  40 ++-
 src/couch_mrview/src/couch_mrview_cleanup.erl  |  18 +-
 src/couch_mrview/src/couch_mrview_index.erl| 133 +-
 src/couch_mrview/src/couch_mrview_test_util.erl|   5 +
 src/couch_mrview/src/couch_mrview_updater.erl  |  14 +-
 src/couch_mrview/src/couch_mrview_util.erl |  23 ++
 .../test/couch_mrview_purge_docs_fabric_tests.erl  | 171 +
 .../test/couch_mrview_purge_docs_tests.erl | 274 +
 10 files changed, 667 insertions(+), 55 deletions(-)

diff --git a/src/couch_index/src/couch_index_epi.erl 
b/src/couch_index/src/couch_index_epi.erl
index 946a590..1c4eb95 100644
--- a/src/couch_index/src/couch_index_epi.erl
+++ b/src/couch_index/src/couch_index_epi.erl
@@ -28,8 +28,9 @@ app() ->
 couch_index.
 
 providers() ->
-[].
-
+[
+{couch_db, couch_index_plugin_couch_db}
+].
 
 services() ->
 [
diff --git a/src/couch_index/src/couch_index_epi.erl 
b/src/couch_index/src/couch_index_plugin_couch_db.erl
similarity index 52%
copy from src/couch_index/src/couch_index_epi.erl
copy to src/couch_index/src/couch_index_plugin_couch_db.erl
index 946a590..3e3b711 100644
--- a/src/couch_index/src/couch_index_epi.erl
+++ b/src/couch_index/src/couch_index_plugin_couch_db.erl
@@ -2,7 +2,7 @@
 % use this file except in compliance with the License. You may obtain a copy of
 % the License at
 %
-% http://www.apache.org/licenses/LICENSE-2.0
+%   http://www.apache.org/licenses/LICENSE-2.0
 %
 % Unless required by applicable law or agreed to in writing, software
 % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
@@ -10,40 +10,15 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
--module(couch_index_epi).
-
--behaviour(couch_epi_plugin).
+-module(couch_index_plugin_couch_db).
 
 -export([
-app/0,
-providers/0,
-services/0,
-data_subscriptions/0,
-data_providers/0,
-processes/0,
-notify/3
+maybe_init_index_purge_state/2
 ]).
 
-app() ->
-couch_index.
-
-providers() ->
-[].
-
-
-services() ->
-[
-{couch_index, couch_index_plugin}
-].
-
-data_subscriptions() ->
-[].
-
-data_providers() ->
-[].
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
 
-processes() ->
-[].
 
-notify(_Key, _Old, _New) ->
-ok.
+maybe_init_index_purge_state(DbName, DDoc) ->
+couch_mrview_index:maybe_create_local_purge_doc(DbName, DDoc).
diff --git a/src/couch_index/src/couch_index_updater.erl 
b/src/couch_index/src/couch_index_updater.erl
index 5ab9ea8..4856c1d 100644
--- a/src/couch_index/src/couch_index_updater.erl
+++ b/src/couch_index/src/couch_index_updater.erl
@@ -141,12 +141,10 @@ update(Idx, Mod, IdxState) ->
 DbUpdateSeq = couch_db:get_update_seq(Db),
 DbCommittedSeq = couch_db:get_committed_update_seq(Db),
 
-PurgedIdxState = case purge_index(Db, Mod, IdxState) of
-{ok, IdxState0} -> IdxState0;
-reset -> exit({reset, self()})
-end,
-
-NumChanges = couch_db:count_changes_since(Db, CurrSeq),
+NumUpdateChanges = couch_db:count_changes_since(Db, CurrSeq),
+NumPurgeChanges = count_pending_purged_docs_since(Db, Mod, IdxState),
+TotalChanges = NumUpdateChanges + NumPurgeChanges,
+{ok, PurgedIdxState} = purge_index(Db, Mod, IdxState),
 
 GetSeq = fun
 (#full_doc_info{update_seq=Seq}) -> Seq;
@@ -185,8 +183,13 @@ update(Idx, Mod, IdxState) ->
 {ok, {NewSt, true}}
 end
 end,
+{ok, InitIdxState} = Mod:start_update(
+Idx,
+PurgedIdxState,
+TotalChanges,
+NumPurgeChanges
+),
 
-{ok, InitIdxState} = Mod:start_update(Idx, PurgedIdxState, NumChanges),
 Acc0 = {InitIdxState, true},
 {ok, Acc} = couch_db:fold_changes(Db, CurrSeq, Proc, Acc0, []),
 {ProcIdxSt, SendLast} = Acc,
@@ -209,11 +212,24 @@ purge_index(Db, Mod, IdxState) ->
 {ok, DbPurgeSeq} = couch_db:get_purge_seq(Db),
 IdxPurgeSeq = Mod:get(purge_seq, IdxState),
 if
-DbPurgeSeq == IdxPurgeSeq ->
+IdxPurgeSeq == DbPurgeSeq ->
 {ok, IdxState};
-DbPurgeSeq == IdxPurgeSeq + 1 ->
-{ok, PurgedIdRevs} = 

[couchdb] 05/19: Rewrite the PSE test suite to use couch_server

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 10169fd7631903767c8e56e1e0ce592bbc12bec6
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 11:41:01 2018 -0500

Rewrite the PSE test suite to use couch_server

It turns out that if any storage engine has to open itself during
a callback it would end up violating the guarantee of a single writer.
This change in the test suite changes things to use couch_server so that
storage engines are now free to do as they want reopening themselves.
---
 src/couch/src/test_engine_attachments.erl   |  35 ++-
 src/couch/src/test_engine_compaction.erl| 154 -
 src/couch/src/test_engine_fold_changes.erl  | 134 
 src/couch/src/test_engine_fold_docs.erl |  99 +++---
 src/couch/src/test_engine_get_set_props.erl |  84 +++--
 src/couch/src/test_engine_open_close_delete.erl |  75 +++--
 src/couch/src/test_engine_purge_docs.erl| 140 -
 src/couch/src/test_engine_read_write_docs.erl   | 300 +-
 src/couch/src/test_engine_ref_counting.erl  |  42 +--
 src/couch/src/test_engine_util.erl  | 400 ++--
 src/couch/test/couch_bt_engine_tests.erl|   2 +-
 11 files changed, 716 insertions(+), 749 deletions(-)

diff --git a/src/couch/src/test_engine_attachments.erl 
b/src/couch/src/test_engine_attachments.erl
index 691d4bd..9763ef5 100644
--- a/src/couch/src/test_engine_attachments.erl
+++ b/src/couch/src/test_engine_attachments.erl
@@ -19,25 +19,26 @@
 
 
 cet_write_attachment() ->
-{ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath),
+{ok, Db1} = test_engine_util:create_db(),
 
 AttBin = crypto:strong_rand_bytes(32768),
 
 try
-[Att0] = test_engine_util:prep_atts(Engine, St1, [
+[Att0] = test_engine_util:prep_atts(Db1, [
 {<<"ohai.txt">>, AttBin}
 ]),
 
 {stream, Stream} = couch_att:fetch(data, Att0),
-?assertEqual(true, Engine:is_active_stream(St1, Stream)),
+?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)),
 
-Actions = [{create, {<<"first">>, [], [Att0]}}],
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions),
-{ok, St3} = Engine:commit_data(St2),
-Engine:terminate(normal, St3),
+Actions = [{create, {<<"first">>, {[]}, [Att0]}}],
+{ok, Db2} = test_engine_util:apply_actions(Db1, Actions),
+{ok, _} = couch_db:ensure_full_commit(Db2),
+test_engine_util:shutdown_db(Db2),
 
-{ok, St4} = Engine:init(DbPath, []),
-[FDI] = Engine:open_docs(St4, [<<"first">>]),
+{ok, Db3} = couch_db:reopen(Db2),
+
+[FDI] = couch_db_engine:open_docs(Db3, [<<"first">>]),
 
 #rev_info{
 rev = {RevPos, PrevRevId},
@@ -52,12 +53,12 @@ cet_write_attachment() ->
 body = DocPtr
 },
 
-Doc1 = Engine:read_doc_body(St4, Doc0),
+Doc1 = couch_db_engine:read_doc_body(Db3, Doc0),
 Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true ->
 couch_compress:decompress(Doc1#doc.atts)
 end,
 
-StreamSrc = fun(Sp) -> Engine:open_read_stream(St4, Sp) end,
+StreamSrc = fun(Sp) -> couch_db_engine:open_read_stream(Db3, Sp) end,
 [Att1] = [couch_att:from_disk_term(StreamSrc, T) || T <- Atts1],
 ReadBin = couch_att:to_binary(Att1),
 ?assertEqual(AttBin, ReadBin)
@@ -72,22 +73,22 @@ cet_write_attachment() ->
 % we ever have something that stores attachemnts in
 % an external object store)
 cet_inactive_stream() ->
-{ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath),
+{ok, Db1} = test_engine_util:create_db(),
 
 AttBin = crypto:strong_rand_bytes(32768),
 
 try
-[Att0] = test_engine_util:prep_atts(Engine, St1, [
+[Att0] = test_engine_util:prep_atts(Db1, [
 {<<"ohai.txt">>, AttBin}
 ]),
 
 {stream, Stream} = couch_att:fetch(data, Att0),
-?assertEqual(true, Engine:is_active_stream(St1, Stream)),
+?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)),
 
-Engine:terminate(normal, St1),
-{ok, St2} = Engine:init(DbPath, []),
+test_engine_util:shutdown_db(Db1),
+{ok, Db2} = couch_db:reopen(Db1),
 
-?assertEqual(false, Engine:is_active_stream(St2, Stream))
+?assertEqual(false, couch_db_engine:is_active_stream(Db2, Stream))
 catch throw:not_supported ->
 ok
 end.
diff --git a/src/couch/src/test_engine_compaction.erl 
b/src/couch/src/test_engine_compaction.erl
index 09a1e4e..44c5357 100644
--- a/src/couch/src/test_engine_compaction.erl
+++ b/src/couch/src/test_engine_compaction.erl
@@ -19,66 +19,75 @@
 
 
 

[couchdb] branch COUCHDB-3326-clustered-purge-davisp-refactor-2 updated (0600be7 -> f0b7ddb)

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


 discard 0600be7  TMP - avoid couch_db:open_int/2 in PSE callbacks
 discard 77023f1  Update HTTP JS tests for clustered purge
 discard c762372  Implement HTTP endpoints for clustered purge
 discard 3daa54c  Fix read-repair for new clustered purge
 discard e9f7d65  Add clustered purge API to fabric
 discard 1563f51  Implement internal replication of purge requests
 discard f12b2b1  Update the view engine to use the new purge APIs
 discard 073b583  Add EPI hook for creating purge docs on compaction
 discard 54edbd2  Update the purge eunit test suites
 discard e1c8d8a  Implement new purge APIs for couch_bt_enigne
 discard 5a152ca  Implement new node local purge APIs
 discard 3d88197  Define new couch_db_engine purge API
 discard b05d6f4  Simplify logic in mem3_rep
 discard 14c678b  Fix typos in couch_db_engine.erl
 add 18f8362  Bump config dependency to 1.0.3
 add 36ecf92  Support queries for endpoints
 add 0a477b5  Merge pull request #1222 from cloudant/issue-820-add-queries
 add 95a78ce  Remove _config call in couch_peruser_test
 add a0c863d  Merge pull request #1130 from 
cloudant/issue-876-remove-_config-call-in-eunit-test
 add 89a727b  Replace resource expensive bcrypt test with shorter version 
(#1231)
 add 45da9f3  Validate password_scheme in user doc
 add 3d702d8  Revert "Revert "re-enable "flaky" test in quest to nail down 
#745""
 add e7c48b3  Improve 413 response handling
 add f0887c1  Allow couch_os_daemons to live in directories with spaces
 add 7bfdedb  Fix DB-specific compaction configuration (#1059)
 add 6f987ae  Merge branch 'master' into daemon-spaces
 add f28d896  make it so
 add 3621725  add bootstrap
 add 0f559a9  add ignore
 add 58c4948  add http stub
 add e8c4966  add basic action handling
 add a5213f7  add Apache License stanza everywhere
 add 404692f  add the plan to readme
 add ecf310a  add note about skipping a step if the node is already setup
 add 38eaa88  add delete_node API
 add 9f1fa23  hack for storing erlang cookie value on new nodes
 add 068bdf1  add action hints
 add 94eab12  add license
 add 3ad82e5  remove leftover
 add 317e5a4  formatting
 add 0145bae  formatting & clarification
 add bc41677  mroe formatting
 add 277ca66  wip: implement setup handling
 add 92da54e  wip: full receive feature, setup now works yay
 add fc39fab  add simple test script
 add 354647b  add finish cluster routine
 add 7c6c3bb  add some more testing
 add 4c423e6  s/_cassim/cassim/ for the time being
 add 7528f5b  add license header
 add 0a676fc  add testing instructions to readme
 add 3304add  hash admin passwords, more resilient port parsing
 add 14e0374  handle GET cluster state
 add 9c3eb0a  show cluster finished state
 add be52f7e  R14 compatibility
 add 9728b34  Remove error-handling clause
 add cd7d0ec  Fix LICENSE indention
 add deeb073  Rename cassim db to _metadata
 add 127e85a  Use _nodes db
 add 372dd8b  fix tests
 add ecb601b  Create _global_changes database on cluster setup
 add 616789b  cluster_enable: add remote_node feature
 add f4fd3fa  whitespace fix
 add aa17a55  use couch_log instead of io:format
 add 5c0e927  Use dynamic handlers
 add ff19be1  add catch-all clause for url_handler
 add bdb8a0c  configure the right http interface
 add 647ffbc  fix enable_cluster_http for admin-party clusters
 add fb61c04  Update to new couch_epi API
 add d0a9b72  Pass supervisor's children to couch_epi
 add 747144e  Return HTTP 200 on GET
 add b9e1f3b  Return HTTP 405 for unsupported request method
 add e8d1e32  feat: cassim is off for now
 add 75a7682  require nodecount on setup
 add dd68945  use config:setineger/3
 add b107042  fix wording
 add 401d776  Merge remote-tracking branch 
'robertkowalski/2594-2598-number-of-nodes'
 add 2590fbc  Fixed some minor errors in the documentation.
 add d75693e  add_node: Don't fail if node name != "couchdb" or "node1"
 add b2b93c1  Merge remote-tracking branch 'adrienverge/COUCHDB-3119'
 add 54623ce  fix cluster setup: use same admin pq salt on all nodes
 add c38d7aa  Merge remote-tracking branch 'asf/salt-distribution'
 add 18314a6  Add support for new ensure_dbs_exist option to GET, 
POST/finish_cluster
 add 92dd9d1  Add new enable_single_node action for cluster_setup endpoint
 add e153d48  address comments from rnewson
 add d61381a  fix typo/compilation error
 add 942c665  chore: whitespace
 add 4b90eca  chore: better log output
 add 4d9bd58  Merge branch '593-setup-single-node' of 
https://github.com/apache/couchdb-setup
 add 68545af  fix: make 

[couchdb] 04/19: Fix race on couch_db:reopen/1

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 9b1c4930102d35ff727351f377ca7a915e1810d6
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 11:39:58 2018 -0500

Fix race on couch_db:reopen/1

This fixes a minor race by opening the database before closing it. This
was never found to be an issue in production and was just caught while
contemplating the PSE test suite.
---
 src/couch/src/couch_db.erl | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 93ea07e..3449274 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -161,8 +161,11 @@ reopen(#db{} = Db) ->
 % We could have just swapped out the storage engine
 % for this database during a compaction so we just
 % reimplement this as a close/open pair now.
-close(Db),
-open(Db#db.name, [{user_ctx, Db#db.user_ctx} | Db#db.options]).
+try
+open(Db#db.name, [{user_ctx, Db#db.user_ctx} | Db#db.options])
+after
+close(Db)
+end.
 
 
 % You shouldn't call this. Its part of the ref counting between

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 18/19: TMP - avoid couch_db:open_int/2 in PSE callbacks

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 53356ba32ac6394e1e3d2d4e049e68a8eb77555e
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 16:17:50 2018 -0500

TMP - avoid couch_db:open_int/2 in PSE callbacks
---
 src/couch/src/couch_bt_engine_compactor.erl | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/couch/src/couch_bt_engine_compactor.erl 
b/src/couch/src/couch_bt_engine_compactor.erl
index 370cae2..697aed5 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -104,9 +104,10 @@ open_compaction_files(SrcHdr, DbFilePath, Options) ->
 
 
 copy_purge_info(DbName, OldSt, NewSt, Retry) ->
-MinPurgeSeq = couch_util:with_db(DbName, fun(Db) ->
-couch_db:get_minimum_purge_seq(Db)
-end),
+%MinPurgeSeq = couch_util:with_db(DbName, fun(Db) ->
+%couch_db:get_minimum_purge_seq(Db)
+%end),
+MinPurgeSeq = 0,
 OldPSTree = OldSt#st.purge_seq_tree,
 StartSeq = couch_bt_engine:get_purge_seq(NewSt) + 1,
 BufferSize = config:get_integer(

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 07/19: Define new couch_db_engine purge API

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 78d7e1f64fa03113ee2bac34f75d58afd895575c
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:23:16 2018 -0500

Define new couch_db_engine purge API
---
 src/couch/src/couch_db_engine.erl | 162 ++
 1 file changed, 130 insertions(+), 32 deletions(-)

diff --git a/src/couch/src/couch_db_engine.erl 
b/src/couch/src/couch_db_engine.erl
index 4974201..5ac9865 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -22,6 +22,8 @@
 -type rev() :: {non_neg_integer(), binary()}.
 -type revs() :: [rev()].
 -type json() :: any().
+-type uuid() :: binary().
+-type purge_seq() :: non_neg_integer().
 
 -type doc_pair() :: {
 #full_doc_info{} | not_found,
@@ -39,7 +41,7 @@
 sync
 ].
 
--type purge_info() :: [{docid(), revs()}].
+-type purge_info() :: {purge_seq(), uuid(), docid(), revs()}.
 -type epochs() :: [{Node::atom(), UpdateSeq::non_neg_integer()}].
 -type size_info() :: [{Name::atom(), Size::non_neg_integer()}].
 
@@ -62,6 +64,10 @@
 {dir, fwd | rev}
 ].
 
+-type purge_fold_options() :: [
+% Need to enumerate these
+].
+
 -type db_handle() :: any().
 
 -type doc_fold_fun() :: fun((#full_doc_info{}, UserAcc::any()) ->
@@ -76,6 +82,10 @@
 {ok, NewUserAcc::any()} |
 {stop, NewUserAcc::any()}).
 
+-type purge_fold_fun() :: fun((purge_info(), UserAcc::any()) ->
+{ok, NewUserAcc::any()} |
+{stop, NewUserAcc::any()}).
+
 
 % This is called by couch_server to determine which
 % engine should be used for the given database. DbPath
@@ -206,13 +216,18 @@
 -callback get_epochs(DbHandle::db_handle()) -> Epochs::epochs().
 
 
-% Get the last purge request performed.
--callback get_last_purged(DbHandle::db_handle()) -> LastPurged::purge_info().
+% Get the current purge sequence known to the engine. This
+% value should be updated during calls to purge_docs.
+-callback get_purge_seq(DbHandle::db_handle()) -> purge_seq().
 
 
-% Get the current purge sequence. This should be incremented
-% for every purge operation.
--callback get_purge_seq(DbHandle::db_handle()) -> PurgeSeq::non_neg_integer().
+% Get the oldest purge sequence known to the engine
+-callback get_oldest_purge_seq(DbHandle::db_handle()) -> purge_seq().
+
+
+% Get the purged infos limit. This should just return the last
+% value that was passed to set_purged_docs_limit/2.
+-callback get_purge_infos_limit(DbHandle::db_handle()) -> pos_integer().
 
 
 % Get the revision limit. This should just return the last
@@ -261,6 +276,11 @@
 -callback set_revs_limit(DbHandle::db_handle(), RevsLimit::pos_integer()) ->
 {ok, NewDbHandle::db_handle()}.
 
+
+-callback set_purge_infos_limit(DbHandle::db_handle(), Limit::pos_integer()) ->
+{ok, NewDbHandle::db_handle()}.
+
+
 -callback set_security(DbHandle::db_handle(), SecProps::any()) ->
 {ok, NewDbHandle::db_handle()}.
 
@@ -301,6 +321,15 @@
 doc().
 
 
+% This function will be called from many contexts concurrently.
+% If the storage engine has a purge_info() record for any of the
+% provided UUIDs, those purge_info() records should be returned. The
+% resulting list should have the same length as the input list of
+% UUIDs.
+-callback load_purge_infos(DbHandle::db_handle(), [uuid()]) ->
+[purge_info() | not_found].
+
+
 % This function is called concurrently by any client process
 % that is writing a document. It should accept a #doc{}
 % record and return a #doc{} record with a mutated body it
@@ -341,31 +370,20 @@
 % #full_doc_info{} records. The first element of the pair is
 % the #full_doc_info{} that exists on disk. The second element
 % is the new version that should be written to disk. There are
-% three basic cases that should be followed:
+% two basic cases that should be followed:
 %
 % 1. {not_found, #full_doc_info{}} - A new document was created
 % 2. {#full_doc_info{}, #full_doc_info{}} - A document was updated
-% 3. {#full_doc_info{}, not_found} - A document was purged completely
 %
-% Number one and two are fairly straight forward as long as proper
-% accounting for moving entries in the udpate sequence are accounted
-% for. However, case 3 you'll notice is "purged completely" which
-% means it needs to be removed from the database including the
-% update sequence. Also, for engines that are not using append
-% only storage like the legacy engine, case 2 can be the result of
-% a purge so special care will be needed to see which revisions
-% should be removed.
+% The cases are fairly straight forward as long as proper
+% accounting for moving entries in the update sequence are accounted
+% for.
 %
 % The LocalDocs variable is applied separately. Its important to
 % note for new storage 

[couchdb] 10/19: Update the purge eunit test suites

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 4782ab5caecd229a0fc2443b8b87ac981d2458f9
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:26:01 2018 -0500

Update the purge eunit test suites
---
 src/couch/src/test_engine_compaction.erl| 197 +-
 src/couch/src/test_engine_fold_purge_infos.erl  | 133 +++
 src/couch/src/test_engine_get_set_props.erl |   2 +
 src/couch/src/test_engine_purge_docs.erl|  40 +-
 src/couch/src/test_engine_util.erl  |  19 +-
 src/couch/test/couch_db_purge_docs_tests.erl| 497 
 src/couch/test/couch_db_purge_seqs_tests.erl| 217 +++
 src/couch/test/couch_db_purge_upgrade_tests.erl |  74 
 8 files changed, 1158 insertions(+), 21 deletions(-)

diff --git a/src/couch/src/test_engine_compaction.erl 
b/src/couch/src/test_engine_compaction.erl
index 44c5357..e49167a 100644
--- a/src/couch/src/test_engine_compaction.erl
+++ b/src/couch/src/test_engine_compaction.erl
@@ -93,10 +93,8 @@ cet_compact_with_everything() ->
 BarRev = test_engine_util:prev_rev(BarFDI),
 
 Actions3 = [
-{batch, [
-{purge, {<<"foo">>, FooRev#rev_info.rev}},
-{purge, {<<"bar">>, BarRev#rev_info.rev}}
-]}
+{purge, {<<"foo">>, FooRev#rev_info.rev}},
+{purge, {<<"bar">>, BarRev#rev_info.rev}}
 ],
 
 {ok, Db4} = test_engine_util:apply_actions(Db3, Actions3),
@@ -106,10 +104,9 @@ cet_compact_with_everything() ->
 {<<"foo">>, [FooRev#rev_info.rev]}
 ],
 
-?assertEqual(
-PurgedIdRevs,
-lists:sort(couch_db_engine:get_last_purged(Db4))
-),
+{ok, PIdRevs4} = couch_db_enigne:fold_purge_infos(
+Db4, 0, fun fold_fun/2, [], []),
+?assertEqual(PurgedIdRevs, PIdRevs4),
 
 {ok, Db5} = try
 [Att0, Att1, Att2, Att3, Att4] = test_engine_util:prep_atts(Db4, [
@@ -179,6 +176,186 @@ cet_recompact_updates() ->
 ?assertEqual(nodiff, Diff).
 
 
+cet_purge_during_compact() ->
+{ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath),
+
+Actions1 = [
+{create, {<<"foo">>, []}},
+{create, {<<"bar">>, []}},
+{conflict, {<<"bar">>, [{<<"vsn">>, 2}]}},
+{create, {<<"baz">>, []}}
+],
+
+{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
+{ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path),
+
+[BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]),
+BarRev = test_engine_util:prev_rev(BarFDI),
+BazRev = test_engine_util:prev_rev(BazFDI),
+Actions2 = [
+{purge, {<<"bar">>, BarRev#rev_info.rev}},
+{purge, {<<"baz">>, BazRev#rev_info.rev}}
+],
+{ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2),
+Db1 = test_engine_util:db_as_term(Engine, St4),
+
+{ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term),
+
+?assertEqual(true, is_pid(NewPid)),
+Ref = erlang:monitor(process, NewPid),
+
+NewTerm = receive
+{'$gen_cast', {compact_done, Engine, Term0}} ->
+Term0;
+{'DOWN', Ref, _, _, Reason} ->
+erlang:error({compactor_died, Reason})
+after 1 ->
+erlang:error(compactor_timed_out)
+end,
+
+{ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm),
+Db2 = test_engine_util:db_as_term(Engine, St6),
+Diff = test_engine_util:term_diff(Db1, Db2),
+?assertEqual(nodiff, Diff).
+
+
+cet_multiple_purge_during_compact() ->
+{ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath),
+
+Actions1 = [
+{create, {<<"foo">>, []}},
+{create, {<<"bar">>, []}},
+{conflict, {<<"bar">>, [{<<"vsn">>, 2}]}},
+{create, {<<"baz">>, []}}
+],
+
+{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
+{ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path),
+
+[BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]),
+BarRev = test_engine_util:prev_rev(BarFDI),
+Actions2 = [
+{purge, {<<"bar">>, BarRev#rev_info.rev}}
+],
+{ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2),
+
+BazRev = test_engine_util:prev_rev(BazFDI),
+Actions3 = [
+{purge, {<<"baz">>, BazRev#rev_info.rev}}
+],
+{ok, St5} = test_engine_util:apply_actions(Engine, St4, Actions3),
+
+Db1 = test_engine_util:db_as_term(Engine, St5),
+{ok, St6, NewPid} = Engine:finish_compaction(St5, DbName, [], Term),
+
+?assertEqual(true, is_pid(NewPid)),
+Ref = erlang:monitor(process, NewPid),
+
+NewTerm = receive
+{'$gen_cast', {compact_done, Engine, Term0}} ->
+Term0;
+{'DOWN', Ref, _, _, Reason} ->
+

[couchdb] 15/19: Fix read-repair for new clustered purge

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 135e6bcac30208ea31a8b14d8587829057a0ad9a
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:27:32 2018 -0500

Fix read-repair for new clustered purge
---
 src/fabric/src/fabric_doc_open.erl  |  40 +++
 src/fabric/src/fabric_doc_open_revs.erl | 146 +
 src/fabric/src/fabric_rpc.erl   | 187 
 3 files changed, 241 insertions(+), 132 deletions(-)

diff --git a/src/fabric/src/fabric_doc_open.erl 
b/src/fabric/src/fabric_doc_open.erl
index b974880..0f8e377 100644
--- a/src/fabric/src/fabric_doc_open.erl
+++ b/src/fabric/src/fabric_doc_open.erl
@@ -25,8 +25,8 @@
 r,
 state,
 replies,
-q_reply,
-replies_by_node=[] %[{Node, Reply}] used for checking if a doc is purged
+node_id_revs = [],
+q_reply
 }).
 
 
@@ -84,8 +84,13 @@ handle_message({rexi_EXIT, _Reason}, Worker, Acc) ->
 end;
 handle_message(Reply, Worker, Acc) ->
 NewReplies = fabric_util:update_counter(Reply, 1, Acc#acc.replies),
-NewNReplies = [{Worker#shard.node, Reply}|Acc#acc.replies_by_node],
-NewAcc = Acc#acc{replies = NewReplies, replies_by_node = NewNReplies},
+NewNodeIdRevs = case Reply of
+{ok, #doc{id = Id, revs = {Pos, [Rev | _]}}} ->
+[{Worker#shard.node, {Id, [{Pos, Rev}]}} | Acc#acc.node_id_revs];
+_ ->
+Acc#acc.node_id_revs
+end,
+NewAcc = Acc#acc{replies = NewReplies, node_id_revs = NewNodeIdRevs},
 case is_r_met(Acc#acc.workers, NewReplies, Acc#acc.r) of
 {true, QuorumReply} ->
 fabric_util:cleanup(lists:delete(Worker, Acc#acc.workers)),
@@ -124,15 +129,14 @@ is_r_met(Workers, Replies, R) ->
 no_more_workers
 end.
 
-read_repair(#acc{dbname=DbName, replies=Replies, replies_by_node=NReplies0}) ->
+read_repair(#acc{dbname=DbName, replies=Replies, node_id_revs=NodeIdRevs}) ->
 Docs = [Doc || {_, {{ok, #doc{}=Doc}, _}} <- Replies],
-NReplies = [{Node, Doc} || {Node, {ok, #doc{}=Doc}} <- NReplies0],
 case Docs of
 % omit local docs from read repair
 [#doc{id = <>} | _] ->
 choose_reply(Docs);
 [#doc{id=Id} | _] ->
-Opts = [replicated_changes, ?ADMIN_CTX, {read_repair, NReplies}],
+Opts = [replicated_changes, ?ADMIN_CTX, {read_repair, NodeIdRevs}],
 Res = fabric:update_docs(DbName, Docs, Opts),
 case Res of
 {ok, []} ->
@@ -323,7 +327,7 @@ handle_message_reply_test() ->
 {ok, Acc0#acc{
 workers=[Worker0, Worker1],
 replies=[fabric_util:kv(foo,1)],
-replies_by_node=[{undefined, foo}]
+node_id_revs=[]
 }},
 handle_message(foo, Worker2, Acc0)
 ),
@@ -332,7 +336,7 @@ handle_message_reply_test() ->
 {ok, Acc0#acc{
 workers=[Worker0, Worker1],
 replies=[fabric_util:kv(bar,1), fabric_util:kv(foo,1)],
-replies_by_node=[{undefined, bar}]
+node_id_revs=[]
 }},
 handle_message(bar, Worker2, Acc0#acc{
 replies=[fabric_util:kv(foo,1)]
@@ -344,8 +348,7 @@ handle_message_reply_test() ->
 % is returned. Bit subtle on the assertions here.
 
 ?assertEqual(
-{stop, Acc0#acc{workers=[],replies=[fabric_util:kv(foo,1)],
-replies_by_node=[{undefined, foo}]}},
+{stop, Acc0#acc{workers=[],replies=[fabric_util:kv(foo,1)]}},
 handle_message(foo, Worker0, Acc0#acc{workers=[Worker0]})
 ),
 
@@ -353,12 +356,12 @@ handle_message_reply_test() ->
 {stop, Acc0#acc{
 workers=[],
 replies=[fabric_util:kv(bar,1), fabric_util:kv(foo,1)],
-replies_by_node =[{undefined, bar}, {undefined, foo}]
+ node_id_revs =[{undefined, foo}]
 }},
 handle_message(bar, Worker0, Acc0#acc{
 workers=[Worker0],
 replies=[fabric_util:kv(foo,1)],
-replies_by_node=[{undefined, foo}]
+node_id_revs=[{undefined, foo}]
 })
 ),
 
@@ -371,12 +374,12 @@ handle_message_reply_test() ->
 replies=[fabric_util:kv(foo,2)],
 state=r_met,
 q_reply=foo,
-replies_by_node =[{undefined, foo}, {undefined, foo}]
+node_id_revs =[{undefined, foo}]
 }},
 handle_message(foo, Worker1, Acc0#acc{
 workers=[Worker0, Worker1],
 replies=[fabric_util:kv(foo,1)],
-replies_by_node =[{undefined, foo}]
+node_id_revs =[{undefined, foo}]
 })
 ),
 
@@ -387,7 +390,7 @@ handle_message_reply_test() ->
 replies=[fabric_util:kv(foo,1)],
 state=r_met,
 q_reply=foo,
-replies_by_node =[{undefined, foo}]
+

[couchdb] 19/19: Update tests for new PSE test suite

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit f0b7ddb8cd9270b7493ba51244aced810e263d36
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 13:57:52 2018 -0500

Update tests for new PSE test suite
---
 src/couch/src/test_engine_compaction.erl   | 181 +
 src/couch/src/test_engine_fold_purge_infos.erl |  74 +-
 src/couch/src/test_engine_get_set_props.erl|   1 -
 src/couch/src/test_engine_purge_docs.erl   |   4 +-
 src/couch/src/test_engine_util.erl |   9 +-
 5 files changed, 113 insertions(+), 156 deletions(-)

diff --git a/src/couch/src/test_engine_compaction.erl 
b/src/couch/src/test_engine_compaction.erl
index e49167a..26714a1 100644
--- a/src/couch/src/test_engine_compaction.erl
+++ b/src/couch/src/test_engine_compaction.erl
@@ -104,7 +104,7 @@ cet_compact_with_everything() ->
 {<<"foo">>, [FooRev#rev_info.rev]}
 ],
 
-{ok, PIdRevs4} = couch_db_enigne:fold_purge_infos(
+{ok, PIdRevs4} = couch_db_engine:fold_purge_infos(
 Db4, 0, fun fold_fun/2, [], []),
 ?assertEqual(PurgedIdRevs, PIdRevs4),
 
@@ -177,180 +177,131 @@ cet_recompact_updates() ->
 
 
 cet_purge_during_compact() ->
-{ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath),
+{ok, Db1} = test_engine_util:create_db(),
 
-Actions1 = [
-{create, {<<"foo">>, []}},
-{create, {<<"bar">>, []}},
-{conflict, {<<"bar">>, [{<<"vsn">>, 2}]}},
-{create, {<<"baz">>, []}}
+Actions1 = lists:map(fun(Seq) ->
+{create, {docid(Seq), {[{<<"int">>, Seq}]}}}
+end, lists:seq(1, 1000)),
+Actions2 = [
+{create, {<<"foo">>, {[]}}},
+{create, {<<"bar">>, {[]}}},
+{create, {<<"baz">>, {[]}}}
 ],
+{ok, Db2} = test_engine_util:apply_batch(Db1, Actions1 ++ Actions2),
+Actions3 = [
+{conflict, {<<"bar">>, {[{<<"vsn">>, 2}]}}}
+],
+{ok, Db3} = test_engine_util:apply_actions(Db2, Actions3),
 
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
-{ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path),
+{ok, Pid} = couch_db:start_compact(Db3),
+catch erlang:suspend_process(Pid),
 
-[BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]),
+[BarFDI, BazFDI] = couch_db_engine:open_docs(Db3, [<<"bar">>, <<"baz">>]),
 BarRev = test_engine_util:prev_rev(BarFDI),
 BazRev = test_engine_util:prev_rev(BazFDI),
-Actions2 = [
+Actions4 = [
 {purge, {<<"bar">>, BarRev#rev_info.rev}},
 {purge, {<<"baz">>, BazRev#rev_info.rev}}
 ],
-{ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2),
-Db1 = test_engine_util:db_as_term(Engine, St4),
 
-{ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term),
+{ok, Db4} = test_engine_util:apply_actions(Db3, Actions4),
+Term1 = test_engine_util:db_as_term(Db4),
 
-?assertEqual(true, is_pid(NewPid)),
-Ref = erlang:monitor(process, NewPid),
+catch erlang:resume_process(Pid),
+test_engine_util:compact(Db4),
 
-NewTerm = receive
-{'$gen_cast', {compact_done, Engine, Term0}} ->
-Term0;
-{'DOWN', Ref, _, _, Reason} ->
-erlang:error({compactor_died, Reason})
-after 1 ->
-erlang:error(compactor_timed_out)
-end,
+{ok, Db5} = couch_db:reopen(Db4),
+Term2 = test_engine_util:db_as_term(Db5),
 
-{ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm),
-Db2 = test_engine_util:db_as_term(Engine, St6),
-Diff = test_engine_util:term_diff(Db1, Db2),
+Diff = test_engine_util:term_diff(Term1, Term2),
 ?assertEqual(nodiff, Diff).
 
 
 cet_multiple_purge_during_compact() ->
-{ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath),
-
-Actions1 = [
-{create, {<<"foo">>, []}},
-{create, {<<"bar">>, []}},
-{conflict, {<<"bar">>, [{<<"vsn">>, 2}]}},
-{create, {<<"baz">>, []}}
-],
-
-{ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1),
-{ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path),
+{ok, Db1} = test_engine_util:create_db(),
 
-[BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]),
-BarRev = test_engine_util:prev_rev(BarFDI),
+Actions1 = lists:map(fun(Seq) ->
+{create, {docid(Seq), {[{<<"int">>, Seq}]}}}
+end, lists:seq(1, 1000)),
 Actions2 = [
-{purge, {<<"bar">>, BarRev#rev_info.rev}}
+{create, {<<"foo">>, {[]}}},
+{create, {<<"bar">>, {[]}}},
+{create, {<<"baz">>, {[]}}}
 ],
-{ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2),
+{ok, Db2} = test_engine_util:apply_batch(Db1, Actions1 ++ Actions2),
 
-

[couchdb] 03/19: Fix default security object handling

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 64b6ea9781633cdff800aec41bfd9132d8832fce
Author: Paul J. Davis 
AuthorDate: Thu Apr 26 11:38:24 2018 -0500

Fix default security object handling

There's a race where if a database is opened with a default_security set
and it crashes before first compact, and is then reopened after the
default_security option has changed that it will pick the second
security option. This ensures change closes that relatively obscure bug
that was only found during testing.
---
 src/couch/src/couch_bt_engine.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/couch/src/couch_bt_engine.erl 
b/src/couch/src/couch_bt_engine.erl
index 43a77b0..2583f10 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -727,7 +727,7 @@ init_state(FilePath, Fd, Header0, Options) ->
 % to be written to disk.
 case Header /= Header0 of
 true ->
-{ok, NewSt} = commit_data(St),
+{ok, NewSt} = commit_data(St#st{needs_commit = true}),
 NewSt;
 false ->
 St

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 01/19: Fix typos in couch_db_engine.erl

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit dbed517f76d890774fac7d64307482832678ebdb
Author: Paul J. Davis 
AuthorDate: Fri Feb 5 11:49:34 2016 -0600

Fix typos in couch_db_engine.erl
---
 src/couch/src/couch_db_engine.erl | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/couch/src/couch_db_engine.erl 
b/src/couch/src/couch_db_engine.erl
index 502faa7..4974201 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -168,7 +168,7 @@
 % value would be hard to report its ok to just return the
 % result of os:timestamp/0 as this will just disable idle
 % databases from automatically closing.
--callback last_activity(DbHandle::db_handle()) -> elrang:timestamp().
+-callback last_activity(DbHandle::db_handle()) -> erlang:timestamp().
 
 
 % All of the get_* functions may be called from many
@@ -436,9 +436,9 @@
 %
 % 1. start_key - Start iteration at the provided key or
 %or just after if the key doesn't exist
-% 2. end_key - Stop iteration prior to visiting the provided
+% 2. end_key - Stop iteration just after the provided key
+% 3. end_key_gt - Stop iteration prior to visiting the provided
 %key
-% 3. end_key_gt - Stop iteration just after the provided key
 % 4. dir - The atom fwd or rev. This is to be able to iterate
 %over documents in reverse order. The logic for comparing
 %start_key, end_key, and end_key_gt are then reversed (ie,
@@ -492,12 +492,12 @@
 % This function is called to fold over the documents (not local
 % documents) in order of their most recent update. Each document
 % in the database should have exactly one entry in this sequence.
-% If a document is updated during a call to this funciton it should
+% If a document is updated during a call to this function it should
 % not be included twice as that will probably lead to Very Bad Things.
 %
 % This should behave similarly to fold_docs/4 in that the supplied
 % user function should be invoked with a #full_doc_info{} record
-% as the first arugment and the current user accumulator as the
+% as the first argument and the current user accumulator as the
 % second argument. The same semantics for the return value from the
 % user function should be handled as in fold_docs/4.
 %

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] 16/19: Implement HTTP endpoints for clustered purge

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit a5fdb2454252553787333b41b12b350d21352946
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:27:58 2018 -0500

Implement HTTP endpoints for clustered purge
---
 src/chttpd/src/chttpd_db.erl   |  46 +++--
 src/chttpd/test/chttpd_purge_tests.erl | 179 +
 2 files changed, 214 insertions(+), 11 deletions(-)

diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index ed0adea..b9652bc 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -496,24 +496,20 @@ db_req(#httpd{path_parts=[_, <<"_bulk_get">>]}=Req, _Db) 
->
 
 
 db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) ->
+couch_stats:increment_counter([couchdb, httpd, purge_requests]),
 chttpd:validate_ctype(Req, "application/json"),
+W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
+Options = [{user_ctx, Req#httpd.user_ctx}, {w, W}],
 {IdsRevs} = chttpd:json_body_obj(Req),
 IdsRevs2 = [{Id, couch_doc:parse_revs(Revs)} || {Id, Revs} <- IdsRevs],
-case fabric:purge_docs(Db, IdsRevs2) of
-{ok, PurgeSeq, PurgedIdsRevs} ->
-PurgedIdsRevs2 = [{Id, couch_doc:revs_to_strs(Revs)} || {Id, Revs}
-<- PurgedIdsRevs],
-send_json(Req, 200, {[
-{<<"purge_seq">>, PurgeSeq},
-{<<"purged">>, {PurgedIdsRevs2}}
-]});
-Error ->
-throw(Error)
-end;
+{ok, Results} = fabric:purge_docs(Db, IdsRevs2, Options),
+{Code, Json} = purge_results_to_json(IdsRevs2, Results),
+send_json(Req, Code, {[{<<"purge_seq">>, null}, {<<"purged">>, {Json}}]});
 
 db_req(#httpd{path_parts=[_,<<"_purge">>]}=Req, _Db) ->
 send_method_not_allowed(Req, "POST");
 
+
 db_req(#httpd{method='GET',path_parts=[_,OP]}=Req, Db) when ?IS_ALL_DOCS(OP) ->
 case chttpd:qs_json_value(Req, "keys", nil) of
 Keys when is_list(Keys) ->
@@ -623,6 +619,20 @@ 
db_req(#httpd{method='GET',path_parts=[_,<<"_revs_limit">>]}=Req, Db) ->
 db_req(#httpd{path_parts=[_,<<"_revs_limit">>]}=Req, _Db) ->
 send_method_not_allowed(Req, "PUT,GET");
 
+db_req(#httpd{method='PUT',path_parts=[_,<<"_purged_infos_limit">>]}=Req, Db) 
->
+Limit = chttpd:json_body(Req),
+Options = [{user_ctx, Req#httpd.user_ctx}],
+case chttpd:json_body(Req) of
+Limit when is_integer(Limit), Limit > 0 ->
+ok = fabric:set_purge_infos_limit(Db, Limit, Options),
+send_json(Req, {[{<<"ok">>, true}]});
+_->
+throw({bad_request, "`purge_infos_limit` must be positive 
integer"})
+end;
+
+db_req(#httpd{method='GET',path_parts=[_,<<"_purged_infos_limit">>]}=Req, Db) 
->
+send_json(Req, fabric:get_purge_infos_limit(Db));
+
 % Special case to enable using an unencoded slash in the URL of design docs,
 % as slashes in document IDs must otherwise be URL encoded.
 db_req(#httpd{method='GET', mochi_req=MochiReq, path_parts=[_DbName, 
<<"_design/", _/binary>> | _]}=Req, _Db) ->
@@ -993,6 +1003,20 @@ update_doc_result_to_json(DocId, Error) ->
 {_Code, ErrorStr, Reason} = chttpd:error_info(Error),
 {[{id, DocId}, {error, ErrorStr}, {reason, Reason}]}.
 
+purge_results_to_json([], []) ->
+{201, []};
+purge_results_to_json([{DocId, _Revs} | RIn], [{ok, PRevs} | ROut]) ->
+{Code, Results} = purge_results_to_json(RIn, ROut),
+{Code, [{DocId, couch_doc:revs_to_strs(PRevs)} | Results]};
+purge_results_to_json([{DocId, _Revs} | RIn], [{accepted, PRevs} | ROut]) ->
+{Code, Results} = purge_results_to_json(RIn, ROut),
+NewResults = [{DocId, couch_doc:revs_to_strs(PRevs)} | Results],
+{erlang:max(Code, 202), NewResults};
+purge_results_to_json([{DocId, _Revs} | RIn], [Error | ROut]) ->
+{Code, Results} = purge_results_to_json(RIn, ROut),
+{NewCode, ErrorStr, Reason} = chttpd:error_info(Error),
+NewResults = [{DocId, {[{error, ErrorStr}, {reason, Reason}]}} | Results],
+{erlang:max(NewCode, Code), NewResults}.
 
 send_updated_doc(Req, Db, DocId, Json) ->
 send_updated_doc(Req, Db, DocId, Json, []).
diff --git a/src/chttpd/test/chttpd_purge_tests.erl 
b/src/chttpd/test/chttpd_purge_tests.erl
new file mode 100644
index 000..ca4edda
--- /dev/null
+++ b/src/chttpd/test/chttpd_purge_tests.erl
@@ -0,0 +1,179 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the 

[couchdb] 13/19: Implement internal replication of purge requests

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 433403813ead35e44e69def74e674ba88c2f03fa
Author: Paul J. Davis 
AuthorDate: Tue Apr 24 12:27:14 2018 -0500

Implement internal replication of purge requests
---
 src/mem3/src/mem3_rep.erl | 204 +++---
 src/mem3/src/mem3_rpc.erl |  72 +++-
 2 files changed, 262 insertions(+), 14 deletions(-)

diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl
index 2cfe43f..88bece4 100644
--- a/src/mem3/src/mem3_rep.erl
+++ b/src/mem3/src/mem3_rep.erl
@@ -17,6 +17,8 @@
 go/2,
 go/3,
 make_local_id/2,
+make_purge_id/2,
+verify_purge_checkpoint/3,
 find_source_seq/4
 ]).
 
@@ -35,6 +37,7 @@
 infos = [],
 seq = 0,
 localid,
+purgeid,
 source,
 target,
 filter,
@@ -73,7 +76,7 @@ go(#shard{} = Source, #shard{} = Target, Opts) ->
 go(Acc).
 
 
-go(#acc{source=Source, batch_count=BC}=Acc0) ->
+go(#acc{source=Source, batch_count=BC}=Acc) ->
 case couch_db:open(Source#shard.name, [?ADMIN_CTX]) of
 {ok, Db} ->
 Resp = try
@@ -118,6 +121,34 @@ make_local_id(SourceThing, TargetThing, Filter) ->
 <<"_local/shard-sync-", S/binary, "-", T/binary, F/binary>>.
 
 
+make_purge_id(SourceUUID, TargetUUID) ->
+<<"_local/purge-mem3-", SourceUUID/binary, "-", TargetUUID/binary>>.
+
+
+verify_purge_checkpoint(_Db, _DocId, Props) ->
+DbName = couch_util:get_value(<<"dbname">>, Props),
+SourceBin = couch_util:get_value(<<"source">>, Props),
+TargetBin = couch_util:get_value(<<"target">>, Props),
+Range = couch_util:get_value(<<"range">>, Props),
+
+Source = binary_to_existing_atom(SourceBin, latin1),
+Target = binary_to_existing_atom(TargetBin, latin1),
+
+try
+Shards = mem3:shards(DbName),
+Nodes = lists:foldl(fun(Shard, Acc) ->
+case Shard#shard.range == Range of
+true -> [Shard#shard.node | Acc];
+false -> Acc
+end
+end, [], mem3:shards(DbName)),
+lists:member(Source, Nodes) andalso lists:member(Target, Nodes)
+catch
+error:database_does_not_exist ->
+false
+end.
+
+
 %% @doc Find and return the largest update_seq in SourceDb
 %% that the client has seen from TargetNode.
 %%
@@ -169,20 +200,131 @@ find_source_seq_int(#doc{body={Props}}, SrcNode0, 
TgtNode0, TgtUUID, TgtSeq) ->
 end.
 
 
-repl(#acc{db = Db} = Acc0) ->
-erlang:put(io_priority, {internal_repl, couch_db:name(Db)}),
-#acc{seq=Seq} = Acc1 = calculate_start_seq(Acc0),
-case Seq >= couch_db:get_update_seq(Db) of
-true ->
-{ok, 0};
-false ->
-Fun = fun ?MODULE:changes_enumerator/2,
-{ok, Acc2} = couch_db:fold_changes(Db, Seq, Fun, Acc1),
-{ok, #acc{seq = LastSeq}} = replicate_batch(Acc2),
-{ok, couch_db:count_changes_since(Db, LastSeq)}
+repl(#acc{db = Db0} = Acc0) ->
+erlang:put(io_priority, {internal_repl, couch_db:name(Db0)}),
+Acc1 = calculate_start_seq(Acc0),
+try
+Acc3 = case config:get_boolean("mem3", "replicate_purges", false) of
+true ->
+Acc2 = pull_purges(Acc1),
+push_purges(Acc2);
+false ->
+Acc1
+end,
+push_changes(Acc3)
+catch
+throw:{finished, Count} ->
+{ok, Count}
 end.
 
 
+pull_purges(#acc{} = Acc0) ->
+#acc{
+batch_size = Count,
+seq = UpdateSeq,
+target = Target
+} = Acc0,
+#shard{
+node = TgtNode,
+name = TgtDbName
+} = Target,
+
+with_src_db(Acc0, fun(Db) ->
+SrcUUID = couch_db:get_uuid(Db),
+{ok, LocalPurgeId, Infos, ThroughSeq, Remaining} =
+mem3_rpc:load_purge_infos(TgtNode, TgtDbName, SrcUUID, Count),
+
+if Infos == [] -> ok; true ->
+{ok, _} = couch_db:purge_docs(Db, Infos, [replicated_edits]),
+Body = purge_cp_body(Acc0, ThroughSeq),
+ok = mem3_rpc:save_purge_checkpoint(
+TgtNode, TgtDbName, LocalPurgeId, Body)
+end,
+
+if Remaining =< 0 -> ok; true ->
+OldestPurgeSeq = couch_db:get_oldest_purge_seq(Db),
+PurgesToPush = couch_db:get_purge_seq(Db) - OldestPurgeSeq,
+Changes = couch_db:count_changes_since(Db, UpdateSeq),
+throw({finished, Remaining + PurgesToPush + Changes})
+end,
+
+Acc0#acc{purgeid = LocalPurgeId}
+end).
+
+
+push_purges(#acc{} = Acc0) ->
+#acc{
+batch_size = BatchSize,
+purgeid = LocalPurgeId,
+seq = UpdateSeq,
+target = Target
+} = Acc0,
+#shard{
+node = TgtNode,
+name = TgtDbName
+} = 

[couchdb] 06/19: Simplify logic in mem3_rep

[davisp]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 5ad0fc1b9d361f300a1f0874f4fe41b3c460
Author: Paul J. Davis 
AuthorDate: Wed Mar 21 12:23:47 2018 -0500

Simplify logic in mem3_rep

Previously there were two separate database references and it was not
clear which was used where. This simplifies things by reducing it to a
single instance so that the logic is simpler.
---
 src/mem3/src/mem3_rep.erl | 17 -
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl
index 3d91877..2cfe43f 100644
--- a/src/mem3/src/mem3_rep.erl
+++ b/src/mem3/src/mem3_rep.erl
@@ -76,9 +76,8 @@ go(#shard{} = Source, #shard{} = Target, Opts) ->
 go(#acc{source=Source, batch_count=BC}=Acc0) ->
 case couch_db:open(Source#shard.name, [?ADMIN_CTX]) of
 {ok, Db} ->
-Acc = Acc0#acc{db=Db},
 Resp = try
-repl(Db, Acc)
+repl(Acc#acc{db = Db})
 catch error:{not_found, no_db_file} ->
 {error, missing_target}
 after
@@ -170,9 +169,9 @@ find_source_seq_int(#doc{body={Props}}, SrcNode0, TgtNode0, 
TgtUUID, TgtSeq) ->
 end.
 
 
-repl(Db, Acc0) ->
+repl(#acc{db = Db} = Acc0) ->
 erlang:put(io_priority, {internal_repl, couch_db:name(Db)}),
-#acc{seq=Seq} = Acc1 = calculate_start_seq(Acc0#acc{source = Db}),
+#acc{seq=Seq} = Acc1 = calculate_start_seq(Acc0),
 case Seq >= couch_db:get_update_seq(Db) of
 true ->
 {ok, 0};
@@ -186,7 +185,7 @@ repl(Db, Acc0) ->
 
 calculate_start_seq(Acc) ->
 #acc{
-source = Db,
+db = Db,
 target = #shard{node=Node, name=Name}
 } = Acc,
 %% Give the target our UUID and ask it to return the checkpoint doc
@@ -222,7 +221,7 @@ calculate_start_seq(Acc) ->
 
 compare_epochs(Acc) ->
 #acc{
-source = Db,
+db = Db,
 target = #shard{node=Node, name=Name}
 } = Acc,
 UUID = couch_db:get_uuid(Db),
@@ -303,13 +302,13 @@ chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks, 
Limit) ->
 ).
 
 
-open_docs(#acc{source=Source, infos=Infos}, Missing) ->
+open_docs(#acc{db=Db, infos=Infos}, Missing) ->
 lists:flatmap(fun({Id, Revs, _}) ->
 FDI = lists:keyfind(Id, #full_doc_info.id, Infos),
 #full_doc_info{rev_tree=RevTree} = FDI,
 {FoundRevs, _} = couch_key_tree:get_key_leafs(RevTree, Revs),
 lists:map(fun({#leaf{deleted=IsDel, ptr=SummaryPtr}, FoundRevPath}) ->
-couch_db:make_doc(Source, Id, IsDel, SummaryPtr, FoundRevPath)
+couch_db:make_doc(Db, Id, IsDel, SummaryPtr, FoundRevPath)
 end, FoundRevs)
 end, Missing).
 
@@ -325,7 +324,7 @@ save_on_target(Node, Name, Docs) ->
 
 
 update_locals(Acc) ->
-#acc{seq=Seq, source=Db, target=Target, localid=Id, history=History} = Acc,
+#acc{seq=Seq, db=Db, target=Target, localid=Id, history=History} = Acc,
 #shard{name=Name, node=Node} = Target,
 NewEntry = [
 {<<"source_node">>, atom_to_binary(node(), utf8)},

-- 
To stop receiving notification emails like this one, please contact
dav...@apache.org.

[couchdb] branch COUCHDB-3326-clustered-purge-davisp-refactor updated: Add test compact with broken purge checkpoint doc

[jiangphcn]

This is an automated email from the ASF dual-hosted git repository.

jiangphcn pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to 
refs/heads/COUCHDB-3326-clustered-purge-davisp-refactor by this push:
 new 059e3d4  Add test compact with broken purge checkpoint doc
059e3d4 is described below

commit 059e3d4d6e0944c398e5b94aaf24fb7aac64bf03
Author: jiangphcn 
AuthorDate: Thu Apr 26 17:46:06 2018 +0800

Add test compact with broken purge checkpoint doc

COUCHDB-3326
---
 src/couch/src/couch_db.erl |   6 +-
 src/couch/test/couch_db_purge_checkpoint_tests.erl | 237 +
 2 files changed, 241 insertions(+), 2 deletions(-)

diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index cf7cd8f..34c1d2a 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -80,6 +80,8 @@
 get_purge_infos/2,
 
 get_minimum_purge_seq/1,
+purge_client_exists/3,
+get_purge_client_fun/2,
 
 update_doc/3,
 update_doc/4,
@@ -478,7 +480,7 @@ get_purge_client_fun(DocId, Props) ->
 M = try
 binary_to_existing_atom(M0, latin1)
 catch error:badarg ->
-Fmt1 = "Missing index module '~s' for purge checkpoint '~s'",
+Fmt1 = "Missing index module '~p' for purge checkpoint '~p'",
 couch_log:error(Fmt1, [M0, DocId]),
 throw(failed)
 end,
@@ -488,7 +490,7 @@ get_purge_client_fun(DocId, Props) ->
 F = binary_to_existing_atom(F0, latin1),
 fun M:F/2
 catch error:badarg ->
-Fmt2 = "Missing function '~s' in '~s' for purge checkpoint '~s'",
+Fmt2 = "Missing function '~p' in '~p' for purge checkpoint '~p'",
 couch_log:error(Fmt2, [F0, M0, DocId]),
 throw(failed)
 end.
diff --git a/src/couch/test/couch_db_purge_checkpoint_tests.erl 
b/src/couch/test/couch_db_purge_checkpoint_tests.erl
new file mode 100644
index 000..1634e61
--- /dev/null
+++ b/src/couch/test/couch_db_purge_checkpoint_tests.erl
@@ -0,0 +1,237 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_db_purge_checkpoint_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+
+setup() ->
+DbName = ?tempdb(),
+{ok, _Db} = create_db(DbName),
+DbName.
+
+teardown(DbName) ->
+delete_db(DbName),
+ok.
+
+couch_db_purge_checkpoint_test_() ->
+{
+"Couch_db purge_checkpoint",
+[
+{
+setup,
+fun test_util:start_couch/0, fun test_util:stop_couch/1,
+[couch_db_purge_checkpoint()]
+}
+]
+
+}.
+
+
+couch_db_purge_checkpoint() ->
+{
+   foreach,
+fun setup/0, fun teardown/1,
+[
+fun test_purge_cp_bad_purgeseq/1,
+fun test_purge_cp_bad_verify_mod/1,
+fun test_purge_cp_bad_verify_fun/1,
+fun test_purge_cp_verify_fun_with_throw/1,
+fun test_purge_cp_verify_fun_without_boolean_rc/1
+]
+}.
+
+
+test_purge_cp_bad_purgeseq(DbName) ->
+?_test(
+begin
+{ok, Db} = couch_db:open_int(DbName, []),
+update_local_purge_doc(
+Db,
+"<>",
+bad_purgeseq,
+<<"couch_db_purge_checkpoint_tests">>,
+<<"normal_verify_fun">>
+),
+{ok, Db2} = couch_db:reopen(Db),
+Result = try
+couch_db:get_minimum_purge_seq(Db2)
+catch _:_ ->
+failed
+end,
+?assertEqual(0, Result)
+end).
+
+
+test_purge_cp_bad_verify_mod(DbName) ->
+?_test(
+begin
+{ok, Db} = couch_db:open_int(DbName, []),
+update_local_purge_doc(
+Db,
+"<>",
+1,
+[invalid_module],
+<<"valid_verify_fun">>
+
+),
+{ok, Db2} = couch_db:reopen(Db),
+FoldFun = fun(#doc{id = DocId, body = {Props}}, SeqAcc) ->
+case DocId of
+<<"_local/purge-", _/binary>> ->
+try
+couch_db:get_purge_client_fun(DocId, Props)
+catch failed ->
+