Enable bulk delete with fabric update_docs Rather than delete each doc one by one, we use fabric:update_docs to bulk delete.
Fixes:COUCHDB-2651 Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/3a8efbee Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/3a8efbee Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/3a8efbee Branch: refs/heads/2651-delete-bulk-docs Commit: 3a8efbee5b954a38fb221a9653eb1682987aa2a0 Parents: c8a2194 Author: Tony Sun <tony....@cloudant.com> Authored: Wed Jun 3 15:53:21 2015 -0400 Committer: Tony Sun <tony....@cloudant.com> Committed: Wed Jun 3 15:53:21 2015 -0400 ---------------------------------------------------------------------- src/mango_crud.erl | 3 +- src/mango_httpd.erl | 17 +++++++++++ src/mango_idx.erl | 62 +++++++++++++++++++++++++++++++++++++++++ src/mango_opts.erl | 26 ++++++++++++++++- src/mango_util.erl | 3 ++ test/01-index-crud-test.py | 30 ++++++++++++++++++++ test/mango.py | 9 ++++++ 7 files changed, 148 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/3a8efbee/src/mango_crud.erl ---------------------------------------------------------------------- diff --git a/src/mango_crud.erl b/src/mango_crud.erl index 68c9d6c..c9eb845 100644 --- a/src/mango_crud.erl +++ b/src/mango_crud.erl @@ -21,7 +21,8 @@ ]). -export([ - collect_cb/2 + collect_cb/2, + maybe_add_user_ctx/2 ]). http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/3a8efbee/src/mango_httpd.erl ---------------------------------------------------------------------- diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl index 28a1578..2fc2535 100644 --- a/src/mango_httpd.erl +++ b/src/mango_httpd.erl @@ -80,6 +80,14 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> end, chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]}); +handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>, + <<"_bulk_delete">>]}=Req, Db) -> + {ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)), + DDocIds = get_bulk_delete_ddocs_ids(Opts), + DelOpts = get_idx_create_opts(Opts), + {Success, Error} = mango_idx:bulk_delete(Db, DDocIds, DelOpts), + chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"error">>, Error}]}); + handle_index_req(#httpd{method='DELETE', path_parts=[A, B, <<"_design">>, DDocId0, Type, Name]}=Req, Db) -> PathParts = [A, B, <<"_design/", DDocId0/binary>>, Type, Name], @@ -157,6 +165,15 @@ get_idx_create_opts(Opts) -> end. +get_bulk_delete_ddocs_ids(Opts) -> + case lists:keyfind(docids, 1, Opts) of + {docids, DDocs} when is_list(DDocs) -> + DDocs; + _ -> + [] + end. + + get_idx_del_opts(Req) -> try WStr = chttpd:qs_value(Req, "w", "2"), http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/3a8efbee/src/mango_idx.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx.erl b/src/mango_idx.erl index 1c15894..31d5e7d 100644 --- a/src/mango_idx.erl +++ b/src/mango_idx.erl @@ -26,6 +26,7 @@ validate/1, add/2, remove/2, + bulk_delete/3, from_ddoc/2, special/1, @@ -134,6 +135,67 @@ remove(DDoc, Idx) -> {ok, NewDDoc#doc{body = Body}}. +bulk_delete(Db, DDocIds, DelOpts0) -> + DelOpts = mango_crud:maybe_add_user_ctx(Db, DelOpts0), + {DeleteDocs, Errors} = lists:foldl(fun(DDocId0, {D, E}) -> + Id = {<<"id">>, DDocId0}, + case get_bulk_delete_ddoc(Db, DDocId0) of + not_found -> + {D, [{[Id, {<<"error">>, <<"does not exist">>}]} | E]}; + invalid_ddoc_lang -> + {D, [{[Id, {<<"error">>, <<"not a query doc">>}]} | E]}; + error_loading_doc -> + {D, [{[Id, {<<"error">>, <<"loading doc">>}]} | E]}; + DDoc -> + {[DDoc#doc{deleted = true, body = {[]}} | D], E } + end + end, {[], []}, DDocIds), + case fabric:update_docs(Db, DeleteDocs, DelOpts) of + {ok, Results} -> + bulk_delete_results(lists:zip(DeleteDocs, Results), Errors); + {accepted, Results} -> + bulk_delete_results(lists:zip(DeleteDocs, Results), Errors); + {aborted, Abort} -> + bulk_delete_results(lists:zip(DeleteDocs, Abort), Errors) + end. + + +bulk_delete_results(DeleteResults, LoadErrors) -> + {Success, Errors} = lists:foldl(fun({#doc{id=DDocId}, Result}, {S, E}) -> + Id = {<<"id">>, DDocId}, + case Result of + {_, {_Pos, _}} -> + {[{[Id, {<<"ok">>, true}]} | S], E}; + {{_Id, _Rev}, Error} -> + {_Code, ErrorStr, _Reason} = chttpd:error_info(Error), + {S, [{[Id, {<<"error">>, ErrorStr}]} | E]}; + Error -> + {_Code, ErrorStr, _Reason} = chttpd:error_info(Error), + {S, [{[Id, {<<"error">>, ErrorStr}]} | E]} + end + end, {[], []}, DeleteResults), + {Success, Errors ++ LoadErrors}. + + +get_bulk_delete_ddoc(Db, Id0) -> + Id = case Id0 of + <<"_design/", _/binary>> -> Id0; + _ -> <<"_design/", Id0/binary>> + end, + try mango_util:open_doc(Db, Id, [deleted, ejson_body]) of + {ok, #doc{deleted = false, body=Body} = Doc} -> + mango_util:check_lang(Doc), + Doc; + not_found -> + not_found + catch + {{mango_error, mango_util, {invalid_ddoc_lang, _}}} -> + invalid_ddoc_lang; + {{mango_error, mango_util, {error_loading_doc, _}}} -> + error_loading_doc + end. + + from_ddoc(Db, {Props}) -> DbName = db_to_name(Db), DDoc = proplists:get_value(<<"_id">>, Props), http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/3a8efbee/src/mango_opts.erl ---------------------------------------------------------------------- diff --git a/src/mango_opts.erl b/src/mango_opts.erl index f7874a6..d7af601 100644 --- a/src/mango_opts.erl +++ b/src/mango_opts.erl @@ -14,7 +14,8 @@ -export([ validate_idx_create/1, - validate_find/1 + validate_find/1, + validate_bulk_delete/1 ]). -export([ @@ -129,6 +130,22 @@ validate_find({Props}) -> validate(Props, Opts). +validate_bulk_delete({Props}) -> + Opts = [ + {<<"docids">>, [ + {tag, docids}, + {validator, fun validate_bulk_docs/1} + ]}, + {<<"w">>, [ + {tag, w}, + {optional, true}, + {default, 2}, + {validator, fun is_pos_integer/1} + ]} + ], + validate(Props, Opts). + + validate(Props, Opts) -> case mango_util:assert_ejson({Props}) of true -> @@ -192,6 +209,13 @@ validate_selector(Else) -> ?MANGO_ERROR({invalid_selector_json, Else}). +validate_bulk_docs(Docs) when is_list(Docs) -> + lists:foreach(fun ?MODULE:is_string/1, Docs), + {ok, Docs}; +validate_bulk_docs(Else) -> + ?MANGO_ERROR({invalid_bulk_docs, Else}). + + validate_use_index(IndexName) when is_binary(IndexName) -> case binary:split(IndexName, <<"/">>) of [DesignId] -> http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/3a8efbee/src/mango_util.erl ---------------------------------------------------------------------- diff --git a/src/mango_util.erl b/src/mango_util.erl index 99e15d5..a48a7fb 100644 --- a/src/mango_util.erl +++ b/src/mango_util.erl @@ -15,6 +15,7 @@ -export([ open_doc/2, + open_doc/3, open_ddocs/1, load_ddoc/2, @@ -37,6 +38,8 @@ has_suffix/2, + check_lang/1, + join/2 ]). http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/3a8efbee/test/01-index-crud-test.py ---------------------------------------------------------------------- diff --git a/test/01-index-crud-test.py b/test/01-index-crud-test.py index 459566b..0a7f595 100644 --- a/test/01-index-crud-test.py +++ b/test/01-index-crud-test.py @@ -150,6 +150,36 @@ class IndexCrudTests(mango.DbPerClass): post_indexes = self.db.list_indexes() assert pre_indexes == post_indexes + def test_bulk_delete(self): + fields = ["field1"] + ret = self.db.create_index(fields, name="idx_01") + assert ret is True + + fields = ["field2"] + ret = self.db.create_index(fields, name="idx_02") + assert ret is True + + fields = ["field3"] + ret = self.db.create_index(fields, name="idx_03") + assert ret is True + + docids = [] + + for idx in self.db.list_indexes(): + if idx["ddoc"] is not None: + docids.append(idx["ddoc"]) + + docids.append("_design/this_is_not_an_index_name") + + ret = self.db.bulk_delete(docids) + print ret + assert ret["error"][0]["id"] == "_design/this_is_not_an_index_name" + assert len(ret["success"]) == 3 + + for idx in self.db.list_indexes(): + assert idx["type"] != "json" + assert idx["type"] != "text" + def test_recreate_index(self): pre_indexes = self.db.list_indexes() for i in range(5): http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/3a8efbee/test/mango.py ---------------------------------------------------------------------- diff --git a/test/mango.py b/test/mango.py index 57dfffb..b39c916 100644 --- a/test/mango.py +++ b/test/mango.py @@ -135,6 +135,15 @@ class Database(object): r = self.sess.delete(self.path(path), params={"w":"3"}) r.raise_for_status() + def bulk_delete(self, docs): + body = { + "docids" : docs, + "w": 3 + } + body = json.dumps(body) + r = self.sess.post(self.path("_index/_bulk_delete"), data=body) + return r.json() + def find(self, selector, limit=25, skip=0, sort=None, fields=None, r=1, conflicts=False, use_index=None, explain=False, bookmark=None, return_raw=False):