This is an automated email from the ASF dual-hosted git repository. garren pushed a commit to branch fdb-mango-indexes in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5294ab841045302efede84f7820a183f44067dc9 Author: Garren Smith <[email protected]> AuthorDate: Thu Feb 6 15:58:19 2020 +0200 more work on background indexer --- src/couch/src/couch_proc_manager.erl | 2 +- src/couch_js/src/couch_js_proc_manager.erl | 2 +- src/mango/src/mango_fdb.erl | 2 +- src/mango/src/mango_httpd.erl | 3 ++- src/mango/src/mango_idx.erl | 31 ++++++++++++++++-------- src/mango/src/mango_idx_view.hrl | 3 +-- src/mango/src/mango_indexer_server.erl | 2 +- src/mango/src/mango_jobs.erl | 5 ++-- src/mango/src/mango_jobs_indexer.erl | 1 - src/mango/src/mango_native_proc.erl | 3 ++- src/mango/test/13-users-db-find-test.py | 4 ++- src/mango/test/eunit/mango_jobs_indexer_test.erl | 9 ++++--- src/mango/test/mango.py | 9 ++----- 13 files changed, 44 insertions(+), 32 deletions(-) diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl index 3366b2b..0f686ef 100644 --- a/src/couch/src/couch_proc_manager.erl +++ b/src/couch/src/couch_proc_manager.erl @@ -108,7 +108,7 @@ init([]) -> ets:new(?SERVERS, [public, named_table, set]), ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")), ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")), - ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]), +%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]), maybe_configure_erlang_native_servers(), {ok, #state{ diff --git a/src/couch_js/src/couch_js_proc_manager.erl b/src/couch_js/src/couch_js_proc_manager.erl index 0964696..0f2916b 100644 --- a/src/couch_js/src/couch_js_proc_manager.erl +++ b/src/couch_js/src/couch_js_proc_manager.erl @@ -108,7 +108,7 @@ init([]) -> ets:new(?SERVERS, [public, named_table, set]), ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")), ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")), - ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]), +%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]), maybe_configure_erlang_native_servers(), {ok, #state{ diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl index a54d658..3e0c48e 100644 --- a/src/mango/src/mango_fdb.erl +++ b/src/mango/src/mango_fdb.erl @@ -224,7 +224,7 @@ fold_cb({Key, _}, Acc) -> {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix), {ok, Doc} = fabric2_db:open_doc(Db, DocId), JSONDoc = couch_doc:to_json_obj(Doc, []), -%% io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]), + io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]), case Callback({doc, JSONDoc}, Cursor) of {ok, Cursor1} -> Acc#{ diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl index d5e9cfa..b8525dc 100644 --- a/src/mango/src/mango_httpd.erl +++ b/src/mango/src/mango_httpd.erl @@ -62,7 +62,8 @@ handle_req_int(_, _) -> handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) -> Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end, chttpd:qs(Req)), - Idxs = lists:sort(mango_idx:list(Db)), + Idxs0 = mango_idx:add_build_status(Db, mango_idx:list(Db)), + Idxs = lists:sort(Idxs0), JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs), TotalRows = length(JsonIdxs0), Limit = case couch_util:get_value(limit, Params, TotalRows) of diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index 3aadd49..c1deaa9 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -26,6 +26,7 @@ add/2, remove/2, from_ddoc/2, + add_build_status/2, special/1, dbname/1, @@ -58,11 +59,9 @@ list(Db) -> rows => [] }, {ok, Indexes} = fabric2_db:fold_design_docs(Db, fun ddoc_fold_cb/2, Acc0, []), -%% io:format("INDEXES ~p ~n", [Indexes]), Indexes ++ special(Db). -% Todo this should all be in fabric2_db ddoc_fold_cb({meta, _}, Acc) -> {ok, Acc}; @@ -237,14 +236,25 @@ from_ddoc(Db, {Props}) -> %% [mango_idx_view] %% end, Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods), + lists:map(fun(Idx) -> + Idx#idx{ + dbname = DbName, + ddoc = DDoc, + partitioned = get_idx_partitioned(Db, Props) + } + end, Idxs). + + +add_build_status(Db, Idxs) -> fabric2_fdb:transactional(Db, fun(TxDb) -> - lists:map(fun(Idx) -> - Idx#idx{ - dbname = DbName, - ddoc = DDoc, - partitioned = get_idx_partitioned(Db, Props), - build_status = mango_fdb:get_build_status(TxDb, DDoc) - } + lists:map(fun + (#idx{type = <<"special">>} = Idx) -> + Idx; + (Idx) -> + DDoc = mango_idx:ddoc(Idx), + Idx#idx{ + build_status = mango_fdb:get_build_status(TxDb, DDoc) + } end, Idxs) end). @@ -255,7 +265,8 @@ special(Db) -> name = <<"_all_docs">>, type = <<"special">>, def = all_docs, - opts = [] + opts = [], + build_status = ?MANGO_INDEX_READY }, % Add one for _update_seq [AllDocs]. diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl index 6ebe68e..14ce87c 100644 --- a/src/mango/src/mango_idx_view.hrl +++ b/src/mango/src/mango_idx_view.hrl @@ -11,5 +11,4 @@ % the License. %%-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}). --define(MAX_JSON_OBJ, <<255>>). -%%-define(MAX_JSON_OBJ, {[{<<"ZZZ">>, <<"ZZZ">>}]}). +-define(MAX_JSON_OBJ, {[{<<"\ufff0">>, <<"\ufff0">>}]}). diff --git a/src/mango/src/mango_indexer_server.erl b/src/mango/src/mango_indexer_server.erl index 29530bb..6942c9f 100644 --- a/src/mango/src/mango_indexer_server.erl +++ b/src/mango/src/mango_indexer_server.erl @@ -31,7 +31,7 @@ ]). --define(MAX_WORKERS, 1). +-define(MAX_WORKERS, 100). start_link() -> diff --git a/src/mango/src/mango_jobs.erl b/src/mango/src/mango_jobs.erl index 6739d62..c5a70ff 100644 --- a/src/mango/src/mango_jobs.erl +++ b/src/mango/src/mango_jobs.erl @@ -39,8 +39,9 @@ build_index(TxDb, #idx{} = Idx) -> {ok, JobId}. -job_id(#{name := DbName}, #idx{ddoc = DDoc}) -> - <<DbName/binary, "-",DDoc/binary>>. +job_id(#{name := DbName}, #idx{ddoc = DDoc} = Idx) -> + Cols = iolist_to_binary(mango_idx:columns(Idx)), + <<DbName/binary, "_",DDoc/binary, Cols/binary>>. job_data(Db, Idx) -> diff --git a/src/mango/src/mango_jobs_indexer.erl b/src/mango/src/mango_jobs_indexer.erl index ce6b850..d68c80b 100644 --- a/src/mango/src/mango_jobs_indexer.erl +++ b/src/mango/src/mango_jobs_indexer.erl @@ -94,7 +94,6 @@ init() -> exit:normal -> ok; Error:Reason -> - io:format("ERROR in index worker ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]), NewRetry = Retries + 1, RetryLimit = retry_limit(), diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl index cbf3622..5a05083 100644 --- a/src/mango/src/mango_native_proc.erl +++ b/src/mango/src/mango_native_proc.erl @@ -47,7 +47,8 @@ start_link() -> - gen_server:start_link(?MODULE, [], []). + throw({error, mango_native_proc_is_no_longer_needed}). +%% gen_server:start_link(?MODULE, [], []). set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 -> diff --git a/src/mango/test/13-users-db-find-test.py b/src/mango/test/13-users-db-find-test.py index 73d15ea..32d919a 100644 --- a/src/mango/test/13-users-db-find-test.py +++ b/src/mango/test/13-users-db-find-test.py @@ -12,9 +12,10 @@ # the License. -import mango, requests +import mango, requests, unittest [email protected]("this FDB doesn't support this") class UsersDbFindTests(mango.UsersDbTests): def test_simple_find(self): docs = self.db.find({"name": {"$eq": "demo02"}}) @@ -57,6 +58,7 @@ class UsersDbFindTests(mango.UsersDbTests): assert len(docs) == 3 [email protected]("this FDB doesn't support this") class UsersDbIndexFindTests(UsersDbFindTests): def setUp(self): self.db.create_index(["name"]) diff --git a/src/mango/test/eunit/mango_jobs_indexer_test.erl b/src/mango/test/eunit/mango_jobs_indexer_test.erl index 7a8cb24..9641163 100644 --- a/src/mango/test/eunit/mango_jobs_indexer_test.erl +++ b/src/mango/test/eunit/mango_jobs_indexer_test.erl @@ -89,7 +89,6 @@ index_lots_of_docs(Db) -> index_can_recover_from_crash(Db) -> meck:new(mango_indexer, [passthrough]), meck:expect(mango_indexer, write_doc, fun (Db, Doc, Idxs) -> - ?debugFmt("doc ~p ~p ~n", [Doc, Idxs]), Id = Doc#doc.id, case Id == <<"2">> of true -> @@ -112,8 +111,12 @@ index_can_recover_from_crash(Db) -> wait_while_ddoc_builds(Db) -> fabric2_fdb:transactional(Db, fun(TxDb) -> Idxs = mango_idx:list(TxDb), - [Idx] = lists:filter(fun (Idx) -> Idx#idx.type == <<"json">> end, Idxs), - if Idx#idx.build_status == ?MANGO_INDEX_READY -> ok; true -> + + Ready = lists:filter(fun (Idx) -> + Idx#idx.build_status == ?MANGO_INDEX_READY + end, mango_idx:add_build_status(TxDb, Idxs)), + + if length(Ready) > 1 -> ok; true -> timer:sleep(100), wait_while_ddoc_builds(Db) end diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py index 92cf211..5b1c7a7 100644 --- a/src/mango/test/mango.py +++ b/src/mango/test/mango.py @@ -110,11 +110,7 @@ class Database(object): def save_docs(self, docs, **kwargs): body = json.dumps({"docs": docs}) r = self.sess.post(self.path("_bulk_docs"), data=body, params=kwargs) - print("DOC") - print(docs) r.raise_for_status() - print("RES") - print(r.json()) for doc, result in zip(docs, r.json()): doc["_id"] = result["id"] doc["_rev"] = result["rev"] @@ -142,7 +138,7 @@ class Database(object): name=None, ddoc=None, partial_filter_selector=None, - selector=None, + selector=None ): body = {"index": {"fields": fields}, "type": idx_type, "w": 3} if name is not None: @@ -167,7 +163,7 @@ class Database(object): for i in self.get_index(r.json()["id"], r.json()["name"]) if i["build_status"] == "ready" ]) < 1: - delay(t=0.1) + delay(t=0.2) return created @@ -285,7 +281,6 @@ class Database(object): else: path = self.path("_find") r = self.sess.post(path, data=body) - print(r.json()) r.raise_for_status() if explain or return_raw: return r.json()
