This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch out-of-disk-handler in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 12634541d82a6c54fed4e4f31233e7fe7fe6bbf0 Author: Robert Newson <rnew...@apache.org> AuthorDate: Fri Jun 16 09:25:41 2023 +0100 Introduce optional countermeasures as we run out of disk space --- rel/overlay/etc/default.ini | 6 + rel/overlay/etc/vm.args | 9 ++ rel/reltool.config | 2 + src/chttpd/src/chttpd.erl | 2 + src/couch/priv/stats_descriptions.cfg | 4 + src/couch/src/couch.app.src | 1 + src/couch/src/couch_db.erl | 9 ++ src/couch/src/couch_disk_monitor.erl | 241 +++++++++++++++++++++++++++++ src/couch/src/couch_secondary_sup.erl | 3 +- src/couch_mrview/src/couch_mrview.erl | 8 +- src/couch_mrview/src/couch_mrview_util.erl | 17 +- src/docs/src/config/disk-monitor.rst | 76 +++++++++ src/docs/src/config/index.rst | 1 + src/fabric/src/fabric_rpc.erl | 8 +- src/fabric/src/fabric_streams.erl | 4 +- src/fabric/src/fabric_view.erl | 9 +- src/fabric/src/fabric_view_map.erl | 6 + src/fabric/src/fabric_view_reduce.erl | 4 + src/ken/src/ken_server.erl | 9 +- 19 files changed, 406 insertions(+), 13 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index bb440e0aa..c3124a643 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -898,3 +898,9 @@ port = {{prometheus_port}} [nouveau] enable = {{with_nouveau}} + +[disk_monitor] +;enable = false +;background_view_indexing_threshold = 80 +;interactive_view_indexing_threshold = 90 +;interactive_database_writes_threshold = 90 diff --git a/rel/overlay/etc/vm.args b/rel/overlay/etc/vm.args index 886bbb903..8b2440fc8 100644 --- a/rel/overlay/etc/vm.args +++ b/rel/overlay/etc/vm.args @@ -122,3 +122,12 @@ # in the features list. # #-crypto fips_mode true + +# OS Mon Settings + +# only start disksup +-os_mon start_cpu_sup false +-os_mon start_memsup false + +# Check disk space every 5 minutes +-os_mon disk_space_check_interval 5 diff --git a/rel/reltool.config b/rel/reltool.config index da9ad6d3b..1dca6ff85 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -23,6 +23,7 @@ sasl, ssl, stdlib, + os_mon, syntax_tools, xmerl, %% couchdb @@ -84,6 +85,7 @@ {app, sasl, [{incl_cond, include}]}, {app, ssl, [{incl_cond, include}]}, {app, stdlib, [{incl_cond, include}]}, + {app, os_mon, [{incl_cond, include}]}, {app, syntax_tools, [{incl_cond, include}]}, {app, xmerl, [{incl_cond, include}]}, diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl index 53abc731f..c8e6fdc97 100644 --- a/src/chttpd/src/chttpd.erl +++ b/src/chttpd/src/chttpd.erl @@ -1138,6 +1138,8 @@ error_info(timeout) -> >>}; error_info({service_unavailable, Reason}) -> {503, <<"service unavailable">>, Reason}; +error_info({insufficient_storage, Reason}) -> + {507, <<"insufficent_storage">>, Reason}; error_info({timeout, _Reason}) -> error_info(timeout); error_info({'EXIT', {Error, _Stack}}) -> diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 6c0d4dad2..1983eed9b 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -266,6 +266,10 @@ {type, counter}, {desc, <<"number of HTTP 503 Service unavailable responses">>} ]}. +{[couchdb, httpd_status_codes, 507], [ + {type, counter}, + {desc, <<"number of HTTP 507 Insufficient Storage responses">>} +]}. {[couchdb, open_databases], [ {type, counter}, {desc, <<"number of open databases">>} diff --git a/src/couch/src/couch.app.src b/src/couch/src/couch.app.src index ef4e5e956..6fc293ac1 100644 --- a/src/couch/src/couch.app.src +++ b/src/couch/src/couch.app.src @@ -33,6 +33,7 @@ sasl, inets, ssl, + os_mon, % Upstream deps ibrowse, diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index 4bc2aca8f..6f07e593a 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -1338,6 +1338,15 @@ update_docs(Db, Docs0, Options, ?REPLICATED_CHANGES) -> ), {ok, DocErrors}; update_docs(Db, Docs0, Options, ?INTERACTIVE_EDIT) -> + BlockInteractiveDatabaseWrites = couch_disk_monitor:block_interactive_database_writes(), + if + BlockInteractiveDatabaseWrites -> + {ok, [{insufficient_storage, <<"database_dir is too full">>}|| _ <- Docs0]}; + true -> + update_docs_interactive(Db, Docs0, Options) + end. + +update_docs_interactive(Db, Docs0, Options) -> Docs = tag_docs(Docs0), AllOrNothing = lists:member(all_or_nothing, Options), diff --git a/src/couch/src/couch_disk_monitor.erl b/src/couch/src/couch_disk_monitor.erl new file mode 100644 index 000000000..62c34085c --- /dev/null +++ b/src/couch/src/couch_disk_monitor.erl @@ -0,0 +1,241 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_disk_monitor). +-behaviour(gen_server). + +% public api +-export([ + block_background_view_indexing/0, + block_interactive_view_indexing/0, + block_interactive_database_writes/0 +]). + +% testing +-export([ + set_database_dir_percent_used/1, + set_view_index_dir_percent_used/1 +]). + +% gen_server callbacks +-export([ + start_link/0, + init/1, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +-include_lib("kernel/include/file.hrl"). + +-define(SECTION, "disk_monitor"). + +-record(st, { + timer +}). + +block_background_view_indexing() -> + Enabled = enabled(), + if + Enabled -> + view_index_dir_percent_used() > background_view_indexing_threshold(); + true -> + false + end. + +block_interactive_view_indexing() -> + Enabled = enabled(), + if + Enabled -> view_index_dir_percent_used() > interactive_view_indexing_threshold(); + true -> false + end. + +block_interactive_database_writes() -> + Enabled = enabled(), + if + Enabled -> database_dir_percent_used() > interactive_database_writes_threshold(); + true -> false + end. + +set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 -> + gen_server:call(?MODULE, {set_database_dir_percent_used, PercentUsed}). + +set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 -> + gen_server:call(?MODULE, {set_view_index_dir_percent_used, PercentUsed}). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init(_Args) -> + ets:new(?MODULE, [named_table, {read_concurrency, true}]), + ets:insert(?MODULE, {database_dir, 0}), + ets:insert(?MODULE, {view_index_dir, 0}), + update_disk_data(), + {ok, TRef} = timer:send_after(timer_interval(), update_disk_data), + {ok, #st{timer = TRef}}. + +handle_call({set_database_dir_percent_used, PercentUsed}, _From, St) -> + ets:insert(?MODULE, {database_dir, PercentUsed}), + {reply, ok, St}; +handle_call({set_view_index_dir_percent_used, PercentUsed}, _From, St) -> + ets:insert(?MODULE, {view_index_dir, PercentUsed}), + {reply, ok, St}; +handle_call(_Msg, _From, St) -> + {reply, {error, unknown_msg}, St}. + +handle_cast(_Msg, St) -> + {noreply, St}. + +handle_info(update_disk_data, St) -> + timer:cancel(St#st.timer), + update_disk_data(), + {ok, TRef} = timer:send_after(timer_interval(), update_disk_data), + {noreply, St#st{timer = TRef}}; +handle_info(_Msg, St) -> + {noreply, St}. + +update_disk_data() -> + DiskData = disksup:get_disk_data(), + update_disk_data(DiskData). + +update_disk_data([]) -> + ok; +update_disk_data([{Id, _, PercentUsed} | Rest]) -> + IsDatabaseDir = is_database_dir(Id), + IsViewIndexDir = is_view_index_dir(Id), + if + IsDatabaseDir -> + ets:insert(?MODULE, {database_dir, PercentUsed}); + true -> + ok + end, + if + IsViewIndexDir -> + ets:insert(?MODULE, {view_index_dir, PercentUsed}); + true -> + ok + end, + update_disk_data(Rest). + +is_database_dir(MntOn) -> + same_device(config:get("couchdb", "database_dir"), MntOn). + +is_view_index_dir(MntOn) -> + same_device(config:get("couchdb", "view_index_dir"), MntOn). + +same_device(DirA, DirB) -> + case {device_id(DirA), device_id(DirB)} of + {{ok, DeviceId}, {ok, DeviceId}} -> + true; + _Else -> + false + end. + +device_id(Dir) -> + case file:read_file_info(Dir) of + {ok, FileInfo} -> + {ok, {FileInfo#file_info.minor_device, FileInfo#file_info.major_device}}; + {error, Reason} -> + {error, Reason} + end. + +database_dir_percent_used() -> + [{database_dir, PercentUsed}] = ets:lookup(?MODULE, database_dir), + PercentUsed. + +view_index_dir_percent_used() -> + [{view_index_dir, PercentUsed}] = ets:lookup(?MODULE, view_index_dir), + PercentUsed. + +background_view_indexing_threshold() -> + config:get_integer(?SECTION, "background_view_indexing_threshold", 80). + +interactive_view_indexing_threshold() -> + config:get_integer(?SECTION, "interactive_view_indexing_threshold", 90). + +interactive_database_writes_threshold() -> + config:get_integer(?SECTION, "interactive_database_writes_threshold", 90). + +enabled() -> + config:get_boolean(?SECTION, "enable", false). + +%% Align our refresh with the os_mon refresh +timer_interval() -> + application:get_env(os_mon, disk_space_check_interval, 30) * 60_000. + +-ifdef(TEST). +-include_lib("couch/include/couch_eunit.hrl"). + +not_enabled_by_default_test() -> + ?assertEqual(false, enabled()), + ?assertEqual(false, block_background_view_indexing()), + ?assertEqual(false, block_interactive_view_indexing()), + ?assertEqual(false, block_interactive_database_writes()). + +setup_all() -> + Ctx = test_util:start_couch(), + config:set_boolean("disk_monitor", "enable", true, _Persist = false), + Ctx. + +teardown_all(Ctx) -> + test_util:stop_couch(Ctx). + +setup() -> + ok. + +teardown(_) -> + ok. + +enabled_test_() -> + { + setup, + fun setup_all/0, + fun teardown_all/1, + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(block_background_view_indexing_test), + ?TDEF_FE(block_interactive_view_indexing_test), + ?TDEF_FE(block_interactive_database_writes_test), + ?TDEF_FE(update_disk_data_test) + ] + } + }. + +block_background_view_indexing_test(_) -> + ?assert(enabled()), + set_view_index_dir_percent_used(80), + ?assertEqual(false, block_background_view_indexing()), + set_view_index_dir_percent_used(81), + ?assertEqual(true, block_background_view_indexing()). + +block_interactive_view_indexing_test(_) -> + ?assert(enabled()), + set_view_index_dir_percent_used(90), + ?assertEqual(false, block_interactive_view_indexing()), + set_view_index_dir_percent_used(91), + ?assertEqual(true, block_interactive_view_indexing()). + +block_interactive_database_writes_test(_) -> + ?assert(enabled()), + set_database_dir_percent_used(90), + ?assertEqual(false, block_interactive_database_writes()), + set_database_dir_percent_used(91), + ?assertEqual(true, block_interactive_database_writes()). + +update_disk_data_test(_) -> + whereis(?MODULE) ! update_disk_data, + ?assertEqual({error, unknown_msg}, gen_server:call(?MODULE, foo)). + +-endif. diff --git a/src/couch/src/couch_secondary_sup.erl b/src/couch/src/couch_secondary_sup.erl index cfe38bbd4..8fc3c9a13 100644 --- a/src/couch/src/couch_secondary_sup.erl +++ b/src/couch/src/couch_secondary_sup.erl @@ -26,7 +26,8 @@ init([]) -> [ {query_servers, {couch_proc_manager, start_link, []}}, {vhosts, {couch_httpd_vhost, start_link, []}}, - {uuids, {couch_uuids, start, []}} + {uuids, {couch_uuids, start, []}}, + {disk_manager, {couch_disk_monitor, start_link, []}} ] ++ couch_index_servers(), MaybeHttp = diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl index 1a4a3ebcc..a50fcd670 100644 --- a/src/couch_mrview/src/couch_mrview.erl +++ b/src/couch_mrview/src/couch_mrview.erl @@ -291,8 +291,8 @@ query_view(Db, DDoc, VName, Args0, Callback, Acc0) -> _ -> {ok, Acc0} end, query_view(Db, VInfo, Args, Callback, Acc1); - ddoc_updated -> - Callback(ok, ddoc_updated) + Error when Error == ddoc_updated; Error == insufficient_storage -> + Callback(ok, Error) end. get_view_index_pid(Db, DDoc, ViewName, Args0) -> @@ -752,8 +752,8 @@ default_cb({final, Info}, []) -> {ok, [Info]}; default_cb({final, _}, Acc) -> {ok, Acc}; -default_cb(ok, ddoc_updated) -> - {ok, ddoc_updated}; +default_cb(ok, Error) when Error == ddoc_updated; Error == insufficient_storage -> + {ok, Error}; default_cb(Row, Acc) -> {ok, [Row | Acc]}. diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index 5913aa3d0..a478685da 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -158,8 +158,8 @@ get_view(Db, DDoc, ViewName, Args0) -> check_range(Args3, view_cmp(View)), Sig = view_sig(Db, State, View, Args3), {ok, {Type, View, Ref}, Sig, Args3}; - ddoc_updated -> - ddoc_updated + Error when Error == ddoc_updated; Error == insufficient_storage -> + Error end. get_view_index_pid(Db, DDoc, ViewName, Args0) -> @@ -176,6 +176,7 @@ get_view_index_state(_, DDoc, _, _, RetryCount) when RetryCount < 0 -> couch_log:warning("DDoc '~s' recreated too frequently", [DDoc#doc.id]), throw({get_view_state, exceeded_retry_count}); get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) -> + BlockInteractiveIndexing = couch_disk_monitor:block_interactive_view_indexing(), try {ok, Pid, Args} = get_view_index_pid(Db, DDoc, ViewName, Args0), UpdateSeq = couch_util:with_db(Db, fun(WDb) -> @@ -190,12 +191,24 @@ get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) -> couch_index:get_state(Pid, 0); false -> couch_index:get_state(Pid, 0); + _ when BlockInteractiveIndexing -> + %% if the view is already fresh enough, proceed. + %% otherwise it's an error until there's room to update. + case couch_index:get_state(Pid, 0) of + {ok, #mrst{update_seq = MaybeSeq} = St} when MaybeSeq >= UpdateSeq -> + {ok, St}; + {ok, #mrst{}} -> + insufficient_storage; + Else0 -> + Else0 + end; _ -> couch_index:get_state(Pid, UpdateSeq) end, case State of {ok, State0} -> {ok, State0, Args}; ddoc_updated -> ddoc_updated; + insufficient_storage -> insufficient_storage; Else -> throw(Else) end catch diff --git a/src/docs/src/config/disk-monitor.rst b/src/docs/src/config/disk-monitor.rst new file mode 100644 index 000000000..ff6ad9e45 --- /dev/null +++ b/src/docs/src/config/disk-monitor.rst @@ -0,0 +1,76 @@ +.. Licensed under the Apache License, Version 2.0 (the "License"); you may not +.. use this file except in compliance with the License. You may obtain a copy of +.. the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +.. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +.. License for the specific language governing permissions and limitations under +.. the License. + +.. default-domain:: config +.. highlight:: ini + +========================== +Disk Monitor Configuration +========================== + +Apache CouchDB can react proactively when disk space gets low. + +.. _config/disk_monitor: + +Disk Monitor Options +==================== + +.. config:section:: disk_monitor :: Disk Monitor Options + + .. versionadded:: 3.4 + + .. config:option:: background_view_indexing_threshold + + The percentage of used disk space on the ``view_index_dir`` above + which CouchDB will no longer start background view indexing jobs. + Defaults to ``80``. :: + + [disk_monitor] + background_view_indexing_threshold = 80 + + .. config:option:: interactive_database_writes_threshold + + The percentage of used disk space on the ``database_dir`` above + which CouchDB will no longer allow interactive document updates + (writes or deletes). + + Replicated updates and database deletions are still permitted. + + In a clustered write an error will be returned if + enough nodes are above the ``interactive_database_writes_threshold``. + + Defaults to ``90``. :: + + [disk_monitor] + interactive_database_writes_threshold = 90 + + .. config:option:: enable :: enable disk monitoring + + Enable disk monitoring subsystem. Defaults to ``false``. :: + + [disk_monitor] + enable = false + + .. config:option:: interactive_view_indexing_threshold + + The percentage of used disk space on the ``view_index_dir`` above + which CouchDB will no longer update stale view indexes when queried. + + View indexes that are already up to date can still be queried, and stale + view indexes can be queried if either ``stale=ok`` or ``update=false`` are + set. + + Attempts to query a stale index without either parameter will yield a + ``507 Insufficient Storage`` error. Defaults to ``90``. :: + + [disk_monitor] + interactive_view_indexing_threshold = 90 diff --git a/src/docs/src/config/index.rst b/src/docs/src/config/index.rst index 7b5f1ba20..252697ffa 100644 --- a/src/docs/src/config/index.rst +++ b/src/docs/src/config/index.rst @@ -23,6 +23,7 @@ Configuration couchdb cluster couch-peruser + disk-monitor http auth compaction diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index b781eea99..5c1ddd573 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -493,7 +493,9 @@ view_cb(complete, Acc) -> ok = rexi:stream_last(complete), {ok, Acc}; view_cb(ok, ddoc_updated) -> - rexi:reply({ok, ddoc_updated}). + rexi:reply({ok, ddoc_updated}); +view_cb(ok, insufficient_storage) -> + rexi:reply({ok, insufficient_storage}). reduce_cb({meta, Meta}, Acc) -> % Map function starting @@ -511,7 +513,9 @@ reduce_cb(complete, Acc) -> ok = rexi:stream_last(complete), {ok, Acc}; reduce_cb(ok, ddoc_updated) -> - rexi:reply({ok, ddoc_updated}). + rexi:reply({ok, ddoc_updated}); +reduce_cb(ok, insufficient_storage) -> + rexi:reply({ok, insufficient_storage}). changes_enumerator(#full_doc_info{} = FDI, Acc) -> changes_enumerator(couch_doc:to_doc_info(FDI), Acc); diff --git a/src/fabric/src/fabric_streams.erl b/src/fabric/src/fabric_streams.erl index 2a3a2b004..318824814 100644 --- a/src/fabric/src/fabric_streams.erl +++ b/src/fabric/src/fabric_streams.erl @@ -144,11 +144,11 @@ handle_stream_start(rexi_STREAM_INIT, {Worker, From}, St) -> {stop, St#stream_acc{workers = [], ready = Ready1}} end end; -handle_stream_start({ok, ddoc_updated}, _, St) -> +handle_stream_start({ok, Error}, _, St) when Error == ddoc_updated; Error == insufficient_storage -> WaitingWorkers = [W || {W, _} <- St#stream_acc.workers], ReadyWorkers = [W || {W, _} <- St#stream_acc.ready], cleanup(WaitingWorkers ++ ReadyWorkers), - {stop, ddoc_updated}; + {stop, Error}; handle_stream_start(Else, _, _) -> exit({invalid_stream_start, Else}). diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl index 82f9c2bc6..bad5e32c2 100644 --- a/src/fabric/src/fabric_view.erl +++ b/src/fabric/src/fabric_view.erl @@ -415,7 +415,14 @@ maybe_update_others( ViewName, #mrargs{update = lazy} = Args ) -> - ShardsNeedUpdated = mem3:shards(DbName) -- ShardsInvolved, + BlockInteractiveIndexing = couch_disk_monitor:block_interactive_view_indexing(), + ShardsNeedUpdated = + case BlockInteractiveIndexing of + true -> + []; + false -> + mem3:shards(DbName) -- ShardsInvolved + end, lists:foreach( fun(#shard{node = Node, name = ShardName}) -> rpc:cast(Node, fabric_rpc, update_mrview, [ShardName, DDoc, ViewName, Args]) diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl index 7b3b75863..43922d5d5 100644 --- a/src/fabric/src/fabric_view_map.erl +++ b/src/fabric/src/fabric_view_map.erl @@ -56,6 +56,10 @@ go(Db, Options, DDoc, View, Args0, Callback, Acc, VInfo) -> of {ok, ddoc_updated} -> Callback({error, ddoc_updated}, Acc); + {ok, insufficient_storage} -> + Callback( + {error, {insufficient_storage, <<"not enough room to update index">>}}, Acc + ); {ok, Workers} -> try go(DbName, Workers, VInfo, CoordArgs, Callback, Acc) @@ -207,6 +211,8 @@ handle_message({execution_stats, _} = Msg, {_, From}, St) -> rexi:stream_ack(From), {Go, St#collector{user_acc = Acc}}; handle_message(ddoc_updated, _Worker, State) -> + {stop, State}; +handle_message(insufficient_storage, _Worker, State) -> {stop, State}. merge_row(Dir, Collation, undefined, Row, Rows0) -> diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl index 90fa523a1..d4d17d5e1 100644 --- a/src/fabric/src/fabric_view_reduce.erl +++ b/src/fabric/src/fabric_view_reduce.erl @@ -47,6 +47,8 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) -> of {ok, ddoc_updated} -> Callback({error, ddoc_updated}, Acc); + {ok, insufficient_storage} -> + Callback({error, insufficient_storage}, Acc); {ok, Workers} -> try go2(DbName, Workers, VInfo, CoordArgs, Callback, Acc) @@ -175,6 +177,8 @@ handle_message(complete, Worker, #collector{counters = Counters0} = State) -> C1 = fabric_dict:update_counter(Worker, 1, Counters0), fabric_view:maybe_send_row(State#collector{counters = C1}); handle_message(ddoc_updated, _Worker, State) -> + {stop, State}; +handle_message(insufficient_storage, _Worker, State) -> {stop, State}. os_proc_needed(<<"_", _/binary>>) -> false; diff --git a/src/ken/src/ken_server.erl b/src/ken/src/ken_server.erl index 929ba47d7..c9cfdf31e 100644 --- a/src/ken/src/ken_server.erl +++ b/src/ken/src/ken_server.erl @@ -487,11 +487,16 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) -> IncrementalChannels = list_to_integer(config("incremental_channels", "80")), BatchChannels = list_to_integer(config("batch_channels", "20")), TotalChannels = IncrementalChannels + BatchChannels, + BlockBackgroundViewIndexing = couch_disk_monitor:block_background_view_indexing(), + % View name has two elements + IsView = tuple_size(Name) == 2, A = get_active_count(), #state{delay = Delay, batch_size = BS} = State, case ets:lookup(ken_workers, Name) of [] -> if + BlockBackgroundViewIndexing andalso IsView -> + false; A < BatchChannels -> true; A < TotalChannels -> @@ -501,7 +506,7 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) -> {ok, CurrentSeq} = hastings_index:await(Pid, 0), (Seq - CurrentSeq) < Threshold; % View name has two elements. - {_, _} -> + _ when IsView -> % Since seq is 0, couch_index:get_state/2 won't % spawn an index update. {ok, MRSt} = couch_index:get_state(Pid, 0), @@ -525,6 +530,8 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) -> Now = erlang:monotonic_time(), DeltaT = erlang:convert_time_unit(Now - LRU, native, millisecond), if + BlockBackgroundViewIndexing andalso IsView -> + false; A < BatchChannels, (Seq - OldSeq) >= BS -> true; A < BatchChannels, DeltaT > Delay ->