This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch out-of-disk-handler
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0bbf094697327b1f9267050f91686fc69e44b108
Author: Robert Newson <rnew...@apache.org>
AuthorDate: Fri Jun 16 09:25:41 2023 +0100
Introduce optional countermeasures as we run out of disk space
---
rel/overlay/etc/default.ini | 6 +
rel/overlay/etc/vm.args | 9 ++
rel/reltool.config | 2 +
src/chttpd/src/chttpd.erl | 2 +
src/couch/priv/stats_descriptions.cfg | 4 +
src/couch/src/couch.app.src | 1 +
src/couch/src/couch_db.erl | 9 ++
src/couch/src/couch_disk_monitor.erl | 241 +++++++++++++++++++++++++++++
src/couch/src/couch_secondary_sup.erl | 3 +-
src/couch_mrview/src/couch_mrview.erl | 8 +-
src/couch_mrview/src/couch_mrview_util.erl | 17 +-
src/docs/src/config/disk-monitor.rst | 76 +++++++++
src/docs/src/config/index.rst | 1 +
src/fabric/src/fabric_rpc.erl | 8 +-
src/fabric/src/fabric_streams.erl | 4 +-
src/fabric/src/fabric_view.erl | 9 +-
src/fabric/src/fabric_view_map.erl | 6 +
src/fabric/src/fabric_view_reduce.erl | 4 +
src/ken/src/ken_server.erl | 9 +-
test/elixir/test/config/suite.elixir | 5 +
test/elixir/test/disk_monitor.exs | 58 +++++++
21 files changed, 469 insertions(+), 13 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index bb440e0aa..c3124a643 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -898,3 +898,9 @@ port = {{prometheus_port}}
[nouveau]
enable = {{with_nouveau}}
+
+[disk_monitor]
+;enable = false
+;background_view_indexing_threshold = 80
+;interactive_view_indexing_threshold = 90
+;interactive_database_writes_threshold = 90
diff --git a/rel/overlay/etc/vm.args b/rel/overlay/etc/vm.args
index 886bbb903..8b2440fc8 100644
--- a/rel/overlay/etc/vm.args
+++ b/rel/overlay/etc/vm.args
@@ -122,3 +122,12 @@
# in the features list.
#
#-crypto fips_mode true
+
+# OS Mon Settings
+
+# only start disksup
+-os_mon start_cpu_sup false
+-os_mon start_memsup false
+
+# Check disk space every 5 minutes
+-os_mon disk_space_check_interval 5
diff --git a/rel/reltool.config b/rel/reltool.config
index da9ad6d3b..1dca6ff85 100644
--- a/rel/reltool.config
+++ b/rel/reltool.config
@@ -23,6 +23,7 @@
sasl,
ssl,
stdlib,
+ os_mon,
syntax_tools,
xmerl,
%% couchdb
@@ -84,6 +85,7 @@
{app, sasl, [{incl_cond, include}]},
{app, ssl, [{incl_cond, include}]},
{app, stdlib, [{incl_cond, include}]},
+ {app, os_mon, [{incl_cond, include}]},
{app, syntax_tools, [{incl_cond, include}]},
{app, xmerl, [{incl_cond, include}]},
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 53abc731f..c8e6fdc97 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -1138,6 +1138,8 @@ error_info(timeout) ->
>>};
error_info({service_unavailable, Reason}) ->
{503, <<"service unavailable">>, Reason};
+error_info({insufficient_storage, Reason}) ->
+ {507, <<"insufficent_storage">>, Reason};
error_info({timeout, _Reason}) ->
error_info(timeout);
error_info({'EXIT', {Error, _Stack}}) ->
diff --git a/src/couch/priv/stats_descriptions.cfg
b/src/couch/priv/stats_descriptions.cfg
index 6c0d4dad2..1983eed9b 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -266,6 +266,10 @@
{type, counter},
{desc, <<"number of HTTP 503 Service unavailable responses">>}
]}.
+{[couchdb, httpd_status_codes, 507], [
+ {type, counter},
+ {desc, <<"number of HTTP 507 Insufficient Storage responses">>}
+]}.
{[couchdb, open_databases], [
{type, counter},
{desc, <<"number of open databases">>}
diff --git a/src/couch/src/couch.app.src b/src/couch/src/couch.app.src
index ef4e5e956..6fc293ac1 100644
--- a/src/couch/src/couch.app.src
+++ b/src/couch/src/couch.app.src
@@ -33,6 +33,7 @@
sasl,
inets,
ssl,
+ os_mon,
% Upstream deps
ibrowse,
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 4bc2aca8f..2ef89ced3 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -1338,6 +1338,15 @@ update_docs(Db, Docs0, Options, ?REPLICATED_CHANGES) ->
),
{ok, DocErrors};
update_docs(Db, Docs0, Options, ?INTERACTIVE_EDIT) ->
+ BlockInteractiveDatabaseWrites =
couch_disk_monitor:block_interactive_database_writes(),
+ if
+ BlockInteractiveDatabaseWrites ->
+ {ok, [{insufficient_storage, <<"database_dir is too full">>} || _
<- Docs0]};
+ true ->
+ update_docs_interactive(Db, Docs0, Options)
+ end.
+
+update_docs_interactive(Db, Docs0, Options) ->
Docs = tag_docs(Docs0),
AllOrNothing = lists:member(all_or_nothing, Options),
diff --git a/src/couch/src/couch_disk_monitor.erl
b/src/couch/src/couch_disk_monitor.erl
new file mode 100644
index 000000000..62c34085c
--- /dev/null
+++ b/src/couch/src/couch_disk_monitor.erl
@@ -0,0 +1,241 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_disk_monitor).
+-behaviour(gen_server).
+
+% public api
+-export([
+ block_background_view_indexing/0,
+ block_interactive_view_indexing/0,
+ block_interactive_database_writes/0
+]).
+
+% testing
+-export([
+ set_database_dir_percent_used/1,
+ set_view_index_dir_percent_used/1
+]).
+
+% gen_server callbacks
+-export([
+ start_link/0,
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2
+]).
+
+-include_lib("kernel/include/file.hrl").
+
+-define(SECTION, "disk_monitor").
+
+-record(st, {
+ timer
+}).
+
+block_background_view_indexing() ->
+ Enabled = enabled(),
+ if
+ Enabled ->
+ view_index_dir_percent_used() >
background_view_indexing_threshold();
+ true ->
+ false
+ end.
+
+block_interactive_view_indexing() ->
+ Enabled = enabled(),
+ if
+ Enabled -> view_index_dir_percent_used() >
interactive_view_indexing_threshold();
+ true -> false
+ end.
+
+block_interactive_database_writes() ->
+ Enabled = enabled(),
+ if
+ Enabled -> database_dir_percent_used() >
interactive_database_writes_threshold();
+ true -> false
+ end.
+
+set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed
=< 100 ->
+ gen_server:call(?MODULE, {set_database_dir_percent_used, PercentUsed}).
+
+set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0,
PercentUsed =< 100 ->
+ gen_server:call(?MODULE, {set_view_index_dir_percent_used, PercentUsed}).
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_Args) ->
+ ets:new(?MODULE, [named_table, {read_concurrency, true}]),
+ ets:insert(?MODULE, {database_dir, 0}),
+ ets:insert(?MODULE, {view_index_dir, 0}),
+ update_disk_data(),
+ {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+ {ok, #st{timer = TRef}}.
+
+handle_call({set_database_dir_percent_used, PercentUsed}, _From, St) ->
+ ets:insert(?MODULE, {database_dir, PercentUsed}),
+ {reply, ok, St};
+handle_call({set_view_index_dir_percent_used, PercentUsed}, _From, St) ->
+ ets:insert(?MODULE, {view_index_dir, PercentUsed}),
+ {reply, ok, St};
+handle_call(_Msg, _From, St) ->
+ {reply, {error, unknown_msg}, St}.
+
+handle_cast(_Msg, St) ->
+ {noreply, St}.
+
+handle_info(update_disk_data, St) ->
+ timer:cancel(St#st.timer),
+ update_disk_data(),
+ {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+ {noreply, St#st{timer = TRef}};
+handle_info(_Msg, St) ->
+ {noreply, St}.
+
+update_disk_data() ->
+ DiskData = disksup:get_disk_data(),
+ update_disk_data(DiskData).
+
+update_disk_data([]) ->
+ ok;
+update_disk_data([{Id, _, PercentUsed} | Rest]) ->
+ IsDatabaseDir = is_database_dir(Id),
+ IsViewIndexDir = is_view_index_dir(Id),
+ if
+ IsDatabaseDir ->
+ ets:insert(?MODULE, {database_dir, PercentUsed});
+ true ->
+ ok
+ end,
+ if
+ IsViewIndexDir ->
+ ets:insert(?MODULE, {view_index_dir, PercentUsed});
+ true ->
+ ok
+ end,
+ update_disk_data(Rest).
+
+is_database_dir(MntOn) ->
+ same_device(config:get("couchdb", "database_dir"), MntOn).
+
+is_view_index_dir(MntOn) ->
+ same_device(config:get("couchdb", "view_index_dir"), MntOn).
+
+same_device(DirA, DirB) ->
+ case {device_id(DirA), device_id(DirB)} of
+ {{ok, DeviceId}, {ok, DeviceId}} ->
+ true;
+ _Else ->
+ false
+ end.
+
+device_id(Dir) ->
+ case file:read_file_info(Dir) of
+ {ok, FileInfo} ->
+ {ok, {FileInfo#file_info.minor_device,
FileInfo#file_info.major_device}};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+database_dir_percent_used() ->
+ [{database_dir, PercentUsed}] = ets:lookup(?MODULE, database_dir),
+ PercentUsed.
+
+view_index_dir_percent_used() ->
+ [{view_index_dir, PercentUsed}] = ets:lookup(?MODULE, view_index_dir),
+ PercentUsed.
+
+background_view_indexing_threshold() ->
+ config:get_integer(?SECTION, "background_view_indexing_threshold", 80).
+
+interactive_view_indexing_threshold() ->
+ config:get_integer(?SECTION, "interactive_view_indexing_threshold", 90).
+
+interactive_database_writes_threshold() ->
+ config:get_integer(?SECTION, "interactive_database_writes_threshold", 90).
+
+enabled() ->
+ config:get_boolean(?SECTION, "enable", false).
+
+%% Align our refresh with the os_mon refresh
+timer_interval() ->
+ application:get_env(os_mon, disk_space_check_interval, 30) * 60_000.
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+not_enabled_by_default_test() ->
+ ?assertEqual(false, enabled()),
+ ?assertEqual(false, block_background_view_indexing()),
+ ?assertEqual(false, block_interactive_view_indexing()),
+ ?assertEqual(false, block_interactive_database_writes()).
+
+setup_all() ->
+ Ctx = test_util:start_couch(),
+ config:set_boolean("disk_monitor", "enable", true, _Persist = false),
+ Ctx.
+
+teardown_all(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+setup() ->
+ ok.
+
+teardown(_) ->
+ ok.
+
+enabled_test_() ->
+ {
+ setup,
+ fun setup_all/0,
+ fun teardown_all/1,
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TDEF_FE(block_background_view_indexing_test),
+ ?TDEF_FE(block_interactive_view_indexing_test),
+ ?TDEF_FE(block_interactive_database_writes_test),
+ ?TDEF_FE(update_disk_data_test)
+ ]
+ }
+ }.
+
+block_background_view_indexing_test(_) ->
+ ?assert(enabled()),
+ set_view_index_dir_percent_used(80),
+ ?assertEqual(false, block_background_view_indexing()),
+ set_view_index_dir_percent_used(81),
+ ?assertEqual(true, block_background_view_indexing()).
+
+block_interactive_view_indexing_test(_) ->
+ ?assert(enabled()),
+ set_view_index_dir_percent_used(90),
+ ?assertEqual(false, block_interactive_view_indexing()),
+ set_view_index_dir_percent_used(91),
+ ?assertEqual(true, block_interactive_view_indexing()).
+
+block_interactive_database_writes_test(_) ->
+ ?assert(enabled()),
+ set_database_dir_percent_used(90),
+ ?assertEqual(false, block_interactive_database_writes()),
+ set_database_dir_percent_used(91),
+ ?assertEqual(true, block_interactive_database_writes()).
+
+update_disk_data_test(_) ->
+ whereis(?MODULE) ! update_disk_data,
+ ?assertEqual({error, unknown_msg}, gen_server:call(?MODULE, foo)).
+
+-endif.
diff --git a/src/couch/src/couch_secondary_sup.erl
b/src/couch/src/couch_secondary_sup.erl
index cfe38bbd4..8fc3c9a13 100644
--- a/src/couch/src/couch_secondary_sup.erl
+++ b/src/couch/src/couch_secondary_sup.erl
@@ -26,7 +26,8 @@ init([]) ->
[
{query_servers, {couch_proc_manager, start_link, []}},
{vhosts, {couch_httpd_vhost, start_link, []}},
- {uuids, {couch_uuids, start, []}}
+ {uuids, {couch_uuids, start, []}},
+ {disk_manager, {couch_disk_monitor, start_link, []}}
] ++ couch_index_servers(),
MaybeHttp =
diff --git a/src/couch_mrview/src/couch_mrview.erl
b/src/couch_mrview/src/couch_mrview.erl
index 1a4a3ebcc..a50fcd670 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -291,8 +291,8 @@ query_view(Db, DDoc, VName, Args0, Callback, Acc0) ->
_ -> {ok, Acc0}
end,
query_view(Db, VInfo, Args, Callback, Acc1);
- ddoc_updated ->
- Callback(ok, ddoc_updated)
+ Error when Error == ddoc_updated; Error == insufficient_storage ->
+ Callback(ok, Error)
end.
get_view_index_pid(Db, DDoc, ViewName, Args0) ->
@@ -752,8 +752,8 @@ default_cb({final, Info}, []) ->
{ok, [Info]};
default_cb({final, _}, Acc) ->
{ok, Acc};
-default_cb(ok, ddoc_updated) ->
- {ok, ddoc_updated};
+default_cb(ok, Error) when Error == ddoc_updated; Error ==
insufficient_storage ->
+ {ok, Error};
default_cb(Row, Acc) ->
{ok, [Row | Acc]}.
diff --git a/src/couch_mrview/src/couch_mrview_util.erl
b/src/couch_mrview/src/couch_mrview_util.erl
index 5913aa3d0..a478685da 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -158,8 +158,8 @@ get_view(Db, DDoc, ViewName, Args0) ->
check_range(Args3, view_cmp(View)),
Sig = view_sig(Db, State, View, Args3),
{ok, {Type, View, Ref}, Sig, Args3};
- ddoc_updated ->
- ddoc_updated
+ Error when Error == ddoc_updated; Error == insufficient_storage ->
+ Error
end.
get_view_index_pid(Db, DDoc, ViewName, Args0) ->
@@ -176,6 +176,7 @@ get_view_index_state(_, DDoc, _, _, RetryCount) when
RetryCount < 0 ->
couch_log:warning("DDoc '~s' recreated too frequently", [DDoc#doc.id]),
throw({get_view_state, exceeded_retry_count});
get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) ->
+ BlockInteractiveIndexing =
couch_disk_monitor:block_interactive_view_indexing(),
try
{ok, Pid, Args} = get_view_index_pid(Db, DDoc, ViewName, Args0),
UpdateSeq = couch_util:with_db(Db, fun(WDb) ->
@@ -190,12 +191,24 @@ get_view_index_state(Db, DDoc, ViewName, Args0,
RetryCount) ->
couch_index:get_state(Pid, 0);
false ->
couch_index:get_state(Pid, 0);
+ _ when BlockInteractiveIndexing ->
+ %% if the view is already fresh enough, proceed.
+ %% otherwise it's an error until there's room to update.
+ case couch_index:get_state(Pid, 0) of
+ {ok, #mrst{update_seq = MaybeSeq} = St} when MaybeSeq
>= UpdateSeq ->
+ {ok, St};
+ {ok, #mrst{}} ->
+ insufficient_storage;
+ Else0 ->
+ Else0
+ end;
_ ->
couch_index:get_state(Pid, UpdateSeq)
end,
case State of
{ok, State0} -> {ok, State0, Args};
ddoc_updated -> ddoc_updated;
+ insufficient_storage -> insufficient_storage;
Else -> throw(Else)
end
catch
diff --git a/src/docs/src/config/disk-monitor.rst
b/src/docs/src/config/disk-monitor.rst
new file mode 100644
index 000000000..ff6ad9e45
--- /dev/null
+++ b/src/docs/src/config/disk-monitor.rst
@@ -0,0 +1,76 @@
+.. Licensed under the Apache License, Version 2.0 (the "License"); you may not
+.. use this file except in compliance with the License. You may obtain a copy
of
+.. the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+.. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+.. License for the specific language governing permissions and limitations
under
+.. the License.
+
+.. default-domain:: config
+.. highlight:: ini
+
+==========================
+Disk Monitor Configuration
+==========================
+
+Apache CouchDB can react proactively when disk space gets low.
+
+.. _config/disk_monitor:
+
+Disk Monitor Options
+====================
+
+.. config:section:: disk_monitor :: Disk Monitor Options
+
+ .. versionadded:: 3.4
+
+ .. config:option:: background_view_indexing_threshold
+
+ The percentage of used disk space on the ``view_index_dir`` above
+ which CouchDB will no longer start background view indexing jobs.
+ Defaults to ``80``. ::
+
+ [disk_monitor]
+ background_view_indexing_threshold = 80
+
+ .. config:option:: interactive_database_writes_threshold
+
+ The percentage of used disk space on the ``database_dir`` above
+ which CouchDB will no longer allow interactive document updates
+ (writes or deletes).
+
+ Replicated updates and database deletions are still permitted.
+
+ In a clustered write an error will be returned if
+ enough nodes are above the ``interactive_database_writes_threshold``.
+
+ Defaults to ``90``. ::
+
+ [disk_monitor]
+ interactive_database_writes_threshold = 90
+
+ .. config:option:: enable :: enable disk monitoring
+
+ Enable disk monitoring subsystem. Defaults to ``false``. ::
+
+ [disk_monitor]
+ enable = false
+
+ .. config:option:: interactive_view_indexing_threshold
+
+ The percentage of used disk space on the ``view_index_dir`` above
+ which CouchDB will no longer update stale view indexes when queried.
+
+ View indexes that are already up to date can still be queried, and
stale
+ view indexes can be queried if either ``stale=ok`` or ``update=false``
are
+ set.
+
+ Attempts to query a stale index without either parameter will yield a
+ ``507 Insufficient Storage`` error. Defaults to ``90``. ::
+
+ [disk_monitor]
+ interactive_view_indexing_threshold = 90
diff --git a/src/docs/src/config/index.rst b/src/docs/src/config/index.rst
index 7b5f1ba20..252697ffa 100644
--- a/src/docs/src/config/index.rst
+++ b/src/docs/src/config/index.rst
@@ -23,6 +23,7 @@ Configuration
couchdb
cluster
couch-peruser
+ disk-monitor
http
auth
compaction
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index b781eea99..5c1ddd573 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -493,7 +493,9 @@ view_cb(complete, Acc) ->
ok = rexi:stream_last(complete),
{ok, Acc};
view_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
+ rexi:reply({ok, ddoc_updated});
+view_cb(ok, insufficient_storage) ->
+ rexi:reply({ok, insufficient_storage}).
reduce_cb({meta, Meta}, Acc) ->
% Map function starting
@@ -511,7 +513,9 @@ reduce_cb(complete, Acc) ->
ok = rexi:stream_last(complete),
{ok, Acc};
reduce_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
+ rexi:reply({ok, ddoc_updated});
+reduce_cb(ok, insufficient_storage) ->
+ rexi:reply({ok, insufficient_storage}).
changes_enumerator(#full_doc_info{} = FDI, Acc) ->
changes_enumerator(couch_doc:to_doc_info(FDI), Acc);
diff --git a/src/fabric/src/fabric_streams.erl
b/src/fabric/src/fabric_streams.erl
index 2a3a2b004..318824814 100644
--- a/src/fabric/src/fabric_streams.erl
+++ b/src/fabric/src/fabric_streams.erl
@@ -144,11 +144,11 @@ handle_stream_start(rexi_STREAM_INIT, {Worker, From}, St)
->
{stop, St#stream_acc{workers = [], ready = Ready1}}
end
end;
-handle_stream_start({ok, ddoc_updated}, _, St) ->
+handle_stream_start({ok, Error}, _, St) when Error == ddoc_updated; Error ==
insufficient_storage ->
WaitingWorkers = [W || {W, _} <- St#stream_acc.workers],
ReadyWorkers = [W || {W, _} <- St#stream_acc.ready],
cleanup(WaitingWorkers ++ ReadyWorkers),
- {stop, ddoc_updated};
+ {stop, Error};
handle_stream_start(Else, _, _) ->
exit({invalid_stream_start, Else}).
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index 22535a9f9..87f464b36 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -419,7 +419,14 @@ maybe_update_others(
ViewName,
#mrargs{update = lazy} = Args
) ->
- ShardsNeedUpdated = mem3:shards(DbName) -- ShardsInvolved,
+ BlockInteractiveIndexing =
couch_disk_monitor:block_interactive_view_indexing(),
+ ShardsNeedUpdated =
+ case BlockInteractiveIndexing of
+ true ->
+ [];
+ false ->
+ mem3:shards(DbName) -- ShardsInvolved
+ end,
lists:foreach(
fun(#shard{node = Node, name = ShardName}) ->
rpc:cast(Node, fabric_rpc, update_mrview, [ShardName, DDoc,
ViewName, Args])
diff --git a/src/fabric/src/fabric_view_map.erl
b/src/fabric/src/fabric_view_map.erl
index 7b3b75863..43922d5d5 100644
--- a/src/fabric/src/fabric_view_map.erl
+++ b/src/fabric/src/fabric_view_map.erl
@@ -56,6 +56,10 @@ go(Db, Options, DDoc, View, Args0, Callback, Acc, VInfo) ->
of
{ok, ddoc_updated} ->
Callback({error, ddoc_updated}, Acc);
+ {ok, insufficient_storage} ->
+ Callback(
+ {error, {insufficient_storage, <<"not enough room to
update index">>}}, Acc
+ );
{ok, Workers} ->
try
go(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
@@ -207,6 +211,8 @@ handle_message({execution_stats, _} = Msg, {_, From}, St) ->
rexi:stream_ack(From),
{Go, St#collector{user_acc = Acc}};
handle_message(ddoc_updated, _Worker, State) ->
+ {stop, State};
+handle_message(insufficient_storage, _Worker, State) ->
{stop, State}.
merge_row(Dir, Collation, undefined, Row, Rows0) ->
diff --git a/src/fabric/src/fabric_view_reduce.erl
b/src/fabric/src/fabric_view_reduce.erl
index 90fa523a1..d4d17d5e1 100644
--- a/src/fabric/src/fabric_view_reduce.erl
+++ b/src/fabric/src/fabric_view_reduce.erl
@@ -47,6 +47,8 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
of
{ok, ddoc_updated} ->
Callback({error, ddoc_updated}, Acc);
+ {ok, insufficient_storage} ->
+ Callback({error, insufficient_storage}, Acc);
{ok, Workers} ->
try
go2(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
@@ -175,6 +177,8 @@ handle_message(complete, Worker, #collector{counters =
Counters0} = State) ->
C1 = fabric_dict:update_counter(Worker, 1, Counters0),
fabric_view:maybe_send_row(State#collector{counters = C1});
handle_message(ddoc_updated, _Worker, State) ->
+ {stop, State};
+handle_message(insufficient_storage, _Worker, State) ->
{stop, State}.
os_proc_needed(<<"_", _/binary>>) -> false;
diff --git a/src/ken/src/ken_server.erl b/src/ken/src/ken_server.erl
index 929ba47d7..c9cfdf31e 100644
--- a/src/ken/src/ken_server.erl
+++ b/src/ken/src/ken_server.erl
@@ -487,11 +487,16 @@ should_start_job(#job{name = Name, seq = Seq, server =
Pid}, State) ->
IncrementalChannels = list_to_integer(config("incremental_channels",
"80")),
BatchChannels = list_to_integer(config("batch_channels", "20")),
TotalChannels = IncrementalChannels + BatchChannels,
+ BlockBackgroundViewIndexing =
couch_disk_monitor:block_background_view_indexing(),
+ % View name has two elements
+ IsView = tuple_size(Name) == 2,
A = get_active_count(),
#state{delay = Delay, batch_size = BS} = State,
case ets:lookup(ken_workers, Name) of
[] ->
if
+ BlockBackgroundViewIndexing andalso IsView ->
+ false;
A < BatchChannels ->
true;
A < TotalChannels ->
@@ -501,7 +506,7 @@ should_start_job(#job{name = Name, seq = Seq, server =
Pid}, State) ->
{ok, CurrentSeq} = hastings_index:await(Pid, 0),
(Seq - CurrentSeq) < Threshold;
% View name has two elements.
- {_, _} ->
+ _ when IsView ->
% Since seq is 0, couch_index:get_state/2 won't
% spawn an index update.
{ok, MRSt} = couch_index:get_state(Pid, 0),
@@ -525,6 +530,8 @@ should_start_job(#job{name = Name, seq = Seq, server =
Pid}, State) ->
Now = erlang:monotonic_time(),
DeltaT = erlang:convert_time_unit(Now - LRU, native, millisecond),
if
+ BlockBackgroundViewIndexing andalso IsView ->
+ false;
A < BatchChannels, (Seq - OldSeq) >= BS ->
true;
A < BatchChannels, DeltaT > Delay ->
diff --git a/test/elixir/test/config/suite.elixir
b/test/elixir/test/config/suite.elixir
index a79b3044b..1cbaae0f6 100644
--- a/test/elixir/test/config/suite.elixir
+++ b/test/elixir/test/config/suite.elixir
@@ -194,6 +194,11 @@
"design doc options - include_desing=true",
"design doc options - local_seq=true"
],
+ "DiskMonitorTest": [
+ "block background view indexing",
+ "block interactive view indexing",
+ "block interactive database writes"
+ ],
"DesignPathTest": [
"design doc path",
"design doc path with slash in db name"
diff --git a/test/elixir/test/disk_monitor.exs
b/test/elixir/test/disk_monitor.exs
new file mode 100644
index 000000000..6da73d585
--- /dev/null
+++ b/test/elixir/test/disk_monitor.exs
@@ -0,0 +1,58 @@
+defmodule DiskMonitorTest do
+ use CouchTestCase
+
+ @moduletag :disk_monitor
+
+ setup_all do
+ set_config({"disk_monitor", "enable", "true"})
+ #on_exit(fn ->
+ # set_config({"disk_monitor", "enable", "false"})
+ #end
+ :ok
+ end
+
+ @tag :with_db
+ test "block background view indexing", context do
+ set_config({"disk_monitor", "background_view_indexing_threshold", "0"})
+ set_config({"disk_monitor", "interactive_view_indexing_threshold", "100"})
+
+ db_name = context[:db_name]
+ resp = Couch.post("/#{db_name}", body: %{:_id => "foo"})
+ assert resp.status_code == 201
+
+ map_doc = %{:views => %{:bar => %{:map => "function(doc) { emit(); }"}}}
+ assert Couch.put("/#{db_name}/_design/foo", body: map_doc).body["ok"]
+ :timer.sleep(500)
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar?stale=ok")
+ assert resp.body["total_rows"] == 0
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar")
+ assert resp.body["total_rows"] == 1
+ end
+
+ @tag :with_db
+ test "block interactive view indexing", context do
+ set_config({"disk_monitor", "background_view_indexing_threshold", "100"})
+ set_config({"disk_monitor", "interactive_view_indexing_threshold", "0"})
+
+ db_name = context[:db_name]
+ resp = Couch.post("/#{db_name}", body: %{:_id => "foo"})
+ assert resp.status_code == 201
+
+ map_doc = %{:views => %{:bar => %{:map => "function(doc) { emit(); }"}}}
+ assert Couch.put("/#{db_name}/_design/foo", body: map_doc).body["ok"]
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar")
+ assert resp.status_code == 507
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar?stale=ok")
+ assert resp.status_code == 200
+ end
+
+ @tag :with_db
+ test "block interactive database writes", context do
+ set_config({"disk_monitor", "interactive_database_writes_threshold", "0"})
+
+ db_name = context[:db_name]
+ resp = Couch.post("/#{db_name}", body: %{:_id => "foo"})
+ assert resp.status_code == 507
+ end
+
+end
