iilyak commented on a change in pull request #3766:
URL: https://github.com/apache/couchdb/pull/3766#discussion_r806760492
##########
File path: src/smoosh/src/smoosh_channel.erl
##########
@@ -65,60 +77,80 @@ close(ServerRef) ->
flush(ServerRef) ->
gen_server:call(ServerRef, flush).
+is_key(ServerRef, Key) ->
+ gen_server:call(ServerRef, {is_key, Key}).
+
+is_activated(ServerRef) ->
+ gen_server:call(ServerRef, is_activated).
+
% gen_server functions.
init(Name) ->
schedule_unpause(),
erlang:send_after(60 * 1000, self(), check_window),
- {ok, #state{name = Name}}.
+ process_flag(trap_exit, true),
+ Waiting = smoosh_priority_queue:new(Name),
+ State = #state{name = Name, waiting = Waiting, paused = true, activated =
false},
+ ok = gen_server:cast(self(), init),
+ {ok, State}.
-handle_call({last_updated, Object}, _From, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+handle_call({last_updated, Object}, _From, State) ->
LastUpdated = smoosh_priority_queue:last_updated(Object,
State#state.waiting),
{reply, LastUpdated, State};
-handle_call(suspend, _From, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+handle_call(suspend, _From, State) ->
#state{active = Active} = State,
[
catch erlang:suspend_process(Pid, [unless_suspending])
|| {_, Pid} <- Active
],
{reply, ok, State#state{paused = true}};
-handle_call(resume, _From, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+handle_call(resume, _From, State) ->
#state{active = Active} = State,
[catch erlang:resume_process(Pid) || {_, Pid} <- Active],
{reply, ok, State#state{paused = false}};
-handle_call(status, _From, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+handle_call(status, _From, State) ->
{reply,
{ok, [
{active, length(State#state.active)},
{starting, length(State#state.starting)},
{waiting, smoosh_priority_queue:info(State#state.waiting)}
]},
State};
-handle_call(close, _From, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+handle_call(close, _From, State) ->
{stop, normal, ok, State};
-handle_call(flush, _From, State0) ->
- {ok, State} = code_change(nil, State0, nil),
- {reply, ok, State#state{waiting = smoosh_priority_queue:new()}}.
+handle_call(flush, _From, #state{waiting = Q} = State) ->
+ {reply, ok, State#state{waiting = smoosh_priority_queue:flush(Q)}};
+handle_call({is_key, Key}, _From, State) ->
+ #state{waiting = Waiting} = State,
+ {reply, smoosh_priority_queue:is_key(Key, Waiting), State};
+handle_call(is_activated, _From, #state{activated = Activated} = State0) ->
+ {reply, Activated, State0}.
-handle_cast({enqueue, _Object, 0}, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+handle_cast(init, State0) ->
+ erlang:send_after(?START_DELAY_IN_MSEC, self(), start_recovery),
+ {noreply, State0};
+handle_cast({enqueue, _Object, 0}, #state{activated = true} = State) ->
{noreply, State};
-handle_cast({enqueue, Object, Priority}, State0) ->
- {ok, State} = code_change(nil, State0, nil),
- {noreply, maybe_start_compaction(add_to_queue(Object, Priority, State))}.
+handle_cast({enqueue, _Object, 0}, #state{activated = false} = State0) ->
Review comment:
I don't see any difference in the body with a clause on line 132. I
think we can combine the two.
```erlang
handle_cast({enqueue, _Object, 0}, #state{} = State) ->
{noreply, State};
```
##########
File path: src/smoosh/test/smoosh_priority_queue_tests.erl
##########
@@ -0,0 +1,164 @@
+-module(smoosh_priority_queue_tests).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+
+-define(PROP_PREFIX, "prop_").
+
+-define(CAPACITY, 3).
+
+-define(RANDOM_FILE, lists:flatten(io_lib:format("~p", [erlang:timestamp()]))).
Review comment:
It is confusing. In fact it is not a file, but a priority queue name.
Should we call this macro `RANDOM_CHANNEL`?
##########
File path: src/smoosh/test/smoosh_priority_queue_tests.erl
##########
@@ -0,0 +1,164 @@
+-module(smoosh_priority_queue_tests).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+
+-define(PROP_PREFIX, "prop_").
+
+-define(CAPACITY, 3).
+
+-define(RANDOM_FILE, lists:flatten(io_lib:format("~p", [erlang:timestamp()]))).
Review comment:
The `smoosh_priority_queue:file_name/1` depend on `config:get("smoosh",
"state_dir", ".")`.
```
file_name(#priority_queue{name = Name}) ->
filename:join(config:get("smoosh", "state_dir", "."), Name ++
".waiting").
```
By default we would use `.` which means that we would pollute
`src/smoosh/.eunit` directory with temporary files. I think we should configure
default value for `[smoosh]->state_dir` in `rel/overide/etc/default.ini` and
override it for tests in `rel/plugins/eunit_plugin.erl` similarly as we do for
`[couchdb]->database_dir`.
`rel/plugins/eunit_plugin.erl`
```erlang
build_eunit_config(Config0, AppFile) ->
Cwd = filename:absname(rebar_utils:get_cwd()),
DataDir = Cwd ++ "/tmp/data",
...
Config4 = rebar_config:set_global(Config3, state_dir, DataDir),
```
`rel/overlay/etc/default.ini`
```ini
[smoosh]
state_dir = {{state_dir}}
```
[`configure`](https://github.com/apache/couchdb/blob/3.x/configure#L234:L261)
```
{state_dir, "./data"}.
```
`setup_eunit.template`
```
{variables, [
...
{data_dir, "/tmp"},
{prefix, "/tmp"},
{view_index_dir, "/tmp"},
{state_dir, "/tmp"}
]}.
...
```
`dev/run`
```python
@log("Prepare configuration files")
def setup_configs(ctx):
....
env = {
"state_dir": toposixpath(
ensure_dir_exists(ctx["devdir"], "lib", node, "data")
),
```
##########
File path: src/smoosh/src/smoosh_channel.erl
##########
@@ -194,18 +224,139 @@ handle_info(check_window, State0) ->
end,
erlang:send_after(60 * 1000, self(), check_window),
{noreply, FinalState};
-handle_info(pause, State0) ->
- {ok, State} = code_change(nil, State0, nil),
+handle_info(start_recovery, #state{name = Name, waiting = Waiting0} = State0)
->
+ RecActive = recover(active_file_name(Name)),
+ Waiting1 = lists:foldl(
+ fun(DbName, Acc) ->
+ case couch_db:is_compacting(DbName) of
+ true ->
+ Priority = smoosh_server:get_priority(Name, DbName),
+ smoosh_priority_queue:in(DbName, Priority, Priority, Acc);
+ false ->
+ Acc
+ end
+ end,
+ Waiting0,
+ RecActive
+ ),
+ State1 = maybe_start_compaction(State0#state{paused = false, waiting =
Waiting1}),
+ couch_log:notice(
+ "~p Previously active compaction jobs (if any) have been successfully
recovered and restarted.",
+ [?MODULE]
+ ),
+ erlang:send_after(?ACTIVATE_DELAY_IN_MSEC, self(), activate),
+ {noreply, State1#state{paused = true}};
+handle_info(activate, #state{name = Name, waiting = Waiting0, requests =
Requests0} = State0) ->
+ RecStarting = recover(starting_file_name(Name)),
+ Starting = lists:foldl(
+ fun(DbName, Acc) ->
+ Priority = smoosh_server:get_priority(Name, DbName),
+ smoosh_priority_queue:in(DbName, Priority, Priority, Acc)
+ end,
+ Waiting0,
+ RecStarting
+ ),
+ Requests1 = lists:reverse(Requests0),
+ Waiting1 = lists:foldl(
+ fun({DbName, Priority}, Acc) ->
+ smoosh_priority_queue:in(DbName, Priority, Priority, Acc)
+ end,
+ Starting,
+ Requests1
+ ),
+ State1 = maybe_start_compaction(State0#state{
+ waiting = Waiting1, paused = false, requests = []
+ }),
+ handle_info(persist_queue, State1),
+ {noreply, State1#state{paused = true, activated = true}};
+handle_info(persist_queue, #state{waiting = Queue} = State) ->
+ write_state_to_file(State),
+ smoosh_priority_queue:write_to_file(Queue),
Review comment:
We write `waiting` into a disk. However, I couldn't find a place where
we read from that file.
##########
File path: src/smoosh/test/smoosh_priority_queue_tests.erl
##########
@@ -0,0 +1,164 @@
+-module(smoosh_priority_queue_tests).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+
+-define(PROP_PREFIX, "prop_").
+
+-define(CAPACITY, 3).
+
+-define(RANDOM_FILE, lists:flatten(io_lib:format("~p", [erlang:timestamp()]))).
+
+setup() ->
+ Ctx = test_util:start_couch(),
+ Ctx.
+
+teardown(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+smoosh_priority_queue_test_() ->
+ {
+ "smoosh priority queue test",
+ {
+ setup,
+ fun setup/0,
+ fun teardown/1,
+ [
+ fun prop_inverse_test_/0,
+ fun no_halt_on_corrupted_file_test/0,
+ fun no_halt_on_missing_file_test/0
+ ]
+ }
+ }.
+
+%% ==========
+%% Tests
+%% ----------
+
+%% define all tests to be able to run them individually
+prop_inverse_test_() ->
+ ?_test(begin
+ test_property(prop_inverse)
+ end).
+
+no_halt_on_corrupted_file_test() ->
+ ?_test(begin
+ Name = ?RANDOM_FILE,
+ Q = smoosh_priority_queue:new(Name),
+ FilePath = smoosh_priority_queue:file_name(Q),
+ ok = file:write_file(FilePath, <<"garbage">>),
Review comment:
We corrupt the file, but we don't test anything after that. Should we
attempt to read from it to see it returns an empty queue?
##########
File path: src/smoosh/test/smoosh_priority_queue_tests.erl
##########
@@ -0,0 +1,164 @@
+-module(smoosh_priority_queue_tests).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+
+-define(PROP_PREFIX, "prop_").
+
+-define(CAPACITY, 3).
+
+-define(RANDOM_FILE, lists:flatten(io_lib:format("~p", [erlang:timestamp()]))).
+
+setup() ->
+ Ctx = test_util:start_couch(),
+ Ctx.
+
+teardown(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+smoosh_priority_queue_test_() ->
+ {
+ "smoosh priority queue test",
+ {
+ setup,
+ fun setup/0,
+ fun teardown/1,
+ [
+ fun prop_inverse_test_/0,
+ fun no_halt_on_corrupted_file_test/0,
+ fun no_halt_on_missing_file_test/0
+ ]
+ }
+ }.
+
+%% ==========
+%% Tests
+%% ----------
+
+%% define all tests to be able to run them individually
+prop_inverse_test_() ->
+ ?_test(begin
+ test_property(prop_inverse)
+ end).
+
+no_halt_on_corrupted_file_test() ->
+ ?_test(begin
+ Name = ?RANDOM_FILE,
+ Q = smoosh_priority_queue:new(Name),
+ FilePath = smoosh_priority_queue:file_name(Q),
+ ok = file:write_file(FilePath, <<"garbage">>),
+ ok
+ end).
+
+no_halt_on_missing_file_test() ->
+ ?_test(begin
+ Name = ?RANDOM_FILE,
+ Q = smoosh_priority_queue:new(Name),
+ FilePath = smoosh_priority_queue:file_name(Q),
+ ok = file:delete(FilePath),
Review comment:
We delete the file, but we don't do anything after it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
