davisp commented on a change in pull request #3127: URL: https://github.com/apache/couchdb/pull/3127#discussion_r486637664
########## File path: src/couch_views/src/couch_views_batch_impl.erl ########## @@ -0,0 +1,147 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_views_batch_impl). + +-behavior(couch_views_batch). + + +-export([ + start/2, + success/4, + failure/2 +]). + + +-include("couch_mrview/include/couch_mrview.hrl"). + + +-record(batch_st, { + start_time, + size, + state = search, + search_incr, + sense_incr, + max_tx_size, + max_tx_time_msec, + threshold_penalty +}). + + +-spec start( + Mrst::#mrst{}, + State::term() + ) -> {NewState::term(), BatchSize::pos_integer()}. +start(Mrst, undefined) -> + St = #batch_st{ + size = get_config("batch_initial_size", "100"), + search_incr = get_config("batch_search_increment", "500"), + sense_incr = get_config("batch_sense_increment", "100"), + max_tx_size = get_config("batch_max_tx_size", "9000000"), + max_tx_time_msec = get_config("batch_max_tx_time_msec", "4500"), + threshold_penalty = get_config("batch_threshold_penalty", "0.2") Review comment: Done. ########## File path: src/couch_views/src/couch_views_batch_impl.erl ########## @@ -0,0 +1,147 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_views_batch_impl). + +-behavior(couch_views_batch). + + +-export([ + start/2, + success/4, + failure/2 +]). + + +-include("couch_mrview/include/couch_mrview.hrl"). + + +-record(batch_st, { + start_time, + size, + state = search, + search_incr, + sense_incr, + max_tx_size, + max_tx_time_msec, + threshold_penalty +}). + + +-spec start( + Mrst::#mrst{}, + State::term() + ) -> {NewState::term(), BatchSize::pos_integer()}. +start(Mrst, undefined) -> + St = #batch_st{ + size = get_config("batch_initial_size", "100"), + search_incr = get_config("batch_search_increment", "500"), + sense_incr = get_config("batch_sense_increment", "100"), + max_tx_size = get_config("batch_max_tx_size", "9000000"), + max_tx_time_msec = get_config("batch_max_tx_time_msec", "4500"), + threshold_penalty = get_config("batch_threshold_penalty", "0.2") + }, + start(Mrst, validate_opts(St)); + +start(_Mrst, #batch_st{size = Size} = St) -> + NewSt = St#batch_st{ + start_time = erlang:monotonic_time() + }, + {NewSt, Size}. + + +-spec success( + Mrst::#mrst{}, + TxSize::non_neg_integer(), + DocsRead::non_neg_integer(), + State::term() + ) -> NewState::term(). +success(_Mrst, TxSize, _DocsRead, #batch_st{} = St) -> + #batch_st{ + start_time = StartTime, + size = Size, + state = State, + search_incr = SearchIncr, + sense_incr = SenseIncr, + max_tx_size = MaxTxSize, + max_tx_time_msec = MaxTxTime, + threshold_penalty = ThresholdPenalty + } = St, + + TxTimeNative = erlang:monotonic_time() - StartTime, + TxTime = erlang:convert_time_unit(TxTimeNative, native, millisecond), + + {NewSize, NewState} = case TxSize > MaxTxSize orelse TxTime > MaxTxTime of + true -> + {round(Size * (1.0 - ThresholdPenalty)), sense}; + false when State == search -> + {Size + SearchIncr, State}; + false when State == sense -> + {Size + SenseIncr, State} + end, + + St#batch_st{ + size = erlang:max(1, NewSize), + state = NewState + }. + + +-spec failure(Mrst::#mrst{}, State::term()) -> NewState::term(). +failure(_Mrst, #batch_st{} = St) -> + St#batch_st{ + size = erlang:max(1, St#batch_st.size div 2), + state = sense + }. + + +validate_opts(St) -> + #batch_st{ + size = Size, + search_incr = SearchIncr, + sense_incr = SenseIncr, + max_tx_size = MaxTxSize, + max_tx_time_msec = MaxTxTime, + threshold_penalty = Penalty + } = St, + St#batch_st{ + size = non_neg_integer(Size, batch_initial_size), + search_incr = non_neg_integer(SearchIncr, batch_search_increment), + sense_incr = non_neg_integer(SenseIncr, batch_sense_increment), + max_tx_size = non_neg_integer(MaxTxSize, batch_max_tx_size), + max_tx_time_msec = non_neg_integer(MaxTxTime, batch_max_tx_time_msec), + threshold_penalty = float_0_to_1(Penalty, batch_threshold_penalty) Review comment: Done. ########## File path: src/couch_views/README.md ########## @@ -7,42 +7,28 @@ Currently only map indexes are supported and it will always return the full inde Code layout: * `couch_views` - Main entry point to query a view -* `couch_views_reader` - Reads from the index for queries +* `couch_views_batch` - Dynamically determine optimal batch sizes for view indexers. +* `couch_views_encoding` - Encodes view keys that are byte comparable following CouchDB view sort order. +* `couch_views_fdb` - Maps view operations to FoundationDB logic. +* `couch_views_http` - View specific helpers for chttpd * `couch_views_indexer` - `couch_jobs` worker that builds an index from the changes feed. +* `couch_views_reader` - Reads from the index for queries * `couch_vews_jobs` - `couch_views` interactions with `couch_jobs`. It handles adding index jobs and subscribes to jobs. -* `couch_views_fdb` - Maps view operations to FoundationDB logic. -* `couch_views_encoding` - Encodes view keys that are byte comparable following CouchDB view sort order. * `couch_views_server` - Spawns `couch_views_indexer` workers to handle index update jobs. +* `couch_views_updater` - Update interactive indexes during doc update transactions +* `couch_views_util` - Various utility functions # Configuration -## Configuring rate limiter - -Here is the example of configuration used in `couch_view` application: - -``` -[couch_rate.views] -limiter = couch_rate_limiter -opts = #{budget => 100, target => 2500, window => 60000, sensitivity => 1000} -``` - -Supported fields in `opts`: - -* `budget` - the initial value for estimated batch size -* `target` - the amount in msec which we try to maintain for batch processing time -* `window` - time interval for contention detector -* `sensitivity` - minimal interval within the `window` - -Unsupported fields in `opts` (if you really know what you are doing): - -* `window_size` - how many batches to consider in contention detector -* `timer` - this is used for testing to fast forward time `fun() -> current_time_in_ms() end` -* `target` - the amount in msec which we try to maintain for batch processing time -* `underload_threshold` - a threshold below which we would try to increase the budget -* `overload_threshold` - a threshold above which we would start decreasing the budget -* `delay_threshold` - a threshold above which we would start introducing delays between batches -* `multiplicative_factor` - determines how fast we are going to decrease budget (must be in (0..1) range) -* `regular_delay` - delay between batches when there is no overload -* `congested_delay` - delay between batches when there is an overload -* `initial_budget` - initial value for budget to start with - +; Batch size sensing parameters +; batch_initial_size = 100 ; Initial batch size in number of documents +; batch_search_increment = 500 ; Size change when searching for the threshold +; batch_sense_increment = 100 ; Size change increment after hitting a threshold +; batch_max_tx_size = 9000000 ; Maximum transaction size in bytes +; batch_max_tx_time = 4500 ; Maximum transaction time in milliseconds Review comment: Done. ########## File path: src/couch_views/test/couch_views_batch_test.erl ########## @@ -0,0 +1,83 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_views_batch_test). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("fabric/test/fabric2_test.hrl"). +-include_lib("couch_mrview/include/couch_mrview.hrl"). + + +batch_test_() -> + { + "Test view batch sizing", + { + setup, + fun setup/0, + fun cleanup/1, + with([ + ?TDEF(basic), + ?TDEF(search_success), + ?TDEF(sense_success), + ?TDEF(failure), + ?TDEF(failure_switches_to_sense) + ]) + } + }. + + +setup() -> + test_util:start_couch(). + + +cleanup(Ctx) -> + test_util:stop_couch(Ctx). + + +basic(_) -> + erase(couch_views_batch), + ?assertEqual(100, couch_views_batch:start(#mrst{})). + + +search_success(_) -> + erase(couch_views_batch), + couch_views_batch:start(#mrst{}), + couch_views_batch:success(#mrst{}, 0, 0), + ?assertEqual(600, couch_views_batch:start(#mrst{})). + + +sense_success(_) -> + erase(couch_views_batch), + couch_views_batch:start(#mrst{}), + % Exceeding our threshold switches from search to sense + couch_views_batch:success(#mrst{}, 10000000, 5000), + ?assertEqual(80, couch_views_batch:start(#mrst{})), + couch_views_batch:success(#mrst{}, 0, 0), + ?assertEqual(180, couch_views_batch:start(#mrst{})). + + +failure(_) -> + erase(couch_views_batch), + couch_views_batch:start(#mrst{}), + couch_views_batch:failure(#mrst{}), + ?assertEqual(50, couch_views_batch:start(#mrst{})). + + +failure_switches_to_sense(_) -> + erase(couch_views_batch), + couch_views_batch:start(#mrst{}), + couch_views_batch:failure(#mrst{}), + couch_views_batch:start(#mrst{}), + couch_views_batch:success(#mrst{}, 0, 0), + ?assertEqual(150, couch_views_batch:start(#mrst{})). + Review comment: Done. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
