svn commit: r1003475 - /couchdb/trunk/src/couchdb/couch_util.erl
Author: fdmanana
Date: Fri Oct 1 10:55:08 2010
New Revision: 1003475
URL: http://svn.apache.org/viewvc?rev=1003475&view=rev
Log:
Remove no longer necessary include macro (forgotten in revision 963038).
Modified:
couchdb/trunk/src/couchdb/couch_util.erl
Modified: couchdb/trunk/src/couchdb/couch_util.erl
URL:
http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_util.erl?rev=1003475&r1=1003474&r2=1003475&view=diff
==
--- couchdb/trunk/src/couchdb/couch_util.erl (original)
+++ couchdb/trunk/src/couchdb/couch_util.erl Fri Oct 1 10:55:08 2010
@@ -31,7 +31,6 @@
-export([encode_doc_id/1]).
-include("couch_db.hrl").
--include_lib("kernel/include/file.hrl").
% arbitrarily chosen amount of memory to use before flushing to disk
-define(FLUSH_MAX_MEM, 1000).
svn commit: r1003509 - in /couchdb/branches/new_replicator/src/couchdb: Makefile.am couch_db.hrl couch_httpd_rep.erl couch_replicator.erl couch_replicator_doc_copiers.erl couch_replicator_rev_finders.
Author: fdmanana
Date: Fri Oct 1 12:25:43 2010
New Revision: 1003509
URL: http://svn.apache.org/viewvc?rev=1003509&view=rev
Log:
New replicator: moving specific code to new files, to make the code shorter and
easier to read/modify.
Added:
couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl
Modified:
couchdb/branches/new_replicator/src/couchdb/Makefile.am
couchdb/branches/new_replicator/src/couchdb/couch_db.hrl
couchdb/branches/new_replicator/src/couchdb/couch_httpd_rep.erl
Modified: couchdb/branches/new_replicator/src/couchdb/Makefile.am
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/Makefile.am?rev=1003509&r1=1003508&r2=1003509&view=diff
==
--- couchdb/branches/new_replicator/src/couchdb/Makefile.am (original)
+++ couchdb/branches/new_replicator/src/couchdb/Makefile.am Fri Oct 1 12:25:43
2010
@@ -79,7 +79,9 @@ source_files = \
couch_view_group.erl \
couch_db_updater.erl \
couch_work_queue.erl \
-couch_replicate.erl \
+couch_replicator.erl \
+couch_replicator_rev_finders.erl \
+couch_replicator_doc_copiers.erl \
couch_replicator_utils.erl \
couch_replication_notifier.erl \
couch_httpd_rep.erl \
@@ -144,7 +146,9 @@ compiled_files = \
couch_view_group.beam \
couch_db_updater.beam \
couch_work_queue.beam \
-couch_replicate.beam \
+couch_replicator.beam \
+couch_replicator_rev_finders.beam \
+couch_replicator_doc_copiers.beam \
couch_replicator_utils.beam \
couch_replication_notifier.beam \
couch_httpd_rep.beam \
Modified: couchdb/branches/new_replicator/src/couchdb/couch_db.hrl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_db.hrl?rev=1003509&r1=1003508&r2=1003509&view=diff
==
--- couchdb/branches/new_replicator/src/couchdb/couch_db.hrl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_db.hrl Fri Oct 1
12:25:43 2010
@@ -288,3 +288,10 @@
include_docs = false
}).
+-record(rep_stats, {
+missing_checked = 0,
+missing_found = 0,
+docs_read = 0,
+docs_written = 0,
+doc_write_failures = 0
+}).
Modified: couchdb/branches/new_replicator/src/couchdb/couch_httpd_rep.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_httpd_rep.erl?rev=1003509&r1=1003508&r2=1003509&view=diff
==
--- couchdb/branches/new_replicator/src/couchdb/couch_httpd_rep.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_httpd_rep.erl Fri Oct 1
12:25:43 2010
@@ -26,7 +26,7 @@
handle_req(#httpd{method = 'POST', user_ctx = UserCtx} = Req) ->
RepDoc = couch_httpd:json_body_obj(Req),
{ok, Rep} = couch_replicator_utils:parse_rep_doc(RepDoc, UserCtx),
-case couch_replicate:replicate(Rep) of
+case couch_replicator:replicate(Rep) of
{error, Reason} ->
try
send_json(Req, 500, {[{error, Reason}]})
Added: couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl?rev=1003509&view=auto
==
--- couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl (added)
+++ couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl Fri Oct 1
12:25:43 2010
@@ -0,0 +1,712 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator).
+-behaviour(gen_server).
+
+% public API
+-export([replicate/1]).
+
+% gen_server callbacks
+-export([init/1, terminate/2, code_change/3]).
+-export([handle_call/3, handle_cast/2, handle_info/2]).
+
+-include("couch_db.hrl").
+-include("couch_api_wrap.hrl").
+
+-import(couch_util, [
+get_value/2,
+get_value/3
+]).
+
+% Can't be greater than the maximum number of child restarts specified
+% in couch_rep_sup.erl.
+-define(MAX_RESTARTS, 3).
+
+
+-record(rep_state, {
+rep_details,
+source_name,
+target_name,
+source,
+target,
+history,
+checkpoint_history,
+start_seq,
+current_t
svn commit: r1003510 - /couchdb/branches/new_replicator/src/couchdb/couch_replicate.erl
Author: fdmanana Date: Fri Oct 1 12:26:45 2010 New Revision: 1003510 URL: http://svn.apache.org/viewvc?rev=1003510&view=rev Log: New replicator: delete old file couch_replicate.erl (forgotten in previous commit). Removed: couchdb/branches/new_replicator/src/couchdb/couch_replicate.erl
svn commit: r1003599 - in /couchdb/branches/new_replicator/src/couchdb: couch_replicator.erl couch_replicator_doc_copiers.erl couch_replicator_rev_finders.erl
Author: fdmanana
Date: Fri Oct 1 18:00:48 2010
New Revision: 1003599
URL: http://svn.apache.org/viewvc?rev=1003599&view=rev
Log:
New replicator: avoid having a replication gen_server receiving 1 message for
each processed source sequence. Now it groups them into lists with size up to
?DOC_BATCH_SIZE.
Modified:
couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl
Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl?rev=1003599&r1=1003598&r2=1003599&view=diff
==
--- couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl Fri Oct 1
18:00:48 2010
@@ -278,42 +278,8 @@ handle_info({seq_start, {Seq, NumChanges
State#rep_state.seqs_in_progress),
{noreply, State#rep_state{seqs_in_progress = SeqsInProgress2}};
-handle_info({seq_changes_done, {Seq, NumChangesDone}}, State) ->
-#rep_state{
-seqs_in_progress = SeqsInProgress,
-next_through_seqs = DoneSeqs,
-is_successor_seq = IsSuccFun
-} = State,
-% Decrement the # changes for this seq by NumChangesDone.
-TotalChanges = gb_trees:get(Seq, SeqsInProgress),
-NewState = case TotalChanges - NumChangesDone of
-0 ->
-% This seq is completely processed. Check to see if it was the
-% smallest seq in progess. If so, we've made progress that can
-% be checkpointed.
-State2 = case gb_trees:smallest(SeqsInProgress) of
-{Seq, _} ->
-{CheckpointSeq, DoneSeqs2} = next_seq_before_gap(
-Seq, DoneSeqs, IsSuccFun),
-State#rep_state{
-current_through_seq = CheckpointSeq,
-next_through_seqs = DoneSeqs2
-};
-_ ->
-DoneSeqs2 = ordsets:add_element(Seq, DoneSeqs),
-State#rep_state{next_through_seqs = DoneSeqs2}
-end,
-State2#rep_state{
-seqs_in_progress = gb_trees:delete(Seq, SeqsInProgress)
-};
-NewTotalChanges when NewTotalChanges > 0 ->
-% There are still some changes that need work done.
-% Put the new count back.
-State#rep_state{
-seqs_in_progress =
-gb_trees:update(Seq, NewTotalChanges, SeqsInProgress)
-}
-end,
+handle_info({seq_changes_done, Changes}, State) ->
+NewState = lists:foldl(fun process_seq_changes_done/2, State, Changes),
{noreply, NewState};
handle_info({add_stat, {StatPos, Val}}, #rep_state{stats = Stats} = State) ->
@@ -710,3 +676,40 @@ has_session_id(SessionId, [{Props} | Res
has_session_id(SessionId, Rest)
end.
+
+process_seq_changes_done({Seq, NumChangesDone}, State) ->
+#rep_state{
+seqs_in_progress = SeqsInProgress,
+next_through_seqs = DoneSeqs,
+is_successor_seq = IsSuccFun
+} = State,
+% Decrement the # changes for this seq by NumChangesDone.
+TotalChanges = gb_trees:get(Seq, SeqsInProgress),
+case TotalChanges - NumChangesDone of
+0 ->
+% This seq is completely processed. Check to see if it was the
+% smallest seq in progess. If so, we've made progress that can
+% be checkpointed.
+State2 = case gb_trees:smallest(SeqsInProgress) of
+{Seq, _} ->
+{CheckpointSeq, DoneSeqs2} = next_seq_before_gap(
+Seq, DoneSeqs, IsSuccFun),
+State#rep_state{
+current_through_seq = CheckpointSeq,
+next_through_seqs = DoneSeqs2
+};
+_ ->
+DoneSeqs2 = ordsets:add_element(Seq, DoneSeqs),
+State#rep_state{next_through_seqs = DoneSeqs2}
+end,
+State2#rep_state{
+seqs_in_progress = gb_trees:delete(Seq, SeqsInProgress)
+};
+NewTotalChanges when NewTotalChanges > 0 ->
+% There are still some changes that need work done.
+% Put the new count back.
+State#rep_state{
+seqs_in_progress =
+gb_trees:update(Seq, NewTotalChanges, SeqsInProgress)
+}
+end.
Modified:
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl?rev=1003599&r1=1003598&r2=1003599&view=diff
==
---
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
(original)
+++
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
Fri Oct 1 18:00:48 2010
@@ -105,6 +105,8
svn commit: r1003620 - /couchdb/trunk/src/couchdb/couch_httpd_auth.erl
Author: jchris
Date: Fri Oct 1 19:09:23 2010
New Revision: 1003620
URL: http://svn.apache.org/viewvc?rev=1003620&view=rev
Log:
downgrade log line from error to debug output
Modified:
couchdb/trunk/src/couchdb/couch_httpd_auth.erl
Modified: couchdb/trunk/src/couchdb/couch_httpd_auth.erl
URL:
http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_auth.erl?rev=1003620&r1=1003619&r2=1003620&view=diff
==
--- couchdb/trunk/src/couchdb/couch_httpd_auth.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_auth.erl Fri Oct 1 19:09:23 2010
@@ -173,7 +173,7 @@ cookie_authentication_handler(#httpd{moc
CurrentTime = make_cookie_time(),
case couch_config:get("couch_httpd_auth", "secret", nil) of
nil ->
-?LOG_ERROR("cookie auth secret is not set",[]),
+?LOG_DEBUG("cookie auth secret is not set",[]),
Req;
SecretStr ->
Secret = ?l2b(SecretStr),
svn commit: r1003701 - in /couchdb/trunk/src/couchdb: couch_view.erl couch_view_group.erl
Author: jchris
Date: Fri Oct 1 23:05:30 2010
New Revision: 1003701
URL: http://svn.apache.org/viewvc?rev=1003701&view=rev
Log:
close db handles after using them to get view groups
Modified:
couchdb/trunk/src/couchdb/couch_view.erl
couchdb/trunk/src/couchdb/couch_view_group.erl
Modified: couchdb/trunk/src/couchdb/couch_view.erl
URL:
http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view.erl?rev=1003701&r1=1003700&r2=1003701&view=diff
==
--- couchdb/trunk/src/couchdb/couch_view.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view.erl Fri Oct 1 23:05:30 2010
@@ -29,9 +29,7 @@ start_link() ->
gen_server:start_link({local, couch_view}, couch_view, [], []).
get_temp_updater(DbName, Language, DesignOptions, MapSrc, RedSrc) ->
-% make temp group
-% do we need to close this db?
-{ok, _Db, Group} =
+{ok, Group} =
couch_view_group:open_temp_group(DbName, Language, DesignOptions,
MapSrc, RedSrc),
case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
{ok, Pid} ->
@@ -41,10 +39,8 @@ get_temp_updater(DbName, Language, Desig
end.
get_group_server(DbName, GroupId) ->
-% get signature for group
case couch_view_group:open_db_group(DbName, GroupId) of
-% do we need to close this db?
-{ok, _Db, Group} ->
+{ok, Group} ->
case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
{ok, Pid} ->
Pid;
Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL:
http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=1003701&r1=1003700&r2=1003701&view=diff
==
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Fri Oct 1 23:05:30 2010
@@ -445,8 +445,8 @@ open_temp_group(DbName, Language, Design
def=MapSrc,
reduce_funs= if RedSrc==[] -> []; true -> [{<<"_temp">>, RedSrc}]
end,
options=DesignOptions},
-
-{ok, Db, set_view_sig(#group{name = <<"_temp">>,lib={[]}, db=Db,
views=[View],
+couch_db:close(Db),
+{ok, set_view_sig(#group{name = <<"_temp">>,lib={[]}, db=Db,
views=[View],
def_lang=Language, design_options=DesignOptions})};
Error ->
Error
@@ -480,7 +480,8 @@ open_db_group(DbName, GroupId) ->
{ok, Db} ->
case couch_db:open_doc(Db, GroupId) of
{ok, Doc} ->
-{ok, Db, design_doc_to_view_group(Doc)};
+couch_db:close(Db),
+{ok, design_doc_to_view_group(Doc)};
Else ->
couch_db:close(Db),
Else
svn commit: r1003706 - /couchdb/trunk/src/couchdb/couch_view_group.erl
Author: jchris
Date: Fri Oct 1 23:31:59 2010
New Revision: 1003706
URL: http://svn.apache.org/viewvc?rev=1003706&view=rev
Log:
remove vestigal db handle that will just be replaced by couch_view_updater
anyway
Modified:
couchdb/trunk/src/couchdb/couch_view_group.erl
Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL:
http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=1003706&r1=1003705&r2=1003706&view=diff
==
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Fri Oct 1 23:31:59 2010
@@ -446,7 +446,7 @@ open_temp_group(DbName, Language, Design
reduce_funs= if RedSrc==[] -> []; true -> [{<<"_temp">>, RedSrc}]
end,
options=DesignOptions},
couch_db:close(Db),
-{ok, set_view_sig(#group{name = <<"_temp">>,lib={[]}, db=Db,
views=[View],
+{ok, set_view_sig(#group{name = <<"_temp">>,lib={[]}, views=[View],
def_lang=Language, design_options=DesignOptions})};
Error ->
Error
svn commit: r1003718 - in /couchdb/branches/1.0.x/src/couchdb: couch_view.erl couch_view_group.erl
Author: jchris
Date: Sat Oct 2 00:05:59 2010
New Revision: 1003718
URL: http://svn.apache.org/viewvc?rev=1003718&view=rev
Log:
backport r1003701 (fix db file ref count in view generation
Modified:
couchdb/branches/1.0.x/src/couchdb/couch_view.erl
couchdb/branches/1.0.x/src/couchdb/couch_view_group.erl
Modified: couchdb/branches/1.0.x/src/couchdb/couch_view.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/1.0.x/src/couchdb/couch_view.erl?rev=1003718&r1=1003717&r2=1003718&view=diff
==
--- couchdb/branches/1.0.x/src/couchdb/couch_view.erl (original)
+++ couchdb/branches/1.0.x/src/couchdb/couch_view.erl Sat Oct 2 00:05:59 2010
@@ -29,9 +29,7 @@ start_link() ->
gen_server:start_link({local, couch_view}, couch_view, [], []).
get_temp_updater(DbName, Language, DesignOptions, MapSrc, RedSrc) ->
-% make temp group
-% do we need to close this db?
-{ok, _Db, Group} =
+{ok, Group} =
couch_view_group:open_temp_group(DbName, Language, DesignOptions,
MapSrc, RedSrc),
case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
{ok, Pid} ->
@@ -41,10 +39,8 @@ get_temp_updater(DbName, Language, Desig
end.
get_group_server(DbName, GroupId) ->
-% get signature for group
case couch_view_group:open_db_group(DbName, GroupId) of
-% do we need to close this db?
-{ok, _Db, Group} ->
+{ok, Group} ->
case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
{ok, Pid} ->
Pid;
Modified: couchdb/branches/1.0.x/src/couchdb/couch_view_group.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/1.0.x/src/couchdb/couch_view_group.erl?rev=1003718&r1=1003717&r2=1003718&view=diff
==
--- couchdb/branches/1.0.x/src/couchdb/couch_view_group.erl (original)
+++ couchdb/branches/1.0.x/src/couchdb/couch_view_group.erl Sat Oct 2 00:05:59
2010
@@ -445,8 +445,8 @@ open_temp_group(DbName, Language, Design
def=MapSrc,
reduce_funs= if RedSrc==[] -> []; true -> [{<<"_temp">>, RedSrc}]
end,
options=DesignOptions},
-
-{ok, Db, set_view_sig(#group{name = <<"_temp">>, db=Db, views=[View],
+couch_db:close(Db),
+{ok, set_view_sig(#group{name = <<"_temp">>, views=[View],
def_lang=Language, design_options=DesignOptions})};
Error ->
Error
@@ -463,7 +463,8 @@ open_db_group(DbName, GroupId) ->
{ok, Db} ->
case couch_db:open_doc(Db, GroupId) of
{ok, Doc} ->
-{ok, Db, design_doc_to_view_group(Doc)};
+couch_db:close(Db),
+{ok, design_doc_to_view_group(Doc)};
Else ->
couch_db:close(Db),
Else
svn commit: r1003723 - /couchdb/trunk/src/couchdb/couch_view.erl
Author: jchris
Date: Sat Oct 2 00:50:49 2010
New Revision: 1003723
URL: http://svn.apache.org/viewvc?rev=1003723&view=rev
Log:
using an infinity timeout in places where we wait on disk io
Modified:
couchdb/trunk/src/couchdb/couch_view.erl
Modified: couchdb/trunk/src/couchdb/couch_view.erl
URL:
http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view.erl?rev=1003723&r1=1003722&r2=1003723&view=diff
==
--- couchdb/trunk/src/couchdb/couch_view.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view.erl Sat Oct 2 00:50:49 2010
@@ -31,7 +31,7 @@ start_link() ->
get_temp_updater(DbName, Language, DesignOptions, MapSrc, RedSrc) ->
{ok, Group} =
couch_view_group:open_temp_group(DbName, Language, DesignOptions,
MapSrc, RedSrc),
-case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
+case gen_server:call(couch_view, {get_group_server, DbName, Group},
infinity) of
{ok, Pid} ->
Pid;
Error ->
@@ -41,7 +41,7 @@ get_temp_updater(DbName, Language, Desig
get_group_server(DbName, GroupId) ->
case couch_view_group:open_db_group(DbName, GroupId) of
{ok, Group} ->
-case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
+case gen_server:call(couch_view, {get_group_server, DbName, Group},
infinity) of
{ok, Pid} ->
Pid;
Error ->
svn commit: r1003724 - in /couchdb/branches/new_replicator/src/couchdb: couch_replicator_doc_copiers.erl couch_replicator_rev_finders.erl
Author: fdmanana
Date: Sat Oct 2 00:53:15 2010
New Revision: 1003724
URL: http://svn.apache.org/viewvc?rev=1003724&view=rev
Log:
New replicator: accumulate stats where possible to avoid flooding replication
gen_servers with tons of messages.
Modified:
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl
Modified:
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl?rev=1003724&r1=1003723&r2=1003724&view=diff
==
---
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
(original)
+++
couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
Sat Oct 2 00:53:15 2010
@@ -32,99 +32,121 @@ spawn_doc_copiers(Cp, Source, Target, Mi
lists:seq(1, CopiersCount)).
+-record(doc_acc, {
+docs = [],
+seqs = [],
+read = 0,
+written = 0,
+wfail = 0,
+cp
+}).
+
doc_copy_loop(CopierId, Cp, Source, Target, MissingRevsQueue) ->
case couch_work_queue:dequeue(MissingRevsQueue, ?DOC_BATCH_SIZE) of
closed ->
?LOG_DEBUG("Doc copier ~p got missing revs queue closed", [CopierId]),
Cp ! {done, CopierId};
+
{ok, [{doc_id, _} | _] = DocIds} ->
-{BulkList, []} = lists:foldl(
+DocAcc = lists:foldl(
fun({doc_id, Id}, Acc) ->
?LOG_DEBUG("Doc copier ~p got {doc_id, ~p}", [CopierId, Id]),
{ok, Acc2} = couch_api_wrap:open_doc_revs(
Source, Id, all, [],
-fun(R, A) -> doc_handler(R, nil, Target, Cp, A) end, Acc),
+fun(R, A) -> doc_handler(R, nil, Target, A) end, Acc),
Acc2
end,
-{[], []}, DocIds),
-bulk_write_docs(lists:reverse(BulkList), [], Target, Cp),
+#doc_acc{cp = Cp}, DocIds),
+maybe_send_stat(DocAcc#doc_acc.read, #rep_stats.docs_read, Cp),
+#doc_acc{written = W, wfail = Wf} = bulk_write_docs(DocAcc, Target),
+maybe_send_stat(W, #rep_stats.docs_written, Cp),
+maybe_send_stat(Wf, #rep_stats.doc_write_failures, Cp),
doc_copy_loop(CopierId, Cp, Source, Target, MissingRevsQueue);
+
{ok, IdRevList} ->
-{Source2, {BulkList, SeqList}} = lists:foldl(
+{Source2, DocAcc} = lists:foldl(
fun({Id, Revs, PossibleAncestors, Seq} = IdRev, {SrcDb, BulkAcc})
->
?LOG_DEBUG("Doc copier ~p got ~p", [CopierId, IdRev]),
SrcDb2 = couch_api_wrap:maybe_reopen_db(SrcDb, Seq),
{ok, BulkAcc2} = couch_api_wrap:open_doc_revs(
SrcDb2, Id, Revs, [{atts_since, PossibleAncestors}],
-fun(R, A) -> doc_handler(R, Seq, Target, Cp, A) end,
+fun(R, A) -> doc_handler(R, Seq, Target, A) end,
BulkAcc),
{SrcDb2, BulkAcc2}
end,
-{Source, {[], []}}, IdRevList),
-bulk_write_docs(
-lists:reverse(BulkList),
-lists:reverse(SeqList),
-Target,
-Cp),
+{Source, #doc_acc{cp = Cp}}, IdRevList),
+maybe_send_stat(DocAcc#doc_acc.read, #rep_stats.docs_read, Cp),
+#doc_acc{written = W, wfail = Wf} = bulk_write_docs(DocAcc, Target),
+maybe_send_stat(W, #rep_stats.docs_written, Cp),
+maybe_send_stat(Wf, #rep_stats.doc_write_failures, Cp),
doc_copy_loop(CopierId, Cp, Source2, Target, MissingRevsQueue)
end.
-doc_handler({ok, #doc{atts = []} = Doc}, Seq, _Target, Cp, Acc) ->
-Cp ! {add_stat, {#rep_stats.docs_read, 1}},
+doc_handler({ok, #doc{atts = []} = Doc}, Seq, _Target, Acc) ->
update_bulk_doc_acc(Acc, Seq, Doc);
-doc_handler({ok, Doc}, Seq, Target, Cp, Acc) ->
-Cp ! {add_stat, {#rep_stats.docs_read, 1}},
-write_doc(Doc, Seq, Target, Cp),
-Acc;
+doc_handler({ok, Doc}, Seq, Target, Acc) ->
+write_doc(Doc, Seq, Target, Acc);
-doc_handler(_, _, _, _, Acc) ->
+doc_handler(_, _, _, Acc) ->
Acc.
-update_bulk_doc_acc({DocAcc, SeqAcc}, nil, Doc) ->
-{[Doc | DocAcc], SeqAcc};
-update_bulk_doc_acc({DocAcc, [{Seq, Count} | RestSeq]}, Seq, Doc) ->
-{[Doc | DocAcc], [{Seq, Count + 1} | RestSeq]};
-update_bulk_doc_acc({DocAcc, SeqAcc}, Seq, Doc) ->
-{[Doc | DocAcc], [{Seq, 1} | SeqAcc]}.
+update_bulk_doc_acc(#doc_acc{docs = Docs, read = Read} = Acc, nil, Doc) ->
+Acc#doc_acc{
+docs = [Doc | Docs],
+read = Read + 1
+};
+
+update_bulk_doc_acc(#doc_acc{seqs = [{Seq, Count} | Rest]} = Acc, Seq, Doc) ->
+Acc#doc_acc{
+docs = [Doc | Acc#doc_acc.docs],
+seqs = [{Seq, Count + 1} | Rest],
+read = Acc#doc_acc.read + 1
+};
+
+update_bulk_doc
svn commit: r1003726 - /couchdb/branches/1.0.x/src/couchdb/couch_view.erl
Author: jchris
Date: Sat Oct 2 01:12:43 2010
New Revision: 1003726
URL: http://svn.apache.org/viewvc?rev=1003726&view=rev
Log:
using an infinity timeout in places where we wait on disk io
Modified:
couchdb/branches/1.0.x/src/couchdb/couch_view.erl
Modified: couchdb/branches/1.0.x/src/couchdb/couch_view.erl
URL:
http://svn.apache.org/viewvc/couchdb/branches/1.0.x/src/couchdb/couch_view.erl?rev=1003726&r1=1003725&r2=1003726&view=diff
==
--- couchdb/branches/1.0.x/src/couchdb/couch_view.erl (original)
+++ couchdb/branches/1.0.x/src/couchdb/couch_view.erl Sat Oct 2 01:12:43 2010
@@ -31,7 +31,7 @@ start_link() ->
get_temp_updater(DbName, Language, DesignOptions, MapSrc, RedSrc) ->
{ok, Group} =
couch_view_group:open_temp_group(DbName, Language, DesignOptions,
MapSrc, RedSrc),
-case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
+case gen_server:call(couch_view, {get_group_server, DbName, Group},
infinity) of
{ok, Pid} ->
Pid;
Error ->
@@ -41,7 +41,7 @@ get_temp_updater(DbName, Language, Desig
get_group_server(DbName, GroupId) ->
case couch_view_group:open_db_group(DbName, GroupId) of
{ok, Group} ->
-case gen_server:call(couch_view, {get_group_server, DbName, Group}) of
+case gen_server:call(couch_view, {get_group_server, DbName, Group},
infinity) of
{ok, Pid} ->
Pid;
Error ->
