Merge remote branch 'cloudant:79066-port-chunkified-replicate_batch'
This closes #26
Signed-off-by: Eric Avdey <eiri@eiri.ca>
diff --git a/src/mem3_rep.erl b/src/mem3_rep.erl
index 938260d..ad7ac55 100644
--- a/src/mem3_rep.erl
+++ b/src/mem3_rep.erl
@@ -252,8 +252,10 @@
[] ->
ok;
Missing ->
- Docs = open_docs(Acc, Missing),
- ok = save_on_target(Node, Name, Docs)
+ lists:map(fun(Chunk) ->
+ Docs = open_docs(Acc, Chunk),
+ ok = save_on_target(Node, Name, Docs)
+ end, chunk_revs(Missing))
end,
update_locals(Acc),
{ok, Acc#acc{revcount=0, infos=[]}}.
@@ -271,6 +273,32 @@
]).
+chunk_revs(Revs) ->
+ Limit = list_to_integer(config:get("mem3", "rev_chunk_size", "5000")),
+ chunk_revs(Revs, Limit).
+
+chunk_revs(Revs, Limit) ->
+ chunk_revs(Revs, {0, []}, [], Limit).
+
+chunk_revs([], {_Count, Chunk}, Chunks, _Limit) ->
+ [Chunk|Chunks];
+chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks, Limit) when length(R) =< Limit - Count ->
+ chunk_revs(
+ Revs,
+ {Count + length(R), [{Id, R, A}|Chunk]},
+ Chunks,
+ Limit
+ );
+chunk_revs([{Id, R, A}|Revs], {Count, Chunk}, Chunks, Limit) ->
+ {This, Next} = lists:split(Limit - Count, R),
+ chunk_revs(
+ [{Id, Next, A}|Revs],
+ {0, []},
+ [[{Id, This, A}|Chunk]|Chunks],
+ Limit
+ ).
+
+
open_docs(#acc{source=Source, infos=Infos}, Missing) ->
lists:flatmap(fun({Id, Revs, _}) ->
FDI = lists:keyfind(Id, #full_doc_info.id, Infos),
