src/couch_replicator/src/couch_replicator_fabric.erl

% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
%
%   http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
% License for the specific language governing permissions and limitations under
% the License.

-module(couch_replicator_fabric).

-export([
   docs/5
]).

-include_lib("fabric/include/fabric.hrl").
-include_lib("mem3/include/mem3.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").

docs(DbName, Options, QueryArgs, Callback, Acc) ->
    Shards = mem3:shards(DbName),
    Workers0 = fabric_util:submit_jobs(
           Shards, couch_replicator_fabric_rpc, docs, [Options, QueryArgs]),
    RexiMon = fabric_util:create_monitors(Workers0),
    try
        case fabric_streams:start(Workers0, #shard.ref) of
            {ok, Workers} ->
                try
                    docs_int(DbName, Workers, QueryArgs, Callback, Acc)
                after
                    fabric_streams:cleanup(Workers)
                end;
            {timeout, NewState} ->
                DefunctWorkers = fabric_util:remove_done_workers(
                    NewState#stream_acc.workers, waiting
                ),
                fabric_util:log_timeout(
                    DefunctWorkers,
                    "replicator docs"
                ),
                Callback({error, timeout}, Acc);
            {error, Error} ->
                Callback({error, Error}, Acc)
        end
    after
        rexi_monitor:stop(RexiMon)
    end.


docs_int(DbName, Workers, QueryArgs, Callback, Acc0) ->
    #mrargs{limit = Limit, skip = Skip} = QueryArgs,
    State = #collector{
        db_name = DbName,
        query_args = QueryArgs,
        callback = Callback,
        counters = fabric_dict:init(Workers, 0),
        skip = Skip,
        limit = Limit,
        user_acc = Acc0,
        update_seq = nil
    },
    case rexi_utils:recv(Workers, #shard.ref, fun handle_message/3,
        State, infinity, 5000) of
    {ok, NewState} ->
        {ok, NewState#collector.user_acc};
    {timeout, NewState} ->
        Callback({error, timeout}, NewState#collector.user_acc);
    {error, Resp} ->
        {ok, Resp}
    end.

handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) ->
    fabric_view:check_down_shards(State, NodeRef);

handle_message({rexi_EXIT, Reason}, Worker, State) ->
    fabric_view:handle_worker_exit(State, Worker, Reason);

handle_message({meta, Meta0}, {Worker, From}, State) ->
    Tot = couch_util:get_value(total, Meta0, 0),
    Off = couch_util:get_value(offset, Meta0, 0),
    #collector{
        callback = Callback,
        counters = Counters0,
        total_rows = Total0,
        offset = Offset0,
        user_acc = AccIn
    } = State,
    % Assert that we don't have other messages from this
    % worker when the total_and_offset message arrives.
    0 = fabric_dict:lookup_element(Worker, Counters0),
    rexi:stream_ack(From),
    Counters1 = fabric_dict:update_counter(Worker, 1, Counters0),
    Total = Total0 + Tot,
    Offset = Offset0 + Off,
    case fabric_dict:any(0, Counters1) of
    true ->
        {ok, State#collector{
            counters = Counters1,
            total_rows = Total,
            offset = Offset
        }};
    false ->
        FinalOffset = erlang:min(Total, Offset+State#collector.skip),
        Meta = [{total, Total}, {offset, FinalOffset}],
        {Go, Acc} = Callback({meta, Meta}, AccIn),
        {Go, State#collector{
            counters = fabric_dict:decrement_all(Counters1),
            total_rows = Total,
            offset = FinalOffset,
            user_acc = Acc
        }}
    end;

handle_message(#view_row{id = Id, doc = Doc} = Row0, {Worker, From}, State) ->
    #collector{query_args = Args, counters = Counters0, rows = Rows0} = State,
    case maybe_fetch_and_filter_doc(Id, Doc, State) of
        {[_ | _]} = NewDoc ->
            Row = Row0#view_row{doc = NewDoc},
            Dir = Args#mrargs.direction,
            Rows = merge_row(Dir, Row#view_row{worker={Worker, From}}, Rows0),
            Counters1 = fabric_dict:update_counter(Worker, 1, Counters0),
            State1 = State#collector{rows=Rows, counters=Counters1},
            fabric_view:maybe_send_row(State1);
        skip ->
            rexi:stream_ack(From),
            {ok, State}
    end;

handle_message(complete, Worker, State) ->
    Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters),
    fabric_view:maybe_send_row(State#collector{counters = Counters}).


merge_row(fwd, Row, Rows) ->
    lists:keymerge(#view_row.id, [Row], Rows);
merge_row(rev, Row, Rows) ->
    lists:rkeymerge(#view_row.id, [Row], Rows).


maybe_fetch_and_filter_doc(Id, undecided, State) ->
    #collector{db_name = DbName, query_args = #mrargs{extra = Extra}} = State,
    FilterStates = proplists:get_value(filter_states, Extra),
    case couch_replicator:active_doc(DbName, Id) of
        {ok, {Props} = DocInfo} ->
            DocState = couch_util:get_value(state, Props),
            couch_replicator_utils:filter_state(DocState, FilterStates, DocInfo);
        {error, not_found} ->
            skip  % could have been deleted
    end;
maybe_fetch_and_filter_doc(_Id, Doc, _State) ->
    Doc.