src/couch_replicator/src/couch_replicator_utils.erl

% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
%
%   http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
% License for the specific language governing permissions and limitations under
% the License.

-module(couch_replicator_utils).

-export([
    replication_id/2,
    sum_stats/2,
    is_deleted/1,
    rep_error_to_binary/1,
    get_json_value/2,
    get_json_value/3,
    pp_rep_id/1,
    iso8601/1,
    filter_state/3,
    normalize_rep/1,
    ejson_state_info/1,
    get_basic_auth_creds/1,
    remove_basic_auth_creds/1,
    normalize_basic_auth/1,
    seq_encode/1
]).

-include_lib("ibrowse/include/ibrowse.hrl").
-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").

-import(couch_util, [
    get_value/2,
    get_value/3
]).

rep_error_to_binary(Error) ->
    couch_util:to_binary(error_reason(Error)).

error_reason({shutdown, Error}) ->
    error_reason(Error);
error_reason({error, {Error, Reason}}) when
    is_atom(Error), is_binary(Reason)
->
    io_lib:format("~s: ~s", [Error, Reason]);
error_reason({error, Reason}) ->
    Reason;
error_reason(Reason) ->
    Reason.

get_json_value(Key, Props) ->
    get_json_value(Key, Props, undefined).

get_json_value(Key, Props, Default) when is_atom(Key) ->
    Ref = make_ref(),
    case get_value(Key, Props, Ref) of
        Ref ->
            get_value(?l2b(atom_to_list(Key)), Props, Default);
        Else ->
            Else
    end;
get_json_value(Key, Props, Default) when is_binary(Key) ->
    Ref = make_ref(),
    case get_value(Key, Props, Ref) of
        Ref ->
            get_value(list_to_atom(?b2l(Key)), Props, Default);
        Else ->
            Else
    end.

% pretty-print replication id
-spec pp_rep_id(#rep{} | rep_id()) -> string().
pp_rep_id(#rep{id = RepId}) ->
    pp_rep_id(RepId);
pp_rep_id({Base, Extension}) ->
    Base ++ Extension.

% NV: TODO: this function is not used outside api wrap module
% consider moving it there during final cleanup
is_deleted(Change) ->
    get_json_value(<<"deleted">>, Change, false).

% NV: TODO: proxy some functions which used to be here, later remove
% these and replace calls to their respective modules
replication_id(Rep, Version) ->
    couch_replicator_ids:replication_id(Rep, Version).

sum_stats(S1, S2) ->
    couch_replicator_stats:sum_stats(S1, S2).

-spec iso8601(erlang:timestamp()) -> binary().
iso8601({_Mega, _Sec, _Micro} = Timestamp) ->
    {{Y, Mon, D}, {H, Min, S}} = calendar:now_to_universal_time(Timestamp),
    Format = "~B-~2..0B-~2..0BT~2..0B:~2..0B:~2..0BZ",
    iolist_to_binary(io_lib:format(Format, [Y, Mon, D, H, Min, S])).

%% Filter replication info ejson by state provided. If it matches return
%% the input value, if it doesn't return 'skip'. This is used from replicator
%% fabric coordinator and worker.
-spec filter_state(atom(), [atom()], {[_ | _]}) -> {[_ | _]} | skip.
filter_state(null = _State, _States, _Info) ->
    skip;
filter_state(_ = _State, [] = _States, Info) ->
    Info;
filter_state(State, States, Info) ->
    case lists:member(State, States) of
        true ->
            Info;
        false ->
            skip
    end.

remove_basic_auth_from_headers(Headers) ->
    Headers1 = mochiweb_headers:make(Headers),
    case mochiweb_headers:get_value("Authorization", Headers1) of
        undefined ->
            {{undefined, undefined}, Headers};
        Auth ->
            {Basic, Base64} = lists:splitwith(fun(X) -> X =/= $\s end, Auth),
            maybe_remove_basic_auth(string:to_lower(Basic), Base64, Headers1)
    end.

maybe_remove_basic_auth("basic", " " ++ Base64, Headers) ->
    Headers1 = mochiweb_headers:delete_any("Authorization", Headers),
    {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)};
maybe_remove_basic_auth(_, _, Headers) ->
    {{undefined, undefined}, mochiweb_headers:to_list(Headers)}.

decode_basic_creds(Base64) ->
    try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of
        [User, Pass] ->
            {User, Pass};
        _ ->
            {undefined, undefined}
    catch
        % Tolerate invalid B64 values here to avoid crashing replicator
        % Changed to `missing_padding` in OTP 26
        error:Reason when Reason =:= function_clause; Reason =:= missing_padding ->
            {undefined, undefined}
    end.

% Normalize a #rep{} record such that it doesn't contain time dependent fields
% pids (like httpc pools), and options / props are sorted. This function would
% used during comparisons.
-spec normalize_rep(#rep{} | nil) -> #rep{} | nil.
normalize_rep(nil) ->
    nil;
normalize_rep(#rep{} = Rep) ->
    #rep{
        source = couch_replicator_api_wrap:normalize_db(Rep#rep.source),
        target = couch_replicator_api_wrap:normalize_db(Rep#rep.target),
        % already sorted in make_options/1
        options = Rep#rep.options,
        type = Rep#rep.type,
        view = Rep#rep.view,
        doc_id = Rep#rep.doc_id,
        db_name = Rep#rep.db_name
    }.

-spec ejson_state_info([_] | binary() | nil) -> tuple() | null.
ejson_state_info(nil) ->
    null;
ejson_state_info(Info) when is_binary(Info) ->
    {[{<<"error">>, Info}]};
ejson_state_info([]) ->
    % Status not set yet => null for compatibility reasons
    null;
ejson_state_info([{_, _} | _] = Info) ->
    {Info};
ejson_state_info(Info) ->
    ErrMsg = couch_replicator_utils:rep_error_to_binary(Info),
    {[{<<"error">>, ErrMsg}]}.

-spec get_basic_auth_creds(#httpdb{}) ->
    {string(), string()} | {undefined, undefined}.
get_basic_auth_creds(#httpdb{auth_props = AuthProps}) ->
    case couch_util:get_value(<<"basic">>, AuthProps) of
        undefined ->
            {undefined, undefined};
        {UserPass} when is_list(UserPass) ->
            User = couch_util:get_value(<<"username">>, UserPass),
            Pass = couch_util:get_value(<<"password">>, UserPass),
            case {User, Pass} of
                _ when is_binary(User), is_binary(Pass) ->
                    {binary_to_list(User), binary_to_list(Pass)};
                _Other ->
                    {undefined, undefined}
            end;
        _Other ->
            {undefined, undefined}
    end.

-spec remove_basic_auth_creds(#httpdb{}) -> #httpdb{}.
remove_basic_auth_creds(#httpdb{auth_props = Props} = HttpDb) ->
    Props1 = lists:keydelete(<<"basic">>, 1, Props),
    HttpDb#httpdb{auth_props = Props1}.

-spec set_basic_auth_creds(string(), string(), #httpdb{}) -> #httpdb{}.
set_basic_auth_creds(undefined, undefined, #httpdb{} = HttpDb) ->
    HttpDb;
set_basic_auth_creds(User, Pass, #httpdb{} = HttpDb) when
    is_list(User), is_list(Pass)
->
    HttpDb1 = remove_basic_auth_creds(HttpDb),
    Props = HttpDb1#httpdb.auth_props,
    UserPass =
        {[
            {<<"username">>, list_to_binary(User)},
            {<<"password">>, list_to_binary(Pass)}
        ]},
    Props1 = lists:keystore(<<"basic">>, 1, Props, {<<"basic">>, UserPass}),
    HttpDb1#httpdb{auth_props = Props1}.

-spec extract_creds_from_url(string()) ->
    {ok, {string() | undefined, string() | undefined}, string()}
    | {error, term()}.
extract_creds_from_url(Url) ->
    case ibrowse_lib:parse_url(Url) of
        {error, Error} ->
            {error, Error};
        #url{username = undefined, password = undefined} ->
            {ok, {undefined, undefined}, Url};
        #url{protocol = Proto, username = User, password = Pass} ->
            % Excise user and pass parts from the url. Try to keep the host,
            % port and path as they were in the original.
            Prefix = lists:concat([Proto, "://", User, ":", Pass, "@"]),
            Suffix = lists:sublist(Url, length(Prefix) + 1, length(Url) + 1),
            NoCreds = lists:concat([Proto, "://", Suffix]),
            {ok, {User, Pass}, NoCreds}
    end.

% Normalize basic auth credentials so they are set only in the auth props
% object. If multiple basic auth credentials are provided, the resulting
% credentials are picked in the following order.
%  1) {"auth": "basic": {"username":.., "password": ...} ...}
%  2) URL userinfo part
%  3) "Authentication" : "basic $base64" headers
%
-spec normalize_basic_auth(#httpdb{}) -> #httpdb{}.
normalize_basic_auth(#httpdb{} = HttpDb) ->
    #httpdb{url = Url, headers = Headers} = HttpDb,
    {HeaderCreds, HeadersNoCreds} = remove_basic_auth_from_headers(Headers),
    {UrlCreds, UrlWithoutCreds} =
        case extract_creds_from_url(Url) of
            {ok, Creds = {_, _}, UrlNoCreds} ->
                {Creds, UrlNoCreds};
            {error, _Error} ->
                % Don't crash replicator if user provided an invalid
                % userinfo part
                {undefined, undefined}
        end,
    AuthCreds = {_, _} = get_basic_auth_creds(HttpDb),
    HttpDb1 = remove_basic_auth_creds(HttpDb#httpdb{
        url = UrlWithoutCreds,
        headers = HeadersNoCreds
    }),
    {User, Pass} =
        case {AuthCreds, UrlCreds, HeaderCreds} of
            {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P};
            {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P};
            {{_, _}, {_, _}, {U, P}} -> {U, P}
        end,
    set_basic_auth_creds(User, Pass, HttpDb1).

seq_encode(Seq) when is_binary(Seq) ->
    % Don't encode a string, we already got it encoded from the changes feed
    Seq;
seq_encode(Seq) ->
    % This could be either an integer sequence from CouchDB 1.x, a
    % [Seq, Opaque] json array from BigCouch 0.4, or any other json
    % object. We are being maximally compatible here.
    ?JSON_ENCODE(Seq).

-ifdef(TEST).

-include_lib("couch/include/couch_eunit.hrl").

remove_basic_auth_from_headers_test_() ->
    [
        ?_assertMatch(
            {{User, Pass}, NoAuthHeaders},
            remove_basic_auth_from_headers(Headers)
        )
     || {{User, Pass, NoAuthHeaders}, Headers} <- [
            {
                {undefined, undefined, []},
                []
            },
            {
                {undefined, undefined, [{"h", "v"}]},
                [{"h", "v"}]
            },
            {
                {undefined, undefined, [{"Authorization", "junk"}]},
                [{"Authorization", "junk"}]
            },
            {
                {undefined, undefined, []},
                [{"Authorization", "basic X"}]
            },
            {
                {"user", "pass", []},
                [{"Authorization", "Basic " ++ b64creds("user", "pass")}]
            },
            {
                {"user", "pass", []},
                [{"AuThorization", "Basic " ++ b64creds("user", "pass")}]
            },
            {
                {"user", "pass", []},
                [{"Authorization", "bAsIc " ++ b64creds("user", "pass")}]
            },
            {
                {"user", "pass", [{"h", "v"}]},
                [
                    {"Authorization", "Basic " ++ b64creds("user", "pass")},
                    {"h", "v"}
                ]
            }
        ]
    ].

b64creds(User, Pass) ->
    base64:encode_to_string(User ++ ":" ++ Pass).

normalize_rep_test_() ->
    {
        setup,
        fun() ->
            meck:expect(
                config,
                get,
                fun(_, _, Default) -> Default end
            )
        end,
        fun(_) -> meck:unload() end,
        ?_test(begin
            EJson1 =
                {[
                    {<<"source">>, <<"http://host.com/source_db">>},
                    {<<"target">>, <<"http://target.local/db">>},
                    {<<"doc_ids">>, [<<"a">>, <<"c">>, <<"b">>]},
                    {<<"other_field">>, <<"some_value">>}
                ]},
            Rep1 = couch_replicator_parse:parse_rep_doc_without_id(EJson1),
            EJson2 =
                {[
                    {<<"other_field">>, <<"unrelated">>},
                    {<<"target">>, <<"http://target.local/db">>},
                    {<<"source">>, <<"http://host.com/source_db">>},
                    {<<"doc_ids">>, [<<"c">>, <<"a">>, <<"b">>]},
                    {<<"other_field2">>, <<"unrelated2">>}
                ]},
            Rep2 = couch_replicator_parse:parse_rep_doc_without_id(EJson2),
            ?assertEqual(normalize_rep(Rep1), normalize_rep(Rep2))
        end)
    }.

get_basic_auth_creds_test() ->
    Check = fun(Props) ->
        get_basic_auth_creds(#httpdb{auth_props = Props})
    end,

    ?assertEqual({undefined, undefined}, Check([])),

    ?assertEqual({undefined, undefined}, Check([null])),

    ?assertEqual({undefined, undefined}, Check([{<<"other">>, <<"x">>}])),

    ?assertEqual({undefined, undefined}, Check([{<<"basic">>, []}])),

    UserPass1 = {[{<<"username">>, <<"u">>}, {<<"password">>, <<"p">>}]},
    ?assertEqual({"u", "p"}, Check([{<<"basic">>, UserPass1}])),

    UserPass3 = {[{<<"username">>, <<"u">>}, {<<"password">>, null}]},
    ?assertEqual({undefined, undefined}, Check([{<<"basic">>, UserPass3}])).

remove_basic_auth_creds_test() ->
    Check = fun(Props) ->
        HttpDb = remove_basic_auth_creds(#httpdb{auth_props = Props}),
        HttpDb#httpdb.auth_props
    end,

    ?assertEqual([], Check([])),

    ?assertEqual([{<<"other">>, {[]}}], Check([{<<"other">>, {[]}}])),

    ?assertEqual(
        [],
        Check([
            {<<"basic">>,
                {[
                    {<<"username">>, <<"u">>},
                    {<<"password">>, <<"p">>}
                ]}}
        ])
    ),

    ?assertEqual(
        [{<<"other">>, {[]}}],
        Check([
            {<<"basic">>,
                {[
                    {<<"username">>, <<"u">>},
                    {<<"password">>, <<"p">>}
                ]}},
            {<<"other">>, {[]}}
        ])
    ).

set_basic_auth_creds_test() ->
    Check = fun(User, Pass, Props) ->
        HttpDb = set_basic_auth_creds(User, Pass, #httpdb{auth_props = Props}),
        HttpDb#httpdb.auth_props
    end,

    ?assertEqual([], Check(undefined, undefined, [])),

    ?assertEqual(
        [{<<"other">>, {[]}}],
        Check(
            undefined,
            undefined,
            [{<<"other">>, {[]}}]
        )
    ),

    ?assertEqual(
        [
            {<<"basic">>,
                {[
                    {<<"username">>, <<"u">>},
                    {<<"password">>, <<"p">>}
                ]}}
        ],
        Check("u", "p", [])
    ),

    ?assertEqual(
        [
            {<<"other">>, {[]}},
            {<<"basic">>,
                {[
                    {<<"username">>, <<"u">>},
                    {<<"password">>, <<"p">>}
                ]}}
        ],
        Check("u", "p", [{<<"other">>, {[]}}])
    ).

normalize_basic_creds_test_() ->
    DefaultHeaders = (#httpdb{})#httpdb.headers,
    [
        ?_assertEqual(Expect, normalize_basic_auth(Input))
     || {Input, Expect} <- [
            {
                #httpdb{url = "http://u:p@x.y/db"},
                #httpdb{url = "http://x.y/db", auth_props = auth_props("u", "p")}
            },
            {
                #httpdb{url = "http://u:p@h:80/db"},
                #httpdb{url = "http://h:80/db", auth_props = auth_props("u", "p")}
            },
            {
                #httpdb{url = "https://u:p@h/db"},
                #httpdb{url = "https://h/db", auth_props = auth_props("u", "p")}
            },
            {
                #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"},
                #httpdb{
                    url = "http://[2001:db8:a1b:12f9::1]/db",
                    auth_props = auth_props("u", "p")
                }
            },
            {
                #httpdb{
                    url = "http://h/db",
                    headers =
                        DefaultHeaders ++
                            [
                                {"Authorization", "Basic " ++ b64creds("u", "p")}
                            ]
                },
                #httpdb{url = "http://h/db", auth_props = auth_props("u", "p")}
            },
            {
                #httpdb{
                    url = "http://h/db",
                    headers =
                        DefaultHeaders ++
                            [
                                {"Authorization", "Basic " ++ b64creds("u", "p@")}
                            ]
                },
                #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@")}
            },
            {
                #httpdb{
                    url = "http://h/db",
                    headers =
                        DefaultHeaders ++
                            [
                                {"Authorization", "Basic " ++ b64creds("u", "p@%40")}
                            ]
                },
                #httpdb{url = "http://h/db", auth_props = auth_props("u", "p@%40")}
            },
            {
                #httpdb{
                    url = "http://h/db",
                    headers =
                        DefaultHeaders ++
                            [
                                {"aUthoriZation", "bASIC " ++ b64creds("U", "p")}
                            ]
                },
                #httpdb{url = "http://h/db", auth_props = auth_props("U", "p")}
            },
            {
                #httpdb{
                    url = "http://u1:p1@h/db",
                    headers =
                        DefaultHeaders ++
                            [
                                {"Authorization", "Basic " ++ b64creds("u2", "p2")}
                            ]
                },
                #httpdb{url = "http://h/db", auth_props = auth_props("u1", "p1")}
            },
            {
                #httpdb{
                    url = "http://u1:p1@h/db",
                    auth_props = [
                        {<<"basic">>,
                            {[
                                {<<"username">>, <<"u2">>},
                                {<<"password">>, <<"p2">>}
                            ]}}
                    ]
                },
                #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")}
            },
            {
                #httpdb{
                    url = "http://u1:p1@h/db",
                    auth_props = [
                        {<<"basic">>,
                            {[
                                {<<"username">>, <<"u2">>},
                                {<<"password">>, <<"p2">>}
                            ]}}
                    ],
                    headers =
                        DefaultHeaders ++
                            [
                                {"Authorization", "Basic " ++ b64creds("u3", "p3")}
                            ]
                },
                #httpdb{url = "http://h/db", auth_props = auth_props("u2", "p2")}
            }
        ]
    ].

auth_props(User, Pass) when is_list(User), is_list(Pass) ->
    [
        {<<"basic">>,
            {[
                {<<"username">>, list_to_binary(User)},
                {<<"password">>, list_to_binary(Pass)}
            ]}}
    ].

-endif.