Merged upstream changes
diff --git a/BSD_LICENSE b/BSD_LICENSE
index 88180d6..318a96e 100644
--- a/BSD_LICENSE
+++ b/BSD_LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2005-2013, Chandrashekhar Mullaparthi
+Copyright (c) 2005-2014, Chandrashekhar Mullaparthi
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
diff --git a/CHANGELOG b/CHANGELOG
index 17749c7..643c928 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,5 +1,12 @@
CONTRIBUTIONS & CHANGE HISTORY
==============================
+18-04-2013 - v4.1.0
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/101
+ * Support for https://github.com/cmullaparthi/ibrowse/issues/90
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/86
+ * Merged various contributions. Please see commit history for details
+ * Introduced the return_raw_request option
+
09-04-2013 - v4.0.2
* Tagging master with new version to cover changes
contributed over the past few months via various pull requests
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index f1bb4c8..e33b41a 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -42,11 +42,16 @@
hyperthunk (https://github.com/hyperthunk/)
Mistagrooves (https://github.com/Mistagrooves/)
tholschuh (https://github.com/tholschuh/)
-https://github.com/nrdufour
-https://github.com/helllamer
-https://github.com/rflynn
-https://github.com/puzza007
-https://github.com/marutha
+https://github.com/apauley
+https://github.com/AeroNotix
https://github.com/dis
+https://github.com/f355
+https://github.com/flycodepl
+https://github.com/helllamer
+https://github.com/marutha
+https://github.com/nrdufour
https://github.com/pib
+https://github.com/puzza007
+https://github.com/rflynn
+https://github.com/Vagabond
diff --git a/LICENSE b/LICENSE
index e260a55..a405991 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,5 +1,5 @@
ibrowse - a HTTP client written in erlang
-Copyright (C) 2005-2013 Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
+Copyright (C) 2005-2014 Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
diff --git a/Makefile b/Makefile
index ad21840..b596b64 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,9 @@
IBROWSE_VSN = $(shell sed -n 's/.*{vsn,.*"\(.*\)"}.*/\1/p' src/ibrowse.app.src)
DIALYZER_PLT=$(CURDIR)/.dialyzer_plt
-DIALYZER_APPS=erts kernel stdlib ssl crypto public_key asn1 compiler hipe edoc gs syntax_tools inets xmerl runtime_tools mnesia
+DIALYZER_APPS=erts kernel stdlib ssl crypto public_key
-all: compile dialyzer
+all: compile
compile:
./rebar compile
diff --git a/README.md b/README.md
index 89ca67c..1618083 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
**Comments to:** chandrashekhar.mullaparthi@gmail.com
-**Current Version:** 4.0.2
+**Current Version:** 4.1.0
**Latest Version:** git://github.com/cmullaparthi/ibrowse.git
diff --git a/src/ibrowse.erl b/src/ibrowse.erl
index 42030af..fc90dc6 100644
--- a/src/ibrowse.erl
+++ b/src/ibrowse.erl
@@ -6,7 +6,7 @@
%%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
%%%-------------------------------------------------------------------
%% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
-%% @copyright 2005-2012 Chandrashekhar Mullaparthi
+%% @copyright 2005-2014 Chandrashekhar Mullaparthi
%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This
%% module implements the API of the HTTP client. There is one named
%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
@@ -158,7 +158,7 @@
%% respHeader() = {headerName(), headerValue()}
%% headerName() = string()
%% headerValue() = string()
-%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, req_id() } | {error, Reason}
+%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, req_id() } | {error, Reason}
%% req_id() = term()
%% ResponseBody = string() | {file, Filename}
%% Reason = term()
@@ -256,6 +256,8 @@
%% to receive the raw data stream when the Transfer-Encoding of the server
%% response is Chunked.
%% </li>
+%% <li> The <code>return_raw_request</code> option enables the caller to get the exact request which was sent by ibrowse to the server, along with the response. When this option is used, the response for synchronous requests is a 5-tuple instead of the usual 4-tuple. For asynchronous requests, the calling process gets a message <code>{ibrowse_async_raw_req, Raw_req}</code>.
+%% </li>
%% </ul>
%%
%% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response()
@@ -289,7 +291,8 @@
%% {give_raw_headers, boolean()} |
%% {preserve_chunked_encoding,boolean()} |
%% {workaround, head_response_with_body} |
-%% {worker_process_options, list()}
+%% {worker_process_options, list()} |
+%% {return_raw_request, true}
%%
%% stream_to() = process() | {process(), once}
%% process() = pid() | atom()
@@ -442,7 +445,7 @@
{'EXIT', {noproc, {gen_server, call, [Conn_Pid, _, _]}}} ->
{error, sel_conn_closed};
{'EXIT', {normal, _}} ->
- {error, req_timedout};
+ {error, sel_conn_closed};
{'EXIT', {connection_closed, _}} ->
{error, sel_conn_closed};
{error, connection_closed} ->
@@ -456,6 +459,13 @@
binary ->
Ret
end;
+ {ok, St_code, Headers, Body, Req} = Ret when is_binary(Body) ->
+ case get_value(response_format, Options, list) of
+ list ->
+ {ok, St_code, Headers, binary_to_list(Body), Req};
+ binary ->
+ Ret
+ end;
Ret ->
Ret
end.
diff --git a/src/ibrowse_http_client.erl b/src/ibrowse_http_client.erl
index 0410c08..64fc443 100644
--- a/src/ibrowse_http_client.erl
+++ b/src/ibrowse_http_client.erl
@@ -63,7 +63,7 @@
stream_chunk_size,
save_response_to_file = false,
tmp_file_name, tmp_file_fd, preserve_chunked_encoding,
- response_format, timer_ref}).
+ response_format, timer_ref, raw_req}).
-import(ibrowse_lib, [
get_value/2,
@@ -215,11 +215,11 @@
handle_info({tcp_closed, _Sock}, State) ->
do_trace("TCP connection closed by peer!~n", []),
handle_sock_closed(State),
- {stop, connection_closed, State};
+ {stop, normal, State};
handle_info({ssl_closed, _Sock}, State) ->
do_trace("SSL connection closed by peer!~n", []),
handle_sock_closed(State),
- {stop, connection_closed, State};
+ {stop, normal, State};
handle_info({tcp_error, _Sock, Reason}, State) ->
do_trace("Error on connection to ~1000.p:~1000.p -> ~1000.p~n",
@@ -239,6 +239,7 @@
{value, #request{stream_to = StreamTo, req_id = ReqId}} ->
catch StreamTo ! {ibrowse_async_response_timeout, ReqId},
shutting_down(State),
+ do_error_reply(State, req_timedout),
{stop, normal, State}
end;
@@ -471,7 +472,9 @@
}=State) ->
#request{from=From, stream_to=StreamTo, req_id=ReqId,
response_format = Resp_format,
- options = Options} = CurReq,
+ options = Options,
+ raw_req = Raw_req
+ } = CurReq,
case IsClosing of
true ->
{_, Reqs_1} = queue:out(Reqs),
@@ -482,11 +485,16 @@
ok = file:close(Fd),
{file, TmpFilename}
end,
+ Give_raw_req = get_value(return_raw_request, Options, false),
Reply = case get_value(give_raw_headers, Options, false) of
- true ->
+ true when Give_raw_req == false->
{ok, Status_line, Raw_headers, Body};
+ true ->
+ {ok, Status_line, Raw_headers, Body, Raw_req};
+ false when Give_raw_req == false ->
+ {ok, SC, Headers, Buf};
false ->
- {ok, SC, Headers, Buf}
+ {ok, SC, Headers, Buf, Raw_req}
end,
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
ok = do_error_reply(State_1#state{reqs = Reqs_1}, connection_closed),
@@ -560,37 +568,63 @@
do_send(Req, #state{socket = Sock, is_ssl = true}) -> ssl:send(Sock, Req);
do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req).
-%% @spec do_send_body(Sock::socket_descriptor(), Source::source_descriptor(), IsSSL::boolean()) -> ok | error()
-%% source_descriptor() = fun_arity_0 |
-%% {fun_arity_0} |
-%% {fun_arity_1, term()}
-%% error() = term()
do_send_body(Source, State, TE) when is_function(Source) ->
do_send_body({Source}, State, TE);
do_send_body({Source}, State, TE) when is_function(Source) ->
- do_send_body1(Source, Source(), State, TE);
+ do_send_body_1(generate_body(Source),
+ State, TE, []);
do_send_body({Source, Source_state}, State, TE) when is_function(Source) ->
- do_send_body1(Source, Source(Source_state), State, TE);
+ do_send_body_1(generate_body({Source, Source_state}),
+ State, TE, []);
do_send_body(Body, State, _TE) ->
- do_send(Body, State).
+ case do_send(Body, State) of
+ ok ->
+ {ok, Body};
+ Ret ->
+ Ret
+ end.
-do_send_body1(Source, Resp, State, TE) ->
+generate_body({Source, Source_state} = In) when is_function(Source) ->
+ case Source(Source_state) of
+ {ok, Data, Source_state_1} ->
+ {{ok, Data}, {Source, Source_state_1}};
+ Ret ->
+ {Ret, In}
+ end;
+generate_body(Source) when is_function(Source) ->
+ {Source(), Source}.
+
+do_send_body_1({Resp, Source}, State, TE, Acc) when is_function(Source) ->
case Resp of
- {ok, Data} when Data == []; Data == <<>> ->
- do_send_body({Source}, State, TE);
+ {ok, Data} when Data == []; Data == <<>> ->
+ do_send_body_1(generate_body(Source), State, TE, Acc);
{ok, Data} ->
- _ = do_send(maybe_chunked_encode(Data, TE), State),
- do_send_body({Source}, State, TE);
- {ok, Data, New_source_state} when Data == []; Data == <<>> ->
- do_send_body({Source, New_source_state}, State, TE);
+ Acc_1 = case TE of
+ true ->
+ ok = do_send(maybe_chunked_encode(Data, TE), State),
+ Acc;
+ false ->
+ [Data | Acc]
+ end,
+ do_send_body_1(generate_body(Source), State, TE, Acc_1);
+ {ok, Data, New_source_state} when Data == []; Data == <<>> ->
+ do_send_body_1(generate_body({Source, New_source_state}), State, TE, Acc);
{ok, Data, New_source_state} ->
- _ = do_send(maybe_chunked_encode(Data, TE), State),
- do_send_body({Source, New_source_state}, State, TE);
+ Acc_1 = case TE of
+ true ->
+ ok = do_send(maybe_chunked_encode(Data, TE), State),
+ Acc;
+ false ->
+ [Data | Acc]
+ end,
+ do_send_body_1(generate_body({Source, New_source_state}), State, TE, Acc_1);
eof when TE == true ->
- _ = do_send(<<"0\r\n\r\n">>, State),
- ok;
+ ok = do_send(<<"0\r\n\r\n">>, State),
+ {ok, []};
eof ->
- ok;
+ Body = list_to_binary(lists:reverse(Acc)),
+ ok = do_send(Body, State),
+ {ok, Body};
Err ->
Err
end.
@@ -701,7 +735,7 @@
case do_send(Req, State) of
ok ->
case do_send_body(Body_1, State_1, TE) of
- ok ->
+ {ok, _Sent_body} ->
trace_request_body(Body_1),
_ = active_once(State_1),
State_1_1 = inc_pipeline_counter(State_1),
@@ -765,6 +799,11 @@
_ ->
erlang:send_after(Timeout, self(), {req_timedout, From})
end,
+ Headers_1 = maybe_modify_headers(Url, Method, Options, Headers, State),
+ {Req, Body_1} = make_request(Method,
+ Headers_1,
+ AbsPath, RelPath, Body, Options, State,
+ ReqId),
NewReq = #request{url = Url,
method = Method,
stream_to = StreamTo,
@@ -779,26 +818,24 @@
preserve_chunked_encoding = get_value(preserve_chunked_encoding, Options, false),
timer_ref = Ref
},
- State_1 = State#state{reqs=queue:in(NewReq, State#state.reqs)},
- Headers_1 = maybe_modify_headers(Url, Method, Options, Headers, State_1),
- {Req, Body_1} = make_request(Method,
- Headers_1,
- AbsPath, RelPath, Body, Options, State_1,
- ReqId),
trace_request(Req),
- _ = do_setopts(Socket, Caller_socket_options, State_1),
+ ok = do_setopts(Socket, Caller_socket_options, State),
TE = is_chunked_encoding_specified(Options),
- case do_send(Req, State_1) of
+ case do_send(Req, State) of
ok ->
- case do_send_body(Body_1, State_1, TE) of
- ok ->
- trace_request_body(Body_1),
+ case do_send_body(Body_1, State, TE) of
+ {ok, Sent_body} ->
+ trace_request_body(Sent_body),
+ Raw_req = list_to_binary([Req, Sent_body]),
+ NewReq_1 = NewReq#request{raw_req = Raw_req},
+ State_1 = State#state{reqs=queue:in(NewReq_1, State#state.reqs)},
State_2 = inc_pipeline_counter(State_1),
_ = active_once(State_2),
State_3 = case Status of
idle ->
- State_2#state{status = get_header,
- cur_req = NewReq};
+ State_2#state{
+ status = get_header,
+ cur_req = NewReq_1};
_ ->
State_2
end,
@@ -806,21 +843,27 @@
undefined ->
ok;
_ ->
- gen_server:reply(From, {ibrowse_req_id, ReqId})
+ gen_server:reply(From, {ibrowse_req_id, ReqId}),
+ case get_value(return_raw_request, Options, false) of
+ false ->
+ ok;
+ true ->
+ catch StreamTo ! {ibrowse_async_raw_req, Raw_req}
+ end
end,
State_4 = set_inac_timer(State_3),
{noreply, State_4};
Err ->
- shutting_down(State_1),
+ shutting_down(State),
do_trace("Send failed... Reason: ~p~n", [Err]),
gen_server:reply(From, {error, {send_failed, Err}}),
- {stop, normal, State_1}
+ {stop, normal, State}
end;
Err ->
- shutting_down(State_1),
+ shutting_down(State),
do_trace("Send failed... Reason: ~p~n", [Err]),
gen_server:reply(From, {error, {send_failed, Err}}),
- {stop, normal, State_1}
+ {stop, normal, State}
end.
maybe_modify_headers(#url{}, connect, _, Headers, State) ->
@@ -1018,7 +1061,8 @@
cur_req = CurReq} = State) ->
#request{from=From, stream_to=StreamTo, req_id=ReqId,
method=Method, response_format = Resp_format,
- options = Options, timer_ref = T_ref
+ options = Options, timer_ref = T_ref,
+ raw_req = Raw_req
} = CurReq,
MaxHeaderSize = ibrowse:get_config_value(max_headers_size, infinity),
case scan_header(Acc, Data) of
@@ -1038,6 +1082,7 @@
State
end,
Give_raw_headers = get_value(give_raw_headers, Options, false),
+ Give_raw_req = get_value(return_raw_request, Options, false),
State_1 = case Give_raw_headers of
true ->
State_0#state{recvd_headers=Headers_1, status=get_body,
@@ -1078,8 +1123,13 @@
%% there was still a body. Issue #67 on Github
{_, Reqs_1} = queue:out(Reqs),
send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
- State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format,
- {ok, StatCode, Headers_1, []}),
+ Reply = case Give_raw_req of
+ false ->
+ {ok, StatCode, Headers_1, []};
+ true ->
+ {ok, StatCode, Headers_1, [], Raw_req}
+ end,
+ State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(T_ref, {eat_message, {req_timedout, From}}),
State_2 = reset_state(State_1_1),
State_3 = set_cur_request(State_2#state{reqs = Reqs_1}),
@@ -1098,8 +1148,13 @@
%% RFC2616 - Sec 4.4
{_, Reqs_1} = queue:out(Reqs),
send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
- State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format,
- {ok, StatCode, Headers_1, []}),
+ Reply = case Give_raw_req of
+ true ->
+ {ok, StatCode, Headers_1, []};
+ false ->
+ {ok, StatCode, Headers_1, [], Raw_req}
+ end,
+ State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(T_ref, {eat_message, {req_timedout, From}}),
State_2 = reset_state(State_1_1),
State_3 = set_cur_request(State_2#state{reqs = Reqs_1}),
@@ -1346,7 +1401,8 @@
tmp_file_name = TmpFilename,
tmp_file_fd = Fd,
options = Options,
- timer_ref = ReqTimer
+ timer_ref = ReqTimer,
+ raw_req = Raw_req
},
#state{http_status_code = SCode,
status_line = Status_line,
@@ -1367,18 +1423,25 @@
{file, TmpFilename}
end,
{Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(RespHeaders, Raw_headers, Options),
+ Give_raw_req = get_value(return_raw_request, Options, false),
Reply = case get_value(give_raw_headers, Options, false) of
- true ->
+ true when Give_raw_req == false ->
{ok, Status_line, Raw_headers_1, ResponseBody};
+ true ->
+ {ok, Status_line, Raw_headers_1, ResponseBody, Raw_req};
+ false when Give_raw_req == false ->
+ {ok, SCode, Resp_headers_1, ResponseBody};
false ->
- {ok, SCode, Resp_headers_1, ResponseBody}
+ {ok, SCode, Resp_headers_1, ResponseBody, Raw_req}
end,
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
set_cur_request(State_1);
handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
response_format = Resp_format,
- options = Options, timer_ref = ReqTimer},
+ options = Options, timer_ref = ReqTimer,
+ raw_req = Raw_req
+ },
#state{http_status_code = SCode,
status_line = Status_line,
raw_headers = Raw_headers,
@@ -1387,11 +1450,16 @@
} = State) ->
Body = RepBuf,
{Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Resp_headers, Raw_headers, Options),
+ Give_raw_req = get_value(return_raw_request, Options, false),
Reply = case get_value(give_raw_headers, Options, false) of
- true ->
+ true when Give_raw_req == false ->
{ok, Status_line, Raw_headers_1, Body};
+ true ->
+ {ok, Status_line, Raw_headers_1, Body, Raw_req};
+ false when Give_raw_req == false ->
+ {ok, SCode, Resp_headers_1, Body};
false ->
- {ok, SCode, Resp_headers_1, Body}
+ {ok, SCode, Resp_headers_1, Body, Raw_req}
end,
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
@@ -1949,5 +2017,7 @@
ok
end.
-to_binary(X) when is_list(X) -> list_to_binary(X);
-to_binary(X) when is_binary(X) -> X.
+to_binary({X, _}) when is_function(X) -> to_binary(X);
+to_binary(X) when is_function(X) -> <<"body generated by function">>;
+to_binary(X) when is_list(X) -> list_to_binary(X);
+to_binary(X) when is_binary(X) -> X.
diff --git a/test/ibrowse_test.erl b/test/ibrowse_test.erl
index de7b4d5..4d759eb 100644
--- a/test/ibrowse_test.erl
+++ b/test/ibrowse_test.erl
@@ -31,7 +31,8 @@
test_303_response_with_no_body/0,
test_303_response_with_no_body/1,
test_303_response_with_a_body/0,
- test_303_response_with_a_body/1
+ test_303_response_with_a_body/1,
+ test_generate_body_0/0
]).
test_stream_once(Url, Method, Options) ->
@@ -653,6 +654,46 @@
exit({timeout, test_failed})
end.
+%%------------------------------------------------------------------------------
+%% Test requests where body is generated using a Fun
+%%------------------------------------------------------------------------------
+test_generate_body_0() ->
+ io:format("Testing that generation of body using fun works...~n", []),
+ Tid = ets:new(ibrowse_test_state, [public]),
+ try
+ Body_1 = <<"Part 1 of the body">>,
+ Body_2 = <<"Part 2 of the body\r\n\r\n">>,
+ Size = size(Body_1) + size(Body_2),
+ Body = list_to_binary([Body_1, Body_2]),
+ Fun = fun() ->
+ case ets:lookup(Tid, body_gen_state) of
+ [] ->
+ ets:insert(Tid, {body_gen_state, 1}),
+ {ok, Body_1};
+ [{_, 1}]->
+ ets:insert(Tid, {body_gen_state, 2}),
+ {ok, Body_2};
+ [{_, 2}] ->
+ eof
+ end
+ end,
+ case ibrowse:send_req("http://localhost:8181/echo_body",
+ [{"Content-Length", Size}],
+ post,
+ Fun,
+ [{response_format, binary},
+ {http_vsn, {1,0}}]) of
+ {ok, "200", _, Body} ->
+ io:format(" Success~n", []),
+ success;
+ Err ->
+ io:format("Test failed : ~p~n", [Err]),
+ {test_failed, Err}
+ end
+ after
+ ets:delete(Tid)
+ end.
+
do_trace(Fmt, Args) ->
do_trace(get(my_trace_flag), Fmt, Args).
diff --git a/test/ibrowse_test_server.erl b/test/ibrowse_test_server.erl
index d3bfabc..8aed065 100644
--- a/test/ibrowse_test_server.erl
+++ b/test/ibrowse_test_server.erl
@@ -15,7 +15,8 @@
start_server(Port, Sock_type) ->
Fun = fun() ->
- register(server_proc_name(Port), self()),
+ Name = server_proc_name(Port),
+ register(Name, self()),
case do_listen(Sock_type, Port, [{active, false},
{reuseaddr, true},
{nodelay, true},
@@ -30,7 +31,8 @@
"Failed to start server on port ~p. ~p~n",
[Port, Err]))),
exit({listen_error, Err})
- end
+ end,
+ unregister(Name)
end,
spawn_link(Fun).
