Add option to delay responses until the end
When set, every response is sent once fully generated on the server
side. This increases memory usage on the nodes but simplifies error
handling for the client as it eliminates the possibility that the
response will be deliberately terminated midway through due to a
timeout.
The config value can be changed at runtime without impacting any
in-flight responses.
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index f3f12ca..16d568f 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -130,6 +130,9 @@
; _dbs_info in a request
max_db_number_for_dbs_info_req = 100
+; set to true to delay the start of a response until the end has been calculated
+;buffer_response = false
+
; authentication handlers
; authentication_handlers = {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
; uncomment the next line to enable proxy authentication
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index adde073..fb7d61a 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -52,8 +52,9 @@
req,
code,
headers,
- first_chunk,
- resp=nil
+ chunks,
+ resp=nil,
+ buffer_response=false
}).
start_link() ->
@@ -780,40 +781,54 @@
end_json_response(Resp) ->
couch_httpd:end_json_response(Resp).
+
start_delayed_json_response(Req, Code) ->
start_delayed_json_response(Req, Code, []).
+
start_delayed_json_response(Req, Code, Headers) ->
start_delayed_json_response(Req, Code, Headers, "").
+
start_delayed_json_response(Req, Code, Headers, FirstChunk) ->
{ok, #delayed_resp{
start_fun = fun start_json_response/3,
req = Req,
code = Code,
headers = Headers,
- first_chunk = FirstChunk}}.
+ chunks = [FirstChunk],
+ buffer_response = buffer_response(Req)}}.
+
start_delayed_chunked_response(Req, Code, Headers) ->
start_delayed_chunked_response(Req, Code, Headers, "").
+
start_delayed_chunked_response(Req, Code, Headers, FirstChunk) ->
{ok, #delayed_resp{
start_fun = fun start_chunked_response/3,
req = Req,
code = Code,
headers = Headers,
- first_chunk = FirstChunk}}.
+ chunks = [FirstChunk],
+ buffer_response = buffer_response(Req)}}.
-send_delayed_chunk(#delayed_resp{}=DelayedResp, Chunk) ->
+
+send_delayed_chunk(#delayed_resp{buffer_response=false}=DelayedResp, Chunk) ->
{ok, #delayed_resp{resp=Resp}=DelayedResp1} =
start_delayed_response(DelayedResp),
{ok, Resp} = send_chunk(Resp, Chunk),
- {ok, DelayedResp1}.
+ {ok, DelayedResp1};
+
+send_delayed_chunk(#delayed_resp{buffer_response=true}=DelayedResp, Chunk) ->
+ #delayed_resp{chunks = Chunks} = DelayedResp,
+ {ok, DelayedResp#delayed_resp{chunks = [Chunk | Chunks]}}.
+
send_delayed_last_chunk(Req) ->
send_delayed_chunk(Req, []).
+
send_delayed_error(#delayed_resp{req=Req,resp=nil}=DelayedResp, Reason) ->
{Code, ErrorStr, ReasonStr} = error_info(Reason),
{ok, Resp} = send_error(Req, Code, ErrorStr, ReasonStr),
@@ -823,6 +838,7 @@
log_error_with_stack_trace(Reason),
throw({http_abort, Resp, Reason}).
+
close_delayed_json_object(Resp, Buffer, Terminator, 0) ->
% Use a separate chunk to close the streamed array to maintain strict
% compatibility with earlier versions. See COUCHDB-2724
@@ -831,10 +847,22 @@
close_delayed_json_object(Resp, Buffer, Terminator, _Threshold) ->
send_delayed_chunk(Resp, [Buffer | Terminator]).
-end_delayed_json_response(#delayed_resp{}=DelayedResp) ->
+
+end_delayed_json_response(#delayed_resp{buffer_response=false}=DelayedResp) ->
{ok, #delayed_resp{resp=Resp}} =
start_delayed_response(DelayedResp),
- end_json_response(Resp).
+ end_json_response(Resp);
+
+end_delayed_json_response(#delayed_resp{buffer_response=true}=DelayedResp) ->
+ #delayed_resp{
+ req = Req,
+ code = Code,
+ headers = Headers,
+ chunks = Chunks
+ } = DelayedResp,
+ {ok, Resp} = start_response_length(Req, Code, Headers, iolist_size(Chunks)),
+ send(Resp, lists:reverse(Chunks)).
+
get_delayed_req(#delayed_resp{req=#httpd{mochi_req=MochiReq}}) ->
MochiReq;
@@ -847,7 +875,7 @@
req=Req,
code=Code,
headers=Headers,
- first_chunk=FirstChunk
+ chunks=[FirstChunk]
}=DelayedResp,
{ok, Resp} = StartFun(Req, Code, Headers),
case FirstChunk of
@@ -858,6 +886,18 @@
start_delayed_response(#delayed_resp{}=DelayedResp) ->
{ok, DelayedResp}.
+
+buffer_response(Req) ->
+ case chttpd:qs_value(Req, "buffer_response") of
+ "false" ->
+ false;
+ "true" ->
+ true;
+ _ ->
+ config:get_boolean("chttpd", "buffer_response", false)
+ end.
+
+
error_info({Error, Reason}) when is_list(Reason) ->
error_info({Error, couch_util:to_binary(Reason)});
error_info(bad_request) ->
diff --git a/src/chttpd/test/eunit/chttpd_delayed_test.erl b/src/chttpd/test/eunit/chttpd_delayed_test.erl
new file mode 100644
index 0000000..64232dc
--- /dev/null
+++ b/src/chttpd/test/eunit/chttpd_delayed_test.erl
@@ -0,0 +1,73 @@
+-module(chttpd_delayed_test).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(USER, "chttpd_view_test_admin").
+-define(PASS, "pass").
+-define(AUTH, {basic_auth, {?USER, ?PASS}}).
+-define(CONTENT_JSON, {"Content-Type", "application/json"}).
+-define(DDOC, "{\"_id\": \"_design/bar\", \"views\": {\"baz\":
+ {\"map\": \"function(doc) {emit(doc._id, doc._id);}\"}}}").
+
+-define(FIXTURE_TXT, ?ABS_PATH(?FILE)).
+-define(i2l(I), integer_to_list(I)).
+-define(TIMEOUT, 60). % seconds
+
+setup() ->
+ Hashed = couch_passwords:hash_admin_password(?PASS),
+ ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false),
+ ok = config:set("chttpd", "buffer_response", "true"),
+ TmpDb = ?tempdb(),
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Url = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]),
+ create_db(Url),
+ Url.
+
+teardown(Url) ->
+ delete_db(Url),
+ ok = config:delete("admins", ?USER, _Persist=false).
+
+create_db(Url) ->
+ {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"),
+ ?assert(Status =:= 201 orelse Status =:= 202).
+
+
+delete_db(Url) ->
+ {ok, 200, _, _} = test_request:delete(Url, [?AUTH]).
+
+
+all_test_() ->
+ {
+ "chttpd delay tests",
+ {
+ setup,
+ fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun test_buffer_response_all_docs/1,
+ fun test_buffer_response_changes/1
+ ]
+ }
+ }
+ }.
+
+
+test_buffer_response_all_docs(Url) ->
+ assert_has_content_length(Url ++ "/_all_docs").
+
+
+test_buffer_response_changes(Url) ->
+ assert_has_content_length(Url ++ "/_changes").
+
+
+assert_has_content_length(Url) ->
+ {timeout, ?TIMEOUT, ?_test(begin
+ {ok, Code, Headers, _Body} = test_request:get(Url, [?AUTH]),
+ ?assertEqual(200, Code),
+ ?assert(lists:keymember("Content-Length", 1, Headers))
+ end)}.
+
\ No newline at end of file
