Fix missing mango execution stats (part 2)
The previous implementation of Mango execution stats relied on
passing the docs_examined count from each shard to the coordinator
in the view_row record. This failed to collect the count of
documents read which weren't followed by a match (in a given shard).
For example, if an index was scanned but no documents were matched,
the docs_examined would be 0, when it should be equal to the number
of documents in the index.
This commit changes the implementation so that docs examined is passed
only when each shard has completed its index scan.
The work is split into 2 commits to support mixed-version cluster
upgrades - the previous commit adds the message handlers only
so can be safely rolled out without breaking in-flight requests.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index f5cd382..f1b753b 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -231,32 +231,26 @@
},
case ViewRow#view_row.doc of
null ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
maybe_send_mango_ping();
undefined ->
- ViewRow2 = ViewRow#view_row{
- value = couch_util:get_value(value, Row)
- },
- ok = rexi:stream2(ViewRow2),
- put(mango_docs_examined, 0),
+ % include_docs=false. Use quorum fetch at coordinator
+ ok = rexi:stream2(ViewRow),
set_mango_msg_timestamp();
Doc ->
+ put(mango_docs_examined, get(mango_docs_examined) + 1),
Selector = couch_util:get_value(selector, Options),
case mango_selector:match(Selector, Doc) of
true ->
- ViewRow2 = ViewRow#view_row{
- value = get(mango_docs_examined) + 1
- },
- ok = rexi:stream2(ViewRow2),
- put(mango_docs_examined, 0),
+ ok = rexi:stream2(ViewRow),
set_mango_msg_timestamp();
false ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
maybe_send_mango_ping()
end
end,
{ok, Acc};
view_cb(complete, Acc) ->
+ % Send shard-level execution stats
+ ok = rexi:stream2({execution_stats, {docs_examined, get(mango_docs_examined)}}),
% Finish view output
ok = rexi:stream_last(complete),
{ok, Acc};
@@ -286,16 +280,16 @@
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
case doc_member(Cursor, Props) of
- {ok, Doc, {execution_stats, ExecutionStats1}} ->
+ {ok, Doc, {execution_stats, Stats}} ->
Cursor1 = Cursor#cursor {
- execution_stats = ExecutionStats1
+ execution_stats = Stats
},
Cursor2 = update_bookmark_keys(Cursor1, Props),
FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
handle_doc(Cursor2, FinalDoc);
- {no_match, _, {execution_stats, ExecutionStats1}} ->
+ {no_match, _, {execution_stats, Stats}} ->
Cursor1 = Cursor#cursor {
- execution_stats = ExecutionStats1
+ execution_stats = Stats
},
{ok, Cursor1};
Error ->
@@ -420,20 +414,14 @@
Opts = Cursor#cursor.opts,
ExecutionStats = Cursor#cursor.execution_stats,
Selector = Cursor#cursor.selector,
- {Matched, Incr} = case couch_util:get_value(value, RowProps) of
- N when is_integer(N) -> {true, N};
- _ -> {false, 1}
- end,
case couch_util:get_value(doc, RowProps) of
{DocProps} ->
- ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats, Incr),
- case Matched of
- true ->
- {ok, {DocProps}, {execution_stats, ExecutionStats1}};
- false ->
- match_doc(Selector, {DocProps}, ExecutionStats1)
- end;
+ % only matching documents are returned; the selector
+ % is evaluated at the shard level in view_cb({row, Row},
+ {ok, {DocProps}, {execution_stats, ExecutionStats}};
undefined ->
+ % an undefined doc was returned, indicating we should
+ % perform a quorum fetch
ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
Id = couch_util:get_value(id, RowProps),
case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
@@ -443,9 +431,9 @@
Else ->
Else
end;
- null ->
- ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats),
- {no_match, null, {execution_stats, ExecutionStats1}}
+ _ ->
+ % no doc, no match
+ {no_match, null, {execution_stats, ExecutionStats}}
end.
@@ -481,28 +469,8 @@
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-runs_match_on_doc_with_no_value_test() ->
- Cursor = #cursor {
- db = <<"db">>,
- opts = [],
- execution_stats = #execution_stats{},
- selector = mango_selector:normalize({[{<<"user_id">>, <<"1234">>}]})
- },
- RowProps = [
- {id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {doc,{
- [
- {<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {<<"_rev">>,<<"1-a954fe2308f14307756067b0e18c2968">>},
- {<<"user_id">>,11}
- ]
- }}
- ],
- {Match, _, _} = doc_member(Cursor, RowProps),
- ?assertEqual(Match, no_match).
-does_not_run_match_on_doc_with_value_test() ->
+does_not_refetch_doc_with_value_test() ->
Cursor = #cursor {
db = <<"db">>,
opts = [],
@@ -512,7 +480,6 @@
RowProps = [
{id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
{key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {value,1},
{doc,{
[
{<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 922cadf..d3687f8 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -53,6 +53,10 @@
)
self.assertEqual(resp["execution_stats"]["results_returned"], len(resp["docs"]))
+ def test_no_matches_index_scan(self):
+ resp = self.db.find({"age": {"$lt": 35}, "nomatch": "me"}, return_raw=True, executionStats=True)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["results_returned"], 0)
@unittest.skipUnless(mango.has_text_service(), "requires text service")
class ExecutionStatsTests_Text(mango.UserDocsTextTests):
