add mango catch-all feature
fall back on the all_docs index if a index for the currently
searched field does not exist.
warn about slower execution and advise to create an index.
PR: #27
PR-URL: https://github.com/apache/couchdb-mango/pull/27
Reviewed-By: Alexander Shorin
Reviewed-By: Tony Sun <tony.sun@cloudant.com>
diff --git a/src/mango_cursor.erl b/src/mango_cursor.erl
index 6cb7fee..911ecdb 100644
--- a/src/mango_cursor.erl
+++ b/src/mango_cursor.erl
@@ -16,7 +16,8 @@
-export([
create/3,
explain/1,
- execute/3
+ execute/3,
+ maybe_filter_indexes/2
]).
@@ -30,29 +31,18 @@
create(Db, Selector0, Opts) ->
Selector = mango_selector:normalize(Selector0),
+ UsableIndexes = mango_idx:get_usable_indexes(Db, Selector0, Opts),
- ExistingIndexes = mango_idx:list(Db),
- if ExistingIndexes /= [] -> ok; true ->
- ?MANGO_ERROR({no_usable_index, no_indexes_defined})
- end,
-
- FilteredIndexes = maybe_filter_indexes(ExistingIndexes, Opts),
- if FilteredIndexes /= [] -> ok; true ->
- ?MANGO_ERROR({no_usable_index, no_index_matching_name})
- end,
-
- SortIndexes = mango_idx:for_sort(FilteredIndexes, Opts),
- if SortIndexes /= [] -> ok; true ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index})
- end,
-
- UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end,
- UsableIndexes = lists:filter(UsableFilter, SortIndexes),
- if UsableIndexes /= [] -> ok; true ->
- ?MANGO_ERROR({no_usable_index, selector_unsupported})
- end,
-
- create_cursor(Db, UsableIndexes, Selector, Opts).
+ {use_index, IndexSpecified} = proplists:lookup(use_index, Opts),
+ case {length(UsableIndexes), length(IndexSpecified)} of
+ {0, 1} ->
+ ?MANGO_ERROR({no_usable_index, selector_unsupported});
+ {0, 0} ->
+ AllDocs = mango_idx:special(Db),
+ create_cursor(Db, AllDocs, Selector, Opts);
+ _ ->
+ create_cursor(Db, UsableIndexes, Selector, Opts)
+ end.
explain(#cursor{}=Cursor) ->
@@ -125,9 +115,9 @@
% queries.
CursorModules = case module_loaded(dreyfus_index) of
true ->
- [mango_cursor_view, mango_cursor_text];
+ [mango_cursor_view, mango_cursor_text, mango_cursor_special];
false ->
- [mango_cursor_view]
+ [mango_cursor_view, mango_cursor_special]
end,
lists:flatmap(fun(CMod) ->
case dict:find(CMod, IdxDict) of
diff --git a/src/mango_cursor_special.erl b/src/mango_cursor_special.erl
new file mode 100644
index 0000000..6058217
--- /dev/null
+++ b/src/mango_cursor_special.erl
@@ -0,0 +1,61 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor_special).
+
+-export([
+ create/4,
+ explain/1,
+ execute/3
+]).
+
+-export([
+ handle_message/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include("mango_cursor.hrl").
+
+
+create(Db, Indexes, Selector, Opts) ->
+ InitialRange = mango_idx_view:field_ranges(Selector),
+ CatchAll = [{<<"_id">>, {'$gt', null, '$lt', mango_json_max}}],
+ FieldRanges = lists:append(CatchAll, InitialRange),
+ Composited = mango_cursor_view:composite_indexes(Indexes, FieldRanges),
+ {Index, IndexRanges} = mango_cursor_view:choose_best_index(Db, Composited),
+
+ Limit = couch_util:get_value(limit, Opts, 10000000000),
+ Skip = couch_util:get_value(skip, Opts, 0),
+ Fields = couch_util:get_value(fields, Opts, all_fields),
+
+ {ok, #cursor{
+ db = Db,
+ index = Index,
+ ranges = IndexRanges,
+ selector = Selector,
+ opts = Opts,
+ limit = Limit,
+ skip = Skip,
+ fields = Fields
+ }}.
+
+
+explain(Cursor) ->
+ mango_cursor_view:explain(Cursor).
+
+execute(Cursor0, UserFun, UserAcc) ->
+ mango_cursor_view:execute(Cursor0, UserFun, UserAcc).
+
+handle_message(Msg, Cursor) ->
+ mango_cursor_view:handle_message(Msg, Cursor).
diff --git a/src/mango_cursor_view.erl b/src/mango_cursor_view.erl
index cb35968..5f109cd 100644
--- a/src/mango_cursor_view.erl
+++ b/src/mango_cursor_view.erl
@@ -19,7 +19,9 @@
]).
-export([
- handle_message/2
+ handle_message/2,
+ composite_indexes/2,
+ choose_best_index/2
]).
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index ef7135c..bde3850 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -181,7 +181,7 @@
chttpd:validate_ctype(Req, "application/json"),
{ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
- {ok, Resp0} = start_find_resp(Req),
+ {ok, Resp0} = start_find_resp(Req, Db, Sel, Opts),
{ok, AccOut} = run_find(Resp0, Db, Sel, Opts),
end_find_resp(AccOut);
@@ -228,8 +228,18 @@
end.
-start_find_resp(Req) ->
- chttpd:start_delayed_json_response(Req, 200, [], "{\"docs\":[").
+start_find_resp(Req, Db, Sel, Opts) ->
+ chttpd:start_delayed_json_response(Req, 200, [], maybe_add_warning(Db, Sel, Opts)).
+
+
+maybe_add_warning(Db, Selector, Opts) ->
+ UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
+ case length(UsableIndexes) of
+ 0 ->
+ "{\"warning\":\"no matching index found, create an index to optimize query time\",\r\n\"docs\":[";
+ _ ->
+ "{\"docs\":["
+ end.
end_find_resp(Acc0) ->
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 377af91..11c713b 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -42,7 +42,8 @@
cursor_mod/1,
idx_mod/1,
to_json/1,
- delete/4
+ delete/4,
+ get_usable_indexes/3
]).
@@ -55,6 +56,27 @@
{ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
Indexes.
+get_usable_indexes(Db, Selector0, Opts) ->
+ Selector = mango_selector:normalize(Selector0),
+
+ ExistingIndexes = mango_idx:list(Db),
+ if ExistingIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, no_indexes_defined})
+ end,
+
+ FilteredIndexes = mango_cursor:maybe_filter_indexes(ExistingIndexes, Opts),
+ if FilteredIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, no_index_matching_name})
+ end,
+
+ SortIndexes = mango_idx:for_sort(FilteredIndexes, Opts),
+ if SortIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, missing_sort_index})
+ end,
+
+ UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end,
+ lists:filter(UsableFilter, SortIndexes).
+
recover(Db) ->
{ok, DDocs0} = mango_util:open_ddocs(Db),
Pred = fun({Props}) ->
@@ -245,7 +267,7 @@
cursor_mod(#idx{type = <<"json">>}) ->
mango_cursor_view;
cursor_mod(#idx{def = all_docs, type= <<"special">>}) ->
- mango_cursor_view;
+ mango_cursor_special;
cursor_mod(#idx{type = <<"text">>}) ->
case module_loaded(dreyfus_index) of
true ->
diff --git a/test/02-basic-find-test.py b/test/02-basic-find-test.py
index 37459a2..c7eb348 100644
--- a/test/02-basic-find-test.py
+++ b/test/02-basic-find-test.py
@@ -236,12 +236,9 @@
assert len(docs) == 15
def test_empty(self):
- try:
- self.db.find({})
- except Exception, e:
- assert e.response.status_code == 400
- else:
- raise AssertionError("bad find")
+ docs = self.db.find({})
+ # 15 users and 9 design docs
+ assert len(docs) == 24
def test_empty_subsel(self):
docs = self.db.find({
diff --git a/test/05-index-selection-test.py b/test/05-index-selection-test.py
index 3a757f8..bbd3aa7 100644
--- a/test/05-index-selection-test.py
+++ b/test/05-index-selection-test.py
@@ -74,6 +74,16 @@
}, use_index=ddocid, explain=True)
assert resp["index"]["ddoc"] == ddocid
+ def test_use_most_columns(self):
+ # ddoc id for the age index
+ ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f"
+ try:
+ self.db.find({}, use_index=ddocid)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
# This doc will not be saved given the new ddoc validation code
# in couch_mrview
def test_manual_bad_view_idx01(self):
