Add config parameter to reject index all text indexes
Text indexes that index all fields can sometimes lead to OOM issues
when users have documents with nested array fields. This change adds
in a config parameter to provide the ability to log users who do this
and also reject new requests. Note that we need to pass in a db
record to validate_new because it contains user and db name info that
will be logged.
COUCHDB-3249
diff --git a/src/mango_error.erl b/src/mango_error.erl
index f0a8ee2..76a8362 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -174,6 +174,12 @@
<<"index_not_found">>,
fmt("Text index ~s not found in this design doc.", [BadIdx])
};
+info(mango_idx_text, index_all_disabled) ->
+ {
+ 403,
+ <<"index_all_disabled">>,
+ <<"New text indexes are forbidden to index all fields.">>
+ };
info(mango_opts, {invalid_bulk_docs, Val}) ->
{
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index bde3850..a088276 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -84,7 +84,7 @@
chttpd:validate_ctype(Req, "application/json"),
{ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)),
{ok, Idx0} = mango_idx:new(Db, Opts),
- {ok, Idx} = mango_idx:validate_new(Idx0),
+ {ok, Idx} = mango_idx:validate_new(Idx0, Db),
{ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
Id = Idx#idx.ddoc,
Name = Idx#idx.name,
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 11c713b..bc88b97 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -23,7 +23,7 @@
for_sort/2,
new/2,
- validate_new/1,
+ validate_new/2,
add/2,
remove/2,
from_ddoc/2,
@@ -136,9 +136,9 @@
}}.
-validate_new(Idx) ->
+validate_new(Idx, Db) ->
Mod = idx_mod(Idx),
- Mod:validate_new(Idx).
+ Mod:validate_new(Idx, Db).
add(DDoc, Idx) ->
diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl
index 40a1fd3..4cfda5a 100644
--- a/src/mango_idx_text.erl
+++ b/src/mango_idx_text.erl
@@ -14,7 +14,7 @@
-export([
- validate_new/1,
+ validate_new/2,
validate_fields/1,
validate_index_def/1,
add/2,
@@ -32,8 +32,9 @@
-include("mango_idx.hrl").
-validate_new(#idx{}=Idx) ->
+validate_new(#idx{}=Idx, Db) ->
{ok, Def} = do_validate(Idx#idx.def),
+ maybe_reject_index_all_req(Def, Db),
{ok, Idx#idx{def=Def}}.
@@ -327,3 +328,81 @@
indexable_fields(Fields, {op_default, _}) ->
[<<"$default">> | Fields].
+
+
+maybe_reject_index_all_req({Def}, #db{name=DbName, user_ctx=Ctx}) ->
+ User = Ctx#user_ctx.name,
+ Fields = couch_util:get_value(fields, Def),
+ case {Fields, forbid_index_all()} of
+ {all_fields, "true"} ->
+ ?MANGO_ERROR(index_all_disabled);
+ {all_fields, "warn"} ->
+ couch_log:warning("User ~p is indexing all fields in db ~p",
+ [User, DbName]);
+ _ ->
+ ok
+ end.
+
+
+forbid_index_all() ->
+ config:get("mango", "index_all_disabled", "false").
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+
+setup() ->
+ test_util:start_couch(),
+ meck:expect(couch_log, warning, 2,
+ fun(_,_) ->
+ throw({test_error, logged_warning})
+ end),
+ %default index all def that generates {fields, all_fields}
+ Index = #idx{def={[]}},
+ Db = #db{name = <<"testdb">>, user_ctx=#user_ctx{name = <<"u1">>}},
+ {Index, Db}.
+
+
+teardown(_) ->
+ ok = config:delete("mango", "index_all_disabled"),
+ test_util:stop_couch().
+
+
+index_all_test_() ->
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ fun forbid_index_all/1,
+ fun default_and_false_index_all/1,
+ fun warn_index_all/1
+ ]
+
+ }.
+
+
+forbid_index_all({Idx, Db}) ->
+ ok = config:set("mango", "index_all_disabled", "true"),
+ ?_assertThrow({mango_error, ?MODULE, index_all_disabled},
+ validate_new(Idx, Db)
+ ).
+
+
+default_and_false_index_all({Idx, Db}) ->
+ {ok, #idx{def={Def}}} = validate_new(Idx, Db),
+ Fields = couch_util:get_value(fields, Def),
+ ?_assertEqual(all_fields, Fields),
+ ok = config:set("mango", "index_all_disabled", "false"),
+ {ok, #idx{def={Def2}}} = validate_new(Idx, Db),
+ Fields2 = couch_util:get_value(fields, Def2),
+ ?_assertEqual(all_fields, Fields2).
+
+
+warn_index_all({Idx, Db}) ->
+ ok = config:set("mango", "index_all_disabled", "warn"),
+ ?_assertThrow({test_error, logged_warning}, validate_new(Idx, Db)).
+
+
+-endif.
diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl
index 0cc713f..8bad34c 100644
--- a/src/mango_idx_view.erl
+++ b/src/mango_idx_view.erl
@@ -14,7 +14,7 @@
-export([
- validate_new/1,
+ validate_new/2,
validate_index_def/1,
add/2,
remove/2,
@@ -36,7 +36,7 @@
-include("mango_idx.hrl").
-validate_new(#idx{}=Idx) ->
+validate_new(#idx{}=Idx, _Db) ->
{ok, Def} = do_validate(Idx#idx.def),
{ok, Idx#idx{def=Def}}.
