enhance smoosh to cleanup search indexes when ddocs change
diff --git a/src/dreyfus/src/dreyfus_fabric_cleanup.erl b/src/dreyfus/src/dreyfus_fabric_cleanup.erl
index e271074..f2a36d4 100644
--- a/src/dreyfus/src/dreyfus_fabric_cleanup.erl
+++ b/src/dreyfus/src/dreyfus_fabric_cleanup.erl
@@ -14,40 +14,17 @@
-module(dreyfus_fabric_cleanup).
--include("dreyfus.hrl").
-include_lib("mem3/include/mem3.hrl").
-include_lib("couch/include/couch_db.hrl").
-export([go/1]).
go(DbName) ->
- {ok, DesignDocs} = fabric:design_docs(DbName),
- ActiveSigs = lists:usort(
- lists:flatmap(
- fun active_sigs/1,
- [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
- )
- ),
+ ActiveSigs = dreyfus_util:active_sigs(DbName),
cleanup_local_purge_doc(DbName, ActiveSigs),
clouseau_rpc:cleanup(DbName, ActiveSigs),
ok.
-active_sigs(#doc{body = {Fields}} = Doc) ->
- try
- {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}),
- {IndexNames, _} = lists:unzip(RawIndexes),
- [
- begin
- {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName),
- Index#index.sig
- end
- || IndexName <- IndexNames
- ]
- catch
- error:{badmatch, _Error} ->
- []
- end.
-
cleanup_local_purge_doc(DbName, ActiveSigs) ->
{ok, BaseDir} = clouseau_rpc:get_root_dir(),
DbNamePattern = <<DbName/binary, ".*">>,
diff --git a/src/dreyfus/src/dreyfus_util.erl b/src/dreyfus/src/dreyfus_util.erl
index 301d388..328f997 100644
--- a/src/dreyfus/src/dreyfus_util.erl
+++ b/src/dreyfus/src/dreyfus_util.erl
@@ -29,7 +29,8 @@
maybe_create_local_purge_doc/2,
maybe_create_local_purge_doc/3,
get_signature_from_idxdir/1,
- verify_index_exists/2
+ verify_index_exists/2,
+ active_sigs/1
]).
get_shards(DbName, #index_query_args{partition = nil} = Args) ->
@@ -428,6 +429,33 @@
false
end.
+active_sigs(DbName) when is_binary(DbName) ->
+ couch_util:with_db(DbName, fun active_sigs/1);
+active_sigs(Db) ->
+ {ok, DesignDocs} = couch_db:get_design_docs(Db),
+ lists:usort(
+ lists:flatmap(
+ fun sigs_from_ddoc/1,
+ [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
+ )
+ ).
+
+sigs_from_ddoc(#doc{body = {Fields}} = Doc) ->
+ try
+ {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}),
+ {IndexNames, _} = lists:unzip(RawIndexes),
+ [
+ begin
+ {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName),
+ Index#index.sig
+ end
+ || IndexName <- IndexNames
+ ]
+ catch
+ error:{badmatch, _Error} ->
+ []
+ end.
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index dfd433d..53087c6 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -590,25 +590,38 @@
cleanup_local_indices_and_purge_checkpoints([]) ->
ok;
cleanup_local_indices_and_purge_checkpoints([_ | _] = Dbs) ->
- AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
- AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
- Sigs = couch_mrview_util:get_signatures(hd(Dbs)),
- ok = cleanup_purges(Sigs, AllPurges, Dbs),
- ok = cleanup_indices(Sigs, AllIndices).
+ MrViewIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
+ MrViewPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
+ MrViewSigs = couch_mrview_util:get_signatures(hd(Dbs)),
+ ok = cleanup_mrview_purges(MrViewSigs, MrViewPurges, Dbs),
+ ok = cleanup_mrview_indices(MrViewSigs, MrViewIndices),
-cleanup_purges(Sigs, AllPurges, Dbs) ->
+ ClouseauSigs = dreyfus_util:active_sigs(hd(Dbs)),
+ ok = cleanup_clouseau_indices(Dbs, ClouseauSigs),
+
+ NouveauSigs = nouveau_util:active_sigs(hd(Dbs)),
+ ok = cleanup_nouveau_indices(Dbs, NouveauSigs).
+
+cleanup_mrview_purges(Sigs, AllPurges, Dbs) ->
Fun = fun(DbPurges, Db) ->
couch_mrview_cleanup:cleanup_purges(Db, Sigs, DbPurges)
end,
lists:zipwith(Fun, AllPurges, Dbs),
ok.
-cleanup_indices(Sigs, AllIndices) ->
+cleanup_mrview_indices(Sigs, AllIndices) ->
Fun = fun(DbIndices) ->
couch_mrview_cleanup:cleanup_indices(Sigs, DbIndices)
end,
lists:foreach(Fun, AllIndices).
+cleanup_clouseau_indices(Dbs, ActiveSigs) ->
+ Fun = fun(Db) -> clouseau_rpc:cleanup(Db, ActiveSigs) end,
+ lists:foreach(Fun, Dbs).
+cleanup_nouveau_indices(Dbs, ActiveSigs) ->
+ Fun = fun(Db) -> nouveau_api:delete_path(nouveau_util:index_name(Db), ActiveSigs) end,
+ lists:foreach(Fun, Dbs).
+
%% @doc clean up index files for a specific db on all nodes
-spec cleanup_index_files_all_nodes(dbname()) -> [reference()].
cleanup_index_files_all_nodes(DbName) ->
diff --git a/src/nouveau/src/nouveau_fabric_cleanup.erl b/src/nouveau/src/nouveau_fabric_cleanup.erl
index cd4128f..e1567ef 100644
--- a/src/nouveau/src/nouveau_fabric_cleanup.erl
+++ b/src/nouveau/src/nouveau_fabric_cleanup.erl
@@ -14,9 +14,6 @@
-module(nouveau_fabric_cleanup).
--include_lib("couch/include/couch_db.hrl").
-
--include("nouveau.hrl").
-include_lib("mem3/include/mem3.hrl").
-export([go/1]).
@@ -26,7 +23,7 @@
ActiveSigs =
lists:usort(
lists:flatmap(
- fun(Doc) -> active_sigs(DbName, Doc) end,
+ fun(Doc) -> nouveau_util:active_sigs(DbName, Doc) end,
[couch_doc:from_json_obj(DD) || DD <- DesignDocs]
)
),
@@ -37,7 +34,3 @@
end,
Shards
).
-
-active_sigs(DbName, #doc{} = Doc) ->
- Indexes = nouveau_util:design_doc_to_indexes(DbName, Doc),
- lists:map(fun(Index) -> Index#index.sig end, Indexes).
diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl
index 86fc3a4..710fa63 100644
--- a/src/nouveau/src/nouveau_util.erl
+++ b/src/nouveau/src/nouveau_util.erl
@@ -22,6 +22,8 @@
index_name/1,
design_doc_to_indexes/2,
design_doc_to_index/3,
+ active_sigs/1,
+ active_sigs/2,
verify_index_exists/2,
ensure_local_purge_docs/2,
maybe_create_local_purge_doc/2,
@@ -98,6 +100,22 @@
{error, InvalidDDocError}
end.
+active_sigs(DbName) when is_binary(DbName) ->
+ couch_util:with_db(DbName, fun active_sigs/1);
+active_sigs(Db) ->
+ DbName = couch_db:name(Db),
+ {ok, DesignDocs} = couch_db:get_design_docs(Db),
+ lists:usort(
+ lists:flatmap(
+ fun(Doc) -> active_sigs(DbName, Doc) end,
+ [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
+ )
+ ).
+
+active_sigs(DbName, #doc{} = Doc) ->
+ Indexes = design_doc_to_indexes(DbName, Doc),
+ lists:map(fun(Index) -> Index#index.sig end, Indexes).
+
verify_index_exists(DbName, Props) ->
try
Type = couch_util:get_value(<<"type">>, Props),
diff --git a/src/smoosh/src/smoosh_server.erl b/src/smoosh/src/smoosh_server.erl
index 10368a5..5e7314d 100644
--- a/src/smoosh/src/smoosh_server.erl
+++ b/src/smoosh/src/smoosh_server.erl
@@ -123,6 +123,9 @@
handle_db_event(DbName, updated, St) ->
enqueue(DbName),
{ok, St};
+handle_db_event(DbName, ddoc_updated, St) ->
+ enqueue({?INDEX_CLEANUP, mem3:dbname(DbName)}),
+ {ok, St};
handle_db_event(DbName, {index_commit, IdxName}, St) ->
enqueue({DbName, IdxName}),
{ok, St};
@@ -342,6 +345,8 @@
find_channel(State, Rest, Object)
end.
+stale_enough({?INDEX_CLEANUP, _}) ->
+ true;
stale_enough(Object) ->
LastUpdatedSec = last_updated(Object),
Staleness = erlang:monotonic_time(second) - LastUpdatedSec,
