Implement database external size calculations
This patch adds calculations to show the "external size" of a database
which is roughly a measure of how much disk space it would take to store
the contents of the database in flat files. It is used to calculate
rough compression ratios for capacity planning.
COUCHDB-2001
diff --git a/include/couch_db.hrl b/include/couch_db.hrl
index 2c015df..2ce5ebe 100644
--- a/include/couch_db.hrl
+++ b/include/couch_db.hrl
@@ -65,7 +65,7 @@
update_seq = 0,
deleted = false,
rev_tree = [],
- leafs_size = 0
+ sizes = {0, 0}
}).
-record(httpd,
@@ -251,6 +251,7 @@
deleted,
ptr,
seq,
- size = nil
+ sizes = {0, 0},
+ atts = []
}).
diff --git a/src/couch_btree.erl b/src/couch_btree.erl
index 9caceb8..ac5681d 100644
--- a/src/couch_btree.erl
+++ b/src/couch_btree.erl
@@ -61,8 +61,8 @@
final_reduce(Reduce, Val);
final_reduce(Reduce, {[], []}) ->
Reduce(reduce, []);
-final_reduce(_Bt, {[], [Red]}) ->
- Red;
+final_reduce(Reduce, {[], [Red]}) ->
+ Reduce(rereduce, [Red]);
final_reduce(Reduce, {[], Reductions}) ->
Reduce(rereduce, Reductions);
final_reduce(Reduce, {KVs, Reductions}) ->
@@ -92,14 +92,14 @@
full_reduce(#btree{root=nil,reduce=Reduce}) ->
{ok, Reduce(reduce, [])};
-full_reduce(#btree{root=Root}) ->
- {ok, element(2, Root)}.
+full_reduce(#btree{root=Root, reduce=Reduce}) ->
+ {ok, Reduce(rereduce, [element(2, Root)])}.
size(#btree{root = nil}) ->
0;
size(#btree{root = {_P, _Red}}) ->
% pre 1.2 format
- nil;
+ undefined;
size(#btree{root = {_P, _Red, Size}}) ->
Size.
diff --git a/src/couch_compress.erl b/src/couch_compress.erl
index 6b47a7a..49876d8 100644
--- a/src/couch_compress.erl
+++ b/src/couch_compress.erl
@@ -14,6 +14,7 @@
-export([compress/2, decompress/1, is_compressed/2]).
-export([get_compression_method/0]).
+-export([uncompressed_length/1]).
-include_lib("couch/include/couch_db.hrl").
@@ -82,3 +83,12 @@
is_compressed(Term, _Method) when not is_binary(Term) ->
false.
+
+uncompressed_length(<<?SNAPPY_PREFIX, _/binary>> = Bin) ->
+ snappy:uncompressed_length(Bin);
+uncompressed_length(<<?COMPRESSED_TERM_PREFIX, _/binary>> = Bin) ->
+ <<131, 80, Size:4/big-unsigned-integer-unit:8, _/binary>> = Bin,
+ Size;
+uncompressed_length(<<?TERM_PREFIX, _/binary>> = Bin) ->
+ size(Bin).
+
diff --git a/src/couch_db.erl b/src/couch_db.erl
index 32a0049..6432e8e 100644
--- a/src/couch_db.erl
+++ b/src/couch_db.erl
@@ -302,44 +302,48 @@
name=Name,
instance_start_time=StartTime,
committed_update_seq=CommittedUpdateSeq,
- id_tree = IdBtree,
- seq_tree = SeqBtree,
- local_tree = LocalBtree
+ id_tree = IdBtree
} = Db,
- {ok, Size} = couch_file:bytes(Fd),
- {ok, DbReduction} = couch_btree:full_reduce(IdBtree),
+ {ok, FileSize} = couch_file:bytes(Fd),
+ {ok, {Count, DelCount, Sizes}} = couch_btree:full_reduce(IdBtree),
+ {ActiveSize0, ExternalSize} = Sizes,
+ ActiveSize = active_size(Db, ActiveSize0),
InfoList = [
{db_name, Name},
- {doc_count, element(1, DbReduction)},
- {doc_del_count, element(2, DbReduction)},
+ {doc_count, Count},
+ {doc_del_count, DelCount},
{update_seq, SeqNum},
{purge_seq, couch_db:get_purge_seq(Db)},
{compact_running, Compactor/=nil},
- {disk_size, Size},
- {data_size, db_data_size(DbReduction, [SeqBtree, IdBtree, LocalBtree])},
+ {disk_size, FileSize},
+ {data_size, ActiveSize},
+ {sizes, {[
+ {file, FileSize},
+ {active, ActiveSize},
+ {external, ExternalSize}
+ ]}},
{instance_start_time, StartTime},
{disk_format_version, DiskVersion},
{committed_update_seq, CommittedUpdateSeq}
],
{ok, InfoList}.
-db_data_size({_Count, _DelCount}, _Trees) ->
- % pre 1.2 format, upgraded on compaction
- null;
-db_data_size({_Count, _DelCount, nil}, _Trees) ->
- null;
-db_data_size({_Count, _DelCount, DocAndAttsSize}, Trees) ->
- sum_tree_sizes(DocAndAttsSize, Trees).
-
-sum_tree_sizes(Acc, []) ->
- Acc;
-sum_tree_sizes(Acc, [T | Rest]) ->
- case couch_btree:size(T) of
- nil ->
- null;
- Sz ->
- sum_tree_sizes(Acc + Sz, Rest)
- end.
+active_size(#db{}=Db, DocActiveSize) ->
+ Trees = [
+ Db#db.id_tree,
+ Db#db.seq_tree,
+ Db#db.local_tree
+ ],
+ lists:foldl(fun(T, Acc) ->
+ case couch_btree:size(T) of
+ _ when Acc == null ->
+ null;
+ undefined ->
+ null;
+ Size ->
+ Acc + Size
+ end
+ end, DocActiveSize, Trees).
get_design_docs(#db{name = <<"shards/", _:18/binary, DbName/binary>>}) ->
{_, Ref} = spawn_monitor(fun() -> exit(fabric:design_docs(DbName)) end),
diff --git a/src/couch_db_updater.erl b/src/couch_db_updater.erl
index 649826a..eb75177 100644
--- a/src/couch_db_updater.erl
+++ b/src/couch_db_updater.erl
@@ -342,40 +342,60 @@
end.
rev_tree(DiskTree) ->
- couch_key_tree:mapfold(fun
- (_RevId, {IsDeleted, BodyPointer, UpdateSeq}, leaf, _Acc) ->
+ couch_key_tree:map(fun
+ (_RevId, {Del, Ptr, Seq}) ->
% pre 1.2 format, will be upgraded on compaction
- {#leaf{deleted=?i2b(IsDeleted), ptr=BodyPointer, seq=UpdateSeq}, nil};
- (_RevId, {IsDeleted, BodyPointer, UpdateSeq}, branch, Acc) ->
- {#leaf{deleted=?i2b(IsDeleted), ptr=BodyPointer, seq=UpdateSeq}, Acc};
- (_RevId, {IsDeleted, BodyPointer, UpdateSeq, Size}, leaf, Acc) ->
- Acc2 = sum_leaf_sizes(Acc, Size),
- {#leaf{deleted=?i2b(IsDeleted), ptr=BodyPointer, seq=UpdateSeq, size=Size}, Acc2};
- (_RevId, {IsDeleted, BodyPointer, UpdateSeq, Size}, branch, Acc) ->
- {#leaf{deleted=?i2b(IsDeleted), ptr=BodyPointer, seq=UpdateSeq, size=Size}, Acc};
- (_RevId, ?REV_MISSING, _Type, Acc) ->
- {?REV_MISSING, Acc}
- end, 0, DiskTree).
+ #leaf{deleted=?i2b(Del), ptr=Ptr, seq=Seq};
+ (_RevId, {Del, Ptr, Seq, Size}) ->
+ % Pre-bigcouch format, will be upgraded on compaction
+ #leaf{
+ deleted = ?i2b(Del),
+ ptr = Ptr,
+ seq = Seq,
+ sizes = {Size, 0},
+ atts = []
+ };
+ (_RevId, {Del, Ptr, Seq, Sizes, Atts}) ->
+ #leaf{
+ deleted = ?i2b(Del),
+ ptr = Ptr,
+ seq = Seq,
+ sizes = Sizes,
+ atts = Atts
+ };
+ (_RevId, ?REV_MISSING) ->
+ ?REV_MISSING
+ end, DiskTree).
disk_tree(RevTree) ->
couch_key_tree:map(fun
(_RevId, ?REV_MISSING) ->
?REV_MISSING;
- (_RevId, #leaf{deleted=IsDeleted, ptr=BodyPointer, seq=UpdateSeq, size=Size}) ->
- {?b2i(IsDeleted), BodyPointer, UpdateSeq, Size}
+ (_RevId, #leaf{}=Leaf) ->
+ #leaf{
+ deleted = Del,
+ ptr = Ptr,
+ seq = Seq,
+ sizes = Sizes,
+ atts = Atts
+ } = Leaf,
+ {?b2i(Del), Ptr, Seq, upgrade_sizes(Sizes), Atts}
end, RevTree).
+upgrade_sizes({_, _} = Sizes) ->
+ Sizes;
+upgrade_sizes(S) when is_integer(S) ->
+ {S, 0}.
+
btree_by_seq_split(#full_doc_info{id=Id, update_seq=Seq, deleted=Del, rev_tree=T}) ->
{Seq, {Id, ?b2i(Del), disk_tree(T)}}.
btree_by_seq_join(Seq, {Id, Del, DiskTree}) when is_integer(Del) ->
- {RevTree, LeafsSize} = rev_tree(DiskTree),
#full_doc_info{
id = Id,
update_seq = Seq,
deleted = ?i2b(Del),
- rev_tree = RevTree,
- leafs_size = LeafsSize
+ rev_tree = rev_tree(DiskTree)
};
btree_by_seq_join(KeySeq, {Id, RevInfos, DeletedRevInfos}) ->
% Older versions stored #doc_info records in the seq_tree.
@@ -389,49 +409,59 @@
[#rev_info{rev=Rev,seq=Seq,deleted=true,body_sp = Bp} ||
{Rev, Seq, Bp} <- DeletedRevInfos]}.
-btree_by_id_split(#full_doc_info{id=Id, update_seq=Seq,
- deleted=Deleted, rev_tree=Tree}) ->
- {Id, {Seq, ?b2i(Deleted), disk_tree(Tree)}}.
+btree_by_id_split(#full_doc_info{}=Info) ->
+ #full_doc_info{
+ id = Id,
+ update_seq = Seq,
+ deleted = Del,
+ sizes = Sizes,
+ rev_tree = Tree
+ } = Info,
+ {Id, {Seq, ?b2i(Del), upgrade_sizes(Sizes), disk_tree(Tree)}}.
btree_by_id_join(Id, {HighSeq, Deleted, DiskTree}) ->
- {Tree, LeafsSize} = rev_tree(DiskTree),
+ % Upgrade from pre-BigCouch disk format
+ ActiveSize = couch_key_tree:fold(fun
+ (_RevId, {_Del, _Ptr, _Seq}, _, Acc) ->
+ Acc;
+ (_RevId, {_Del, _Ptr, _Seq, Size}, _, Acc) ->
+ Acc + Size;
+ (_RevId, {_Del, _Ptr, _Seq, Sizes, _Atts}, _, Acc) ->
+ {Active, _} = Sizes,
+ Active + Acc;
+ (_RevId, ?REV_MISSING, _, Acc) ->
+ Acc
+ end, 0, DiskTree),
+ btree_by_id_join(Id, {HighSeq, Deleted, {ActiveSize, 0}, DiskTree});
+
+btree_by_id_join(Id, {HighSeq, Deleted, Sizes, DiskTree}) ->
#full_doc_info{
id = Id,
update_seq = HighSeq,
deleted = ?i2b(Deleted),
- rev_tree = Tree,
- leafs_size = LeafsSize
+ sizes = Sizes,
+ rev_tree = rev_tree(DiskTree)
}.
btree_by_id_reduce(reduce, FullDocInfos) ->
- lists:foldl(
- fun(Info, {NotDeleted, Deleted, Size}) ->
- Size2 = sum_leaf_sizes(Size, Info#full_doc_info.leafs_size),
- case Info#full_doc_info.deleted of
- true ->
- {NotDeleted, Deleted + 1, Size2};
- false ->
- {NotDeleted + 1, Deleted, Size2}
- end
- end,
- {0, 0, 0}, FullDocInfos);
-btree_by_id_reduce(rereduce, Reds) ->
- lists:foldl(
- fun({NotDeleted, Deleted}, {AccNotDeleted, AccDeleted, _AccSize}) ->
- % pre 1.2 format, will be upgraded on compaction
- {AccNotDeleted + NotDeleted, AccDeleted + Deleted, nil};
- ({NotDeleted, Deleted, Size}, {AccNotDeleted, AccDeleted, AccSize}) ->
- AccSize2 = sum_leaf_sizes(AccSize, Size),
- {AccNotDeleted + NotDeleted, AccDeleted + Deleted, AccSize2}
- end,
- {0, 0, 0}, Reds).
+ lists:foldl(fun
+ (#full_doc_info{deleted=false, sizes=Sizes}, {NotDel, Del, SAcc}) ->
+ {NotDel + 1, Del, reduce_sizes(Sizes, SAcc)};
+ (#full_doc_info{deleted=true, sizes=Sizes}, {NotDel, Del, SAcc}) ->
+ {NotDel, Del + 1, reduce_sizes(Sizes, SAcc)}
+ end, {0, 0, {0, 0}}, FullDocInfos);
+btree_by_id_reduce(rereduce, Reductions) ->
+ lists:foldl(fun
+ ({NotDel, Del}, {NDAcc, DAcc, SAcc}) ->
+ {NotDel + NDAcc, Del + DAcc, SAcc};
+ ({NotDel, Del, Sizes}, {NDAcc, DAcc, SAcc}) ->
+ {NotDel + NDAcc, Del + DAcc, reduce_sizes(Sizes, SAcc)}
+ end, {0, 0, {0, 0}}, Reductions).
-sum_leaf_sizes(nil, _) ->
- nil;
-sum_leaf_sizes(_, nil) ->
- nil;
-sum_leaf_sizes(Size1, Size2) ->
- Size1 + Size2.
+reduce_sizes({A1, E1}, {A2, E2}) ->
+ {A1 + A2, E1 + E2};
+reduce_sizes(S, {_, _} = Acc) when is_integer(S) ->
+ reduce_sizes({S, 0}, Acc).
btree_by_seq_reduce(reduce, DocInfos) ->
% count the number of documents
@@ -549,10 +579,15 @@
flush_trees(#db{fd = Fd} = Db,
[InfoUnflushed | RestUnflushed], AccFlushed) ->
#full_doc_info{update_seq=UpdateSeq, rev_tree=Unflushed} = InfoUnflushed,
- {Flushed, LeafsSize} = couch_key_tree:mapfold(
+ {Flushed, FinalAcc} = couch_key_tree:mapfold(
fun(_Rev, Value, Type, Acc) ->
case Value of
- #doc{deleted = IsDeleted, body = {summary, Summary, AttsFd}} ->
+ #doc{} = Doc ->
+ #doc{
+ deleted = IsDeleted,
+ body = {summary, Summary, AttsFd},
+ atts = Atts
+ } = Doc,
% this node value is actually an unwritten document summary,
% write to disk.
% make sure the Fd in the written bins is the same Fd we are
@@ -571,31 +606,44 @@
" changed. Possibly retrying.", []),
throw(retry)
end,
- {ok, NewSummaryPointer, SummarySize} =
- couch_file:append_raw_chunk(Fd, Summary),
- TotalSize = lists:foldl(
- fun(#att{att_len = L}, A) -> A + L end,
- SummarySize, Value#doc.atts),
- NewValue = #leaf{deleted=IsDeleted, ptr=NewSummaryPointer,
- seq=UpdateSeq, size=TotalSize},
- case Type of
- leaf ->
- {NewValue, Acc + TotalSize};
- branch ->
- {NewValue, Acc}
- end;
- {_, _, _, LeafSize} when Type =:= leaf, LeafSize =/= nil ->
- {Value, Acc + LeafSize};
- _ ->
+ AttsInfo = lists:usort([
+ {P, L} || #att{data = {_, P}, att_len = L} <- Atts
+ ]),
+ [_, _, SummaryBin] = Summary,
+ ExternalSize = couch_compress:uncompressed_length(SummaryBin),
+ {ok, NewPtr, ActiveSize}
+ = couch_file:append_raw_chunk(Fd, Summary),
+ Leaf = #leaf{
+ deleted = IsDeleted,
+ ptr = NewPtr,
+ seq = UpdateSeq,
+ sizes = {ActiveSize, ExternalSize},
+ atts = AttsInfo
+ },
+ {Leaf, add_sizes(Type, Leaf, Acc)};
+ #leaf{} = Leaf ->
+ {Value, add_sizes(Type, Leaf, Acc)};
+ ?REV_MISSING ->
{Value, Acc}
end
- end, 0, Unflushed),
+ end, {0, 0, []}, Unflushed),
+ {FinalAS, FinalES, FinalAtts} = FinalAcc,
+ TotalAttSize = lists:foldl(fun({_, S}, A) -> S + A end, 0, FinalAtts),
InfoFlushed = InfoUnflushed#full_doc_info{
rev_tree = Flushed,
- leafs_size = LeafsSize
+ sizes = {FinalAS + TotalAttSize, FinalES + TotalAttSize}
},
flush_trees(Db, RestUnflushed, [InfoFlushed | AccFlushed]).
+add_sizes(branch, _, Acc) ->
+ Acc;
+add_sizes(leaf, #leaf{sizes=Sizes, atts=AttSizes}, Acc) ->
+ {ActiveSize, ExternalSize} = upgrade_sizes(Sizes),
+ {ASAcc, ESAcc, AttsAcc} = Acc,
+ NewASAcc = ActiveSize + ASAcc,
+ NewESAcc = ExternalSize + ESAcc,
+ NewAttsAcc = lists:umerge(AttSizes, AttsAcc),
+ {NewASAcc, NewESAcc, NewAttsAcc}.
send_result(Client, Ref, NewResult) ->
% used to send a result to the client
@@ -896,23 +944,34 @@
A =< B
end, merge_lookups(MixedInfos, LookupResults)),
- NewInfos1 = lists:map(
- fun(#full_doc_info{rev_tree=RevTree}=Info) ->
- Info#full_doc_info{rev_tree=couch_key_tree:map(
- fun(_, _, branch) ->
- ?REV_MISSING;
- (_Rev, #leaf{ptr=Sp}=Leaf, leaf) ->
- {_Body, AttsInfo} = Summary = copy_doc_attachments(
- Db, Sp, DestFd),
- SummaryChunk = make_doc_summary(NewDb, Summary),
- {ok, Pos, SummarySize} = couch_file:append_raw_chunk(
- DestFd, SummaryChunk),
- TotalLeafSize = lists:foldl(
- fun({_, _, _, AttLen, _, _, _, _}, S) -> S + AttLen end,
- SummarySize, AttsInfo),
- Leaf#leaf{ptr=Pos, size=TotalLeafSize}
- end, RevTree)}
- end, NewInfos0),
+ NewInfos1 = lists:map(fun(Info) ->
+ {NewRevTree, FinalAcc} = couch_key_tree:mapfold(fun
+ (_Rev, #leaf{ptr=Sp}=Leaf, leaf, SizesAcc) ->
+ {Body, AttInfos} = copy_doc_attachments(Db, Sp, DestFd),
+ Summary = make_doc_summary(NewDb, {Body, AttInfos}),
+ [_, _, SummaryBin] = Summary,
+ ExternalSize = couch_compress:uncompressed_length(SummaryBin),
+ {ok, Pos, ActiveSize}
+ = couch_file:append_raw_chunk(DestFd, Summary),
+ AttSizes = [{element(3, A), element(4, A)} || A <- AttInfos],
+ NewLeaf = Leaf#leaf{
+ ptr = Pos,
+ sizes = {ActiveSize, ExternalSize},
+ atts = lists:usort(AttSizes)
+ },
+ {NewLeaf, add_sizes(leaf, NewLeaf, SizesAcc)};
+ (_Rev, _Value, branch, SizesAcc) ->
+ {?REV_MISSING, SizesAcc}
+ end, {0, 0, []}, Info#full_doc_info.rev_tree),
+ {FinalAS, FinalES, FinalAtts} = FinalAcc,
+ TotalAttSize = lists:foldl(fun({_, S}, A) -> S + A end, 0, FinalAtts),
+ NewActiveSize = FinalAS + TotalAttSize,
+ NewExternalSize = FinalES + TotalAttSize,
+ Info#full_doc_info{
+ rev_tree = NewRevTree,
+ sizes = {NewActiveSize, NewExternalSize}
+ }
+ end, NewInfos0),
NewInfos = stem_full_doc_infos(Db, NewInfos1),
RemoveSeqs =
