Merge pull request #46 from basho/rdb-counter-clear
Expose the clear/1 fun from counter in the folsom api
diff --git a/README.md b/README.md
index bfa2153..6d60521 100644
--- a/README.md
+++ b/README.md
@@ -73,6 +73,45 @@
> folsom_metrics:histogram_timed_update(Name, Fun).
> folsom_metrics:notify({Name, Value}).
+###### Histogram sample types
+
+Each histogram draws its values from a `reservoir` of readings. You can select a `sample type` for a histogram by passing the name of the sample type as an atom when you create a new histogram.
+Some sample types have further arguments. The purpose of a sample type is to control the size and charecteristics of the reservoir of readings the histogram performs analysis upon.
+
+Folsom currently provides the following sample types:
+
+###### `uniform`
+
+This is a random uniform sample over the stream of readings. This is the default sample type, bounded in size to 1028 readings. When `size` readings have been taken, new readings replace older readings
+in the reservoir at random. You can set the sample size at creation time:
+
+ > folsom_metrics:new_histogram(Name, uniform, Size::integer()).
+
+Be sure you understand _why_ before you do this.
+
+###### `exdec`
+
+This is a sample that exponentially decays less significant readings over time so as to give greater significance to newer readings. Read more here -
+[Forward Decay...](http://www.research.att.com/people/Cormode_Graham/library/publications/CormodeShkapenyukSrivastavaXu09.pdf).
+Again you can change defaults at creation time, if you think you need to:
+
+ > folsom_metrics:new_histogram(Name, exdec, Size::integer(), Alpha::float()).
+
+###### `slide`
+
+This is a sliding window in time over a stream of readings. The default window size is 60 seconds. Every reading that occurs in a sliding sixty second window is stored,
+with older readings being discarded. If you have a lot of readings per
+minute the `reservoir` may get pretty big and so it will take more time to calculate statistics. You can set the `window` size by providing a number of seconds.
+
+ > folsom_metrics:new_histogram(Name, slide, Seconds::integer()).
+
+###### `slide_uniform`
+
+This is a sliding window in time over a stream of readings with a random uniform sample per second, to bound the size of the total number of readings. The maximum size of the reservoir will be
+ `window size * sample size`. Default is a window of 60 seconds and a sample size of 1028. Again, you can change these at creation time:
+
+ > folsom_metrics:new_histogram(Name, slide_uniform, {Secs::interger(), Size::integer()).
+
##### Histories
Histories are a collection of past events, such as errors or log messages.
@@ -88,6 +127,13 @@
> folsom_metrics:new_meter(Name).
> folsom_metrics:notify({Name, Value}).
+###### `Spiral` meter
+
+A `spiral` is a type of meter that has a one minute sliding window count. The meter tracks an increment only counter and a total for the last minute. This is a sliding count with older readings dropping off per second.
+
+ > folsom_metrics:new_spiral(Name).
+ > folsom_metrics:notify({Name, Count}).
+
##### Meter Reader
Meter readers are like a meter except that the values passed to it are monotonically increasing, e.g., reading from a water or gas meter, CPU jiffies, or I/O operation count.
diff --git a/include/folsom.hrl b/include/folsom.hrl
index e408cc7..a7aa543 100644
--- a/include/folsom.hrl
+++ b/include/folsom.hrl
@@ -17,7 +17,7 @@
-record(spiral, {
tid = folsom_metrics_histogram_ets:new(folsom_spiral,
- [ordered_set,
+ [set,
{write_concurrency, true},
public]),
server
@@ -33,7 +33,7 @@
-record(slide_uniform, {
window = ?DEFAULT_SLIDING_WINDOW,
size = ?DEFAULT_SIZE,
- reservoir = folsom_metrics_histogram_ets:new(folsom_slide_uniform,[ordered_set, {write_concurrency, true}, public]),
+ reservoir = folsom_metrics_histogram_ets:new(folsom_slide_uniform,[set, {write_concurrency, true}, public]),
seed = now(),
server
}).
@@ -88,7 +88,7 @@
dist_buf_busy_limit,
%fullsweep_after, % included in garbage_collection
garbage_collection,
- global_heaps_size,
+ %global_heaps_size, % deprecated
heap_sizes,
heap_type,
info,
diff --git a/rebar.config b/rebar.config
index 479ba7b..0f72aae 100644
--- a/rebar.config
+++ b/rebar.config
@@ -1,7 +1,7 @@
{sub_dirs, ["deps"]}.
{deps, [
- {'bear', ".*", {git, "git://github.com/boundary/bear.git", "master"}},
+ {'bear', ".*", {git, "git://github.com/boundary/bear.git", {tag, "0.1.2"}}},
{meck, ".*", {git, "git://github.com/eproxus/meck", "master"}}
]}.
diff --git a/src/folsom.erl b/src/folsom.erl
index 0f08da0..9fbe2e0 100644
--- a/src/folsom.erl
+++ b/src/folsom.erl
@@ -36,24 +36,29 @@
start(_Type, _Args) ->
{ok, Pid} = folsom_sup:start_link(),
- lists:foreach(
- fun ({K, New}) ->
- case application:get_env(?APP, K) of
- {ok, Name} when is_atom(Name) ->
- New(Name);
- {ok, Names} when is_list(Names) ->
- lists:foreach(New, Names);
- undefined ->
- ok
- end
- end,
- [{counter, fun folsom_metrics:new_counter/1},
- {gauge, fun folsom_metrics:new_gauge/1},
- {histogram, fun folsom_metrics:new_histogram/1},
- {history, fun folsom_metrics:new_history/1},
- {meter, fun folsom_metrics:new_meter/1},
- {meter_reader, fun folsom_metrics:new_meter_reader/1}]),
+ lists:foreach(fun configure/1,
+ [{counter, new_counter},
+ {gauge, new_gauge},
+ {histogram, new_histogram},
+ {history, new_history},
+ {meter, new_meter},
+ {meter_reader, new_meter_reader}]),
{ok, Pid}.
stop(_State) ->
ok.
+
+%% internal
+configure({K, New}) ->
+ case application:get_env(?APP, K) of
+ {ok, Specs} when is_list(Specs) ->
+ [configure_metric(New, Spec) || Spec <- Specs];
+ {ok, Spec} ->
+ configure_metric(New, Spec);
+ undefined -> ok
+ end.
+
+configure_metric(New, Spec) when is_list(Spec) ->
+ apply(folsom_metrics, New, Spec);
+configure_metric(New, Name) ->
+ folsom_metrics:New(Name).
diff --git a/src/folsom_ets.erl b/src/folsom_ets.erl
index 27857f7..2e6eb5e 100644
--- a/src/folsom_ets.erl
+++ b/src/folsom_ets.erl
@@ -251,10 +251,12 @@
true = ets:delete(?FOLSOM_TABLE, Name),
ok;
delete_metric(Name, meter) ->
+ ok = folsom_meter_timer_server:unregister(Name),
true = ets:delete(?METER_TABLE, Name),
true = ets:delete(?FOLSOM_TABLE, Name),
ok;
delete_metric(Name, meter_reader) ->
+ ok = folsom_meter_timer_server:unregister(Name),
true = ets:delete(?METER_READER_TABLE, Name),
true = ets:delete(?FOLSOM_TABLE, Name),
ok;
diff --git a/src/folsom_ewma.erl b/src/folsom_ewma.erl
index fc82781..77b2413 100644
--- a/src/folsom_ewma.erl
+++ b/src/folsom_ewma.erl
@@ -29,9 +29,10 @@
-module(folsom_ewma).
--define(M1_ALPHA, 1 - math:exp(-5 / 60.0)).
--define(M5_ALPHA, 1 - math:exp(-5 / 60.0 / 5)).
--define(M15_ALPHA, 1 - math:exp(-5 / 60.0 / 15)).
+-define(M1_ALPHA, 1 - math:exp(-5 / 60.0)).
+-define(M5_ALPHA, 1 - math:exp(-5 / 60.0 / 5)).
+-define(M15_ALPHA, 1 - math:exp(-5 / 60.0 / 15)).
+-define(D1_ALPHA, 1 - math:exp(-5 / 60.0 / 1440)).
-record(ewma, {
alpha,
@@ -47,7 +48,8 @@
tick/1,
one_minute_ewma/0,
five_minute_ewma/0,
- fifteen_minute_ewma/0]).
+ fifteen_minute_ewma/0,
+ one_day_ewma/0]).
% API
@@ -61,6 +63,9 @@
fifteen_minute_ewma() ->
new(?M15_ALPHA, 5).
+one_day_ewma() ->
+ new(?D1_ALPHA, 5).
+
new(Alpha, Interval) ->
#ewma{alpha = Alpha, interval = Interval}.
diff --git a/src/folsom_meter_timer_server.erl b/src/folsom_meter_timer_server.erl
index 5a38c07..c6043a1 100644
--- a/src/folsom_meter_timer_server.erl
+++ b/src/folsom_meter_timer_server.erl
@@ -28,7 +28,7 @@
-behaviour(gen_server).
%% API
--export([start_link/0, register/2, dump/0]).
+-export([start_link/0, register/2, unregister/1, dump/0]).
%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -98,6 +98,15 @@
#state{registered_timers = NewList}
end,
{reply, ok, NewState};
+handle_call({unregister, Name}, _From, State) ->
+ NewState = case proplists:is_defined(Name, State#state.registered_timers) of
+ true ->
+ Ref = proplists:get_value(Name, State#state.registered_timers),
+ {ok, cancel} = timer:cancel(Ref),
+ #state{registered_timers = proplists:delete(Name, State#state.registered_timers)};
+ false -> State
+ end,
+ {reply, ok, NewState};
handle_call(dump, _From, State) ->
{reply, State, State}.
@@ -160,5 +169,8 @@
register(Name, Module) ->
gen_server:call(?SERVER, {register, Name, Module}).
+unregister(Name) ->
+ gen_server:call(?SERVER, {unregister, Name}).
+
dump() ->
gen_server:call(?SERVER, dump).
diff --git a/src/folsom_metrics.erl b/src/folsom_metrics.erl
index db0936f..621fb7d 100644
--- a/src/folsom_metrics.erl
+++ b/src/folsom_metrics.erl
@@ -55,7 +55,9 @@
get_history_values/2,
histogram_timed_update/2,
histogram_timed_update/3,
- histogram_timed_update/4
+ histogram_timed_update/4,
+ histogram_timed_begin/1,
+ histogram_timed_notify/1
]).
-include("folsom.hrl").
@@ -163,3 +165,11 @@
{Time, Value} = timer:tc(Mod, Fun, Args),
ok = notify({Name, Time}),
Value.
+
+histogram_timed_begin(Name) ->
+ {Name, os:timestamp()}.
+
+histogram_timed_notify({Name, Begin}) ->
+ Now = os:timestamp(),
+ Time = timer:now_diff(Now, Begin),
+ ok = notify({Name, Time}).
diff --git a/src/folsom_metrics_counter.erl b/src/folsom_metrics_counter.erl
index 9698150..1861ffe 100644
--- a/src/folsom_metrics_counter.erl
+++ b/src/folsom_metrics_counter.erl
@@ -32,27 +32,34 @@
get_value/1,
clear/1]).
+-define(WIDTH, 16). %% Keep this a power of two
+
-include("folsom.hrl").
new(Name) ->
- Counter = {Name, 0},
- ets:insert(?COUNTER_TABLE, Counter).
+ Counters = [{{Name,N}, 0} || N <- lists:seq(0,?WIDTH-1)],
+ ets:insert(?COUNTER_TABLE, Counters).
inc(Name) ->
- ets:update_counter(?COUNTER_TABLE, Name, 1).
+ ets:update_counter(?COUNTER_TABLE, key(Name), 1).
inc(Name, Value) ->
- ets:update_counter(?COUNTER_TABLE, Name, Value).
+ ets:update_counter(?COUNTER_TABLE, key(Name), Value).
dec(Name) ->
- ets:update_counter(?COUNTER_TABLE, Name, -1).
+ ets:update_counter(?COUNTER_TABLE, key(Name), -1).
dec(Name, Value) ->
- ets:update_counter(?COUNTER_TABLE, Name, -Value).
+ ets:update_counter(?COUNTER_TABLE, key(Name), -Value).
get_value(Name) ->
- [{_, Values}] = ets:lookup(?COUNTER_TABLE, Name),
- Values.
+ Count = lists:sum(ets:select(?COUNTER_TABLE, [{{{Name,'_'},'$1'},[],['$1']}])),
+ Count.
clear(Name) ->
new(Name).
+
+key(Name) ->
+ X = erlang:system_info(scheduler_id),
+ Rnd = X band (?WIDTH-1),
+ {Name, Rnd}.
diff --git a/src/folsom_metrics_duration.erl b/src/folsom_metrics_duration.erl
index 6b53566..da937f0 100644
--- a/src/folsom_metrics_duration.erl
+++ b/src/folsom_metrics_duration.erl
@@ -45,10 +45,10 @@
ets:insert(?DURATION_TABLE, Dur).
update(Name, timer_start) ->
- StartTime = erlang:now(),
+ StartTime = os:timestamp(),
ets:update_element(?DURATION_TABLE, Name, {3, StartTime});
update(Name, timer_end) ->
- EndTime = erlang:now(),
+ EndTime = os:timestamp(),
case ets:lookup_element(?DURATION_TABLE, Name, 3) of
undefined ->
ok;
diff --git a/src/folsom_metrics_histogram.erl b/src/folsom_metrics_histogram.erl
index 401e29f..b24db65 100644
--- a/src/folsom_metrics_histogram.erl
+++ b/src/folsom_metrics_histogram.erl
@@ -57,8 +57,14 @@
update(Name, Value) ->
Hist = get_value(Name),
- NewSample = folsom_sample:update(Hist#histogram.type, Hist#histogram.sample, Value),
- ets:insert(?HISTOGRAM_TABLE, {Name, Hist#histogram{sample = NewSample}}).
+ Sample = Hist#histogram.sample,
+ case folsom_sample:update(Hist#histogram.type, Hist#histogram.sample, Value) of
+ Sample ->
+ %% sample didn't change, don't need to write it back
+ true;
+ NewSample ->
+ ets:insert(?HISTOGRAM_TABLE, {Name, Hist#histogram{sample = NewSample}})
+ end.
% gets the histogram record from ets
get_value(Name) ->
diff --git a/src/folsom_metrics_meter.erl b/src/folsom_metrics_meter.erl
index 5f8236b..7646b0d 100644
--- a/src/folsom_metrics_meter.erl
+++ b/src/folsom_metrics_meter.erl
@@ -37,6 +37,7 @@
one,
five,
fifteen,
+ day,
count = 0,
start_time
}).
@@ -47,26 +48,31 @@
OneMin = folsom_ewma:one_minute_ewma(),
FiveMin = folsom_ewma:five_minute_ewma(),
FifteenMin = folsom_ewma:fifteen_minute_ewma(),
+ OneDay = folsom_ewma:one_day_ewma(),
ets:insert(?METER_TABLE,
{Name, #meter{one = OneMin,
five = FiveMin,
fifteen = FifteenMin,
+ day = OneDay,
start_time = folsom_utils:now_epoch_micro()}}).
tick(Name) ->
#meter{one = OneMin,
five = FiveMin,
- fifteen = FifteenMin} = Meter = get_value(Name),
+ fifteen = FifteenMin,
+ day = OneDay} = Meter = get_value(Name),
OneMin1 = folsom_ewma:tick(OneMin),
FiveMin1 = folsom_ewma:tick(FiveMin),
FifteenMin1 = folsom_ewma:tick(FifteenMin),
+ OneDay1 = folsom_ewma:tick(OneDay),
ets:insert(?METER_TABLE,
{Name, Meter#meter{one = OneMin1,
five = FiveMin1,
- fifteen = FifteenMin1}}).
+ fifteen = FifteenMin1,
+ day = OneDay1}}).
mark(Name) ->
mark(Name, 1).
@@ -75,21 +81,25 @@
#meter{count = Count,
one = OneMin,
five = FiveMin,
- fifteen = FifteenMin} = Meter = get_value(Name),
+ fifteen = FifteenMin,
+ day = OneDay} = Meter = get_value(Name),
OneMin1 = folsom_ewma:update(OneMin, Value),
FiveMin1 = folsom_ewma:update(FiveMin, Value),
FifteenMin1 = folsom_ewma:update(FifteenMin, Value),
+ OneDay1 = folsom_ewma:update(OneDay, Value),
ets:insert(?METER_TABLE, {Name, Meter#meter{count = Count + Value,
one = OneMin1,
five = FiveMin1,
- fifteen = FifteenMin1}}).
+ fifteen = FifteenMin1,
+ day = OneDay1}}).
get_values(Name) ->
#meter{one = OneMin,
five = FiveMin,
fifteen = FifteenMin,
+ day = OneDay,
count = Count} = Meter = get_value(Name),
L = [
@@ -97,6 +107,7 @@
{one, get_rate(OneMin)},
{five, get_rate(FiveMin)},
{fifteen, get_rate(FifteenMin)},
+ {day, get_rate(OneDay)},
{mean, get_mean_rate(Meter)},
{acceleration, get_acceleration(Name)}
],
diff --git a/src/folsom_metrics_spiral.erl b/src/folsom_metrics_spiral.erl
index 547fe8b..d206dc4 100644
--- a/src/folsom_metrics_spiral.erl
+++ b/src/folsom_metrics_spiral.erl
@@ -34,6 +34,7 @@
%% size of the window in seconds
-define(WINDOW, 60).
+-define(WIDTH, 16). %% Keep this a power of two
-include("folsom.hrl").
@@ -42,15 +43,17 @@
Pid = folsom_sample_slide_sup:start_slide_server(?MODULE,
Spiral#spiral.tid,
?WINDOW),
- ets:insert_new(Spiral#spiral.tid, {count, 0}),
+ ets:insert_new(Spiral#spiral.tid,
+ [{{count, N}, 0} || N <- lists:seq(0,?WIDTH-1)]),
ets:insert(?SPIRAL_TABLE, {Name, Spiral#spiral{server=Pid}}).
update(Name, Value) ->
#spiral{tid=Tid} = get_value(Name),
Moment = folsom_utils:now_epoch(),
- ets:insert_new(Tid, {Moment, 0}),
- ets:update_counter(Tid, Moment, Value),
- ets:update_counter(Tid, count, Value).
+ X = erlang:system_info(scheduler_id),
+ Rnd = X band (?WIDTH-1),
+ folsom_utils:update_counter(Tid, {Moment, Rnd}, Value),
+ ets:update_counter(Tid, {count, Rnd}, Value).
get_value(Name) ->
[{Name, Spiral}] = ets:lookup(?SPIRAL_TABLE, Name),
@@ -58,13 +61,13 @@
trim(Tid, _Window) ->
Oldest = oldest(),
- ets:select_delete(Tid, [{{'$1','_'}, [{is_integer, '$1'}, {'<', '$1', Oldest}], ['true']}]).
+ ets:select_delete(Tid, [{{{'$1','_'},'_'}, [{is_integer, '$1'}, {'<', '$1', Oldest}], ['true']}]).
get_values(Name) ->
Oldest = oldest(),
#spiral{tid=Tid} = get_value(Name),
- [{count, Count}] = ets:lookup(Tid, count),
- One =lists:sum(ets:select(Tid, [{{'$1','$2'},[{is_integer, '$1'}, {'>=', '$1', Oldest}],['$2']}])),
+ Count = lists:sum(ets:select(Tid, [{{{count,'_'},'$1'},[],['$1']}])),
+ One = lists:sum(ets:select(Tid, [{{{'$1','_'},'$2'},[{is_integer, '$1'}, {'>=', '$1', Oldest}],['$2']}])),
[{count, Count}, {one, One}].
diff --git a/src/folsom_sample_slide_uniform.erl b/src/folsom_sample_slide_uniform.erl
index 7e7b5eb..1d0204c 100644
--- a/src/folsom_sample_slide_uniform.erl
+++ b/src/folsom_sample_slide_uniform.erl
@@ -38,19 +38,19 @@
Pid = folsom_sample_slide_sup:start_slide_server(?MODULE, Sample#slide_uniform.reservoir, Sample#slide_uniform.window),
Sample#slide_uniform{server=Pid}.
-update(#slide_uniform{reservoir = Reservoir, size = Size, seed = Seed} = Sample0, Value) ->
- Moment = moment(),
- ets:insert_new(Reservoir, {Moment, 0}),
- MCnt = ets:update_counter(Reservoir, Moment, 1),
+update(#slide_uniform{reservoir = Reservoir, size = Size} = Sample0, Value) ->
+ Now = folsom_utils:timestamp(),
+ Moment = folsom_utils:now_epoch(Now),
+ MCnt = folsom_utils:update_counter(Reservoir, Moment, 1),
Sample = case MCnt > Size of
true ->
- {Rnd, NewSeed} = random:uniform_s(Size, Seed),
+ {Rnd, _NewSeed} = random:uniform_s(MCnt, Now),
maybe_update(Reservoir, {{Moment, Rnd}, Value}, Size),
- Sample0#slide_uniform{seed = NewSeed};
- false ->
- ets:insert(Reservoir, {{Moment, MCnt}, Value}),
- Sample0
- end,
+ Sample0;
+ false ->
+ ets:insert(Reservoir, {{Moment, MCnt}, Value}),
+ Sample0
+ end,
Sample.
maybe_update(Reservoir, {{_Moment, Rnd}, _Value}=Obj, Size) when Rnd =< Size ->
diff --git a/src/folsom_utils.erl b/src/folsom_utils.erl
index d6192cd..bbea804 100644
--- a/src/folsom_utils.erl
+++ b/src/folsom_utils.erl
@@ -28,8 +28,11 @@
to_atom/1,
convert_tags/1,
now_epoch/0,
+ now_epoch/1,
now_epoch_micro/0,
- get_ets_size/1
+ timestamp/0,
+ get_ets_size/1,
+ update_counter/3
]).
to_atom(Binary) when is_binary(Binary) ->
@@ -41,12 +44,39 @@
[to_atom(Tag) || Tag <- Tags].
now_epoch() ->
- {Mega, Sec, _} = erlang:now(),
+ now_epoch(os:timestamp()).
+
+now_epoch({Mega, Sec, _}) ->
(Mega * 1000000 + Sec).
now_epoch_micro() ->
- {Mega, Sec, Micro} = erlang:now(),
+ {Mega, Sec, Micro} = os:timestamp(),
(Mega * 1000000 + Sec) * 1000000 + Micro.
+%% useful because you can't meck os:timestamp for some reason
+timestamp() ->
+ os:timestamp().
+
get_ets_size(Tab) ->
ets:info(Tab, size).
+
+%% @doc
+%% Same as {@link ets:update_counter/3} but inserts `{Key, Value}' if object
+%% is missing in the table.
+update_counter(Tid, Key, Value) when is_integer(Value) ->
+ %% try to update the counter, will badarg if it doesn't exist
+ try ets:update_counter(Tid, Key, Value) of
+ Res ->
+ Res
+ catch
+ error:badarg ->
+ %% row didn't exist, create it
+ %% use insert_new to avoid races
+ case ets:insert_new(Tid, {Key, Value}) of
+ true ->
+ Value;
+ false ->
+ %% someone beat us to it
+ ets:update_counter(Tid, Key, Value)
+ end
+ end.
diff --git a/src/folsom_vm_metrics.erl b/src/folsom_vm_metrics.erl
index 72eedb6..4ff72d1 100644
--- a/src/folsom_vm_metrics.erl
+++ b/src/folsom_vm_metrics.erl
@@ -35,7 +35,7 @@
]).
%% exported for eunit test
--export([convert_cpu_topology/2]).
+-export([convert_system_info/1]).
-include("folsom.hrl").
@@ -69,10 +69,6 @@
convert_statistics({context_switches, {ContextSwitches, 0}}) ->
ContextSwitches;
-convert_statistics({exact_reductions, {TotalExactReductions,
- ExactReductionsSinceLastCall}}) ->
- [{"total_exact_reductions", TotalExactReductions},
- {"exact_reductions_since_last_call", ExactReductionsSinceLastCall}];
convert_statistics({garbage_collection, {NumberofGCs, WordsReclaimed, 0}}) ->
[{"number_of_gcs", NumberofGCs}, {"words_reclaimed", WordsReclaimed}];
convert_statistics({io, {Input, Output}}) ->
@@ -98,8 +94,10 @@
[{compiler, Compiler}, {version, convert_c_compiler_version(Version)}];
convert_system_info({cpu_topology, undefined}) ->
undefined;
-convert_system_info({cpu_topology, List}) ->
+convert_system_info({cpu_topology, List}) when is_list(List) ->
[{Type, convert_cpu_topology(Item, [])} || {Type, Item} <- List];
+convert_system_info({cpu_topology, {logical,Item}}) ->
+ convert_system_info({cpu_topology, [{processor,[{core,{logical,Item}}]}]});
convert_system_info({dist_ctrl, List}) ->
lists:map(fun({Node, Socket}) ->
{ok, Stats} = inet:getstat(Socket),
@@ -130,22 +128,24 @@
convert_c_compiler_version({A, B, C}) ->
list_to_binary(io_lib:format("~p.~p.~p", [A, B, C]));
convert_c_compiler_version({A, B}) ->
- list_to_binary(io_lib:format("~p.~p", [A, B])).
+ list_to_binary(io_lib:format("~p.~p", [A, B]));
+convert_c_compiler_version(A) ->
+ list_to_binary(io_lib:format("~p", [A])).
convert_cpu_topology([{core, Value}| Tail], Acc) when is_tuple(Value) ->
- convert_cpu_topology(Tail, lists:append(Acc, [{core, tuple_to_list(Value)}]));
+ convert_cpu_topology(Tail, lists:append(Acc, [{core, tuple_to_list(Value)}]));
convert_cpu_topology([{core, Value}| Tail], Acc) when is_list(Value) ->
- convert_cpu_topology(Tail, lists:append(Acc, [{core, convert_cpu_topology(Value, [])}]));
+ convert_cpu_topology(Tail, lists:append(Acc, [{core, convert_cpu_topology(Value, [])}]));
convert_cpu_topology([{thread, Value}| Tail], Acc) ->
- convert_cpu_topology(Tail, lists:append(Acc, [{thread, tuple_to_list(Value)}]));
-convert_cpu_topology({logical, Value}, Acc) ->
- convert_cpu_topology([], lists:append(Acc, [logical, Value]));
+ convert_cpu_topology(Tail, lists:append(Acc, [{thread, tuple_to_list(Value)}]));
convert_cpu_topology([{node, Value}| Tail], Acc) ->
- convert_cpu_topology(Tail, lists:append(Acc, [{node, convert_cpu_topology(Value, [])}]));
+ convert_cpu_topology(Tail, lists:append(Acc, [{node, convert_cpu_topology(Value, [])}]));
convert_cpu_topology([{processor, Value}| Tail], Acc) ->
- convert_cpu_topology(Tail, lists:append(Acc, [{processor, convert_cpu_topology(Value, [])}]));
+ convert_cpu_topology(Tail, lists:append(Acc, [{processor, convert_cpu_topology(Value, [])}]));
+convert_cpu_topology({Key, Value}, _) ->
+ [{Key, Value}];
convert_cpu_topology([], Acc) ->
- Acc.
+ Acc.
get_process_info(Pid) ->
Info = [process_info(Pid, Key) || Key <- ?PROCESS_INFO],
diff --git a/test/cpu_topo_data b/test/cpu_topo_data
new file mode 100644
index 0000000..1ec265a
--- /dev/null
+++ b/test/cpu_topo_data
@@ -0,0 +1,137 @@
+[
+%% Intel(R) Xeon(R) CPU X5647 @ 2.93GHz (Dual CPU)
+
+[{node,[{processor,[{core,[{thread,{logical,1}},
+ {thread,{logical,9}}]},
+ {core,[{thread,{logical,3}},{thread,{logical,11}}]},
+ {core,[{thread,{logical,5}},{thread,{logical,13}}]},
+ {core,[{thread,{logical,7}},{thread,{logical,15}}]}]}]},
+ {node,[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,8}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,10}}]},
+ {core,[{thread,{logical,4}},{thread,{logical,12}}]},
+ {core,[{thread,{logical,6}},{thread,{logical,14}}]}]}]}],
+
+%% Intel(R) Xeon(R) CPU X5647 @ 2.93GHz
+
+[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,4}}]},
+ {core,[{thread,{logical,1}},{thread,{logical,5}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,6}}]},
+ {core,[{thread,{logical,3}},{thread,{logical,7}}]}]}],
+
+%% Intel Core 2 Quad Q8300 LGA775 'Yorkfield' 2.5GHz 4MB-cache (1333FSB) Processor
+
+[{processor,[{core,{logical,0}},
+ {core,{logical,1}},
+ {core,{logical,2}},
+ {core,{logical,3}}]}],
+
+%% Intel® Xeon® Six Core E5-2620 (2.0 GHz, 7.20 GT/s, 15M L3 Cache)
+
+[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,6}}]},
+ {core,[{thread,{logical,1}},{thread,{logical,7}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,8}}]},
+ {core,[{thread,{logical,3}},{thread,{logical,9}}]},
+ {core,[{thread,{logical,4}},{thread,{logical,10}}]},
+ {core,[{thread,{logical,5}},{thread,{logical,11}}]}]}],
+
+%% OSX
+
+undefined,
+
+%% Intel(R) Xeon(R) CPU W3530 @ 2.80GHz
+
+[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,4}}]},
+ {core,[{thread,{logical,1}},{thread,{logical,5}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,6}}]},
+ {core,[{thread,{logical,3}},{thread,{logical,7}}]}]}],
+
+%% Intel(R) Atom(TM) CPU N2800 @ 1.86GHz (8GB memory)
+
+[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,1}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,3}}]}]}],
+
+%% Intel(R) Core(TM) i7-2675QM CPU @ 2.20GHz
+
+[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,4}}]},
+ {core,[{thread,{logical,1}},{thread,{logical,5}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,6}}]},
+ {core,[{thread,{logical,3}},{thread,{logical,7}}]}]}],
+
+%% Intel(R) Xeon(R) CPU L3426 @ 1.87GHz
+
+[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,4}}]},
+ {core,[{thread,{logical,1}},{thread,{logical,5}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,6}}]},
+ {core,[{thread,{logical,3}},{thread,{logical,7}}]}]}],
+
+%% Intel(R) Core(TM) i7 CPU M 640 @ 2.80GHz (in a vmware fusion vm, one cpu enabled)
+
+[{processor,{logical,0}}],
+
+%% Intel(R) Core(TM) i7 CPU M 640 @ 2.80GHz (in a vmware fusion vm, two cpus enabled)
+
+[{processor,{logical,0}},{processor,{logical,1}}],
+
+%% Intel(R) Core(TM) i7 CPU M 640 @ 2.80GHz (in a vmware fusion vm, four cpus enabled)
+
+[{processor,{logical,0}},
+ {processor,{logical,1}},
+ {processor,{logical,2}},
+ {processor,{logical,3}}],
+
+%% Intel(R) Xeon(R) CPU E5620 @ 2.40GHz (EC2 hi1.4xlarge)
+
+[{processor,[{thread,{logical,0}},
+ {thread,{logical,1}},
+ {thread,{logical,2}},
+ {thread,{logical,3}},
+ {thread,{logical,4}},
+ {thread,{logical,5}},
+ {thread,{logical,6}},
+ {thread,{logical,7}},
+ {thread,{logical,8}},
+ {thread,{logical,9}},
+ {thread,{logical,10}},
+ {thread,{logical,11}},
+ {thread,{logical,12}},
+ {thread,{logical,13}},
+ {thread,{logical,14}},
+ {thread,{logical,15}}]}],
+
+%% Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz (EC2 cc2.8xlarge)
+
+[{processor,[{core,[{thread,{logical,0}},
+ {thread,{logical,16}}]},
+ {core,[{thread,{logical,1}},{thread,{logical,17}}]},
+ {core,[{thread,{logical,2}},{thread,{logical,18}}]},
+ {core,[{thread,{logical,3}},{thread,{logical,19}}]},
+ {core,[{thread,{logical,4}},{thread,{logical,20}}]},
+ {core,[{thread,{logical,5}},{thread,{logical,21}}]},
+ {core,[{thread,{logical,6}},{thread,{logical,22}}]},
+ {core,[{thread,{logical,7}},{thread,{logical,23}}]}]},
+ {processor,[{core,[{thread,{logical,8}},
+ {thread,{logical,24}}]},
+ {core,[{thread,{logical,9}},{thread,{logical,25}}]},
+ {core,[{thread,{logical,10}},{thread,{logical,26}}]},
+ {core,[{thread,{logical,11}},{thread,{logical,27}}]},
+ {core,[{thread,{logical,12}},{thread,{logical,28}}]},
+ {core,[{thread,{logical,13}},{thread,{logical,29}}]},
+ {core,[{thread,{logical,14}},{thread,{logical,30}}]},
+ {core,[{thread,{logical,15}},{thread,{logical,31}}]}]}],
+
+%% Unknown
+
+[{processor,{logical,0}},{processor,{logical,1}}],
+
+%% single-core ppc32
+
+{logical,0}
+
+].
diff --git a/test/folsom_erlang_checks.erl b/test/folsom_erlang_checks.erl
index fe17a16..dc580a9 100644
--- a/test/folsom_erlang_checks.erl
+++ b/test/folsom_erlang_checks.erl
@@ -33,7 +33,8 @@
delete_metrics/0,
vm_metrics/0,
counter_metric/2,
- cpu_topology/0
+ cpu_topology/0,
+ c_compiler_used/0
]).
-define(DATA, [0, 1, 5, 10, 100, 200, 500, 750, 1000, 2000, 5000]).
@@ -56,6 +57,7 @@
ok = folsom_metrics:new_histogram(nonea, none, 5000),
ok = folsom_metrics:new_histogram(timed, none, 5000),
+ ok = folsom_metrics:new_histogram(timed2, none, 5000),
ok = folsom_metrics:new_history(<<"history">>),
ok = folsom_metrics:new_meter(meter),
@@ -67,7 +69,7 @@
ok = folsom_metrics:new_spiral(spiral),
?debugFmt("ensuring meter tick is registered with gen_server~n", []),
- ok = ensure_meter_tick_exists(),
+ ok = ensure_meter_tick_exists(2),
?debugFmt("ensuring multiple timer registrations dont cause issues", []),
ok = folsom_meter_timer_server:register(meter, folsom_metrics_meter),
@@ -106,6 +108,9 @@
3.141592653589793 = folsom_metrics:histogram_timed_update(timed, math, pi, []),
+ Begin = folsom_metrics:histogram_timed_begin(timed2),
+ folsom_metrics:histogram_timed_notify(Begin),
+
PopulateDuration = fun() ->
ok = folsom_metrics:notify_existing_metric(duration, timer_start, duration),
timer:sleep(10),
@@ -180,6 +185,9 @@
List = folsom_metrics:get_metric_value(timed),
?debugFmt("timed update value: ~p", [List]),
+ List2 = folsom_metrics:get_metric_value(timed2),
+ ?debugFmt("timed update value begin/end: ~p", [List2]),
+
1 = length(folsom_metrics:get_metric_value(<<"history">>)),
1 = length(folsom_metrics:get_metric_value(historya)),
@@ -192,12 +200,18 @@
_ ->
error
end,
+ ok = case proplists:get_value(day, Meter) of
+ Value1 when Value1 > 0.005 ->
+ ok;
+ _ ->
+ error
+ end,
?debugFmt("checking meter reader~n", []),
MeterReader = folsom_metrics:get_metric_value(meter_reader),
?debugFmt("~p~n", [MeterReader]),
ok = case proplists:get_value(one, MeterReader) of
- Value1 when Value1 > 1 ->
+ Value2 when Value2 > 1 ->
ok;
_ ->
error
@@ -228,8 +242,11 @@
ok = folsom_metrics:delete_metric(nonea),
ok = folsom_metrics:delete_metric(timed),
+ ok = folsom_metrics:delete_metric(timed2),
ok = folsom_metrics:delete_metric(testcounter),
+ ok = ensure_meter_tick_exists(2),
+
1 = length(ets:tab2list(?METER_TABLE)),
ok = folsom_metrics:delete_metric(meter),
0 = length(ets:tab2list(?METER_TABLE)),
@@ -238,6 +255,8 @@
ok = folsom_metrics:delete_metric(meter_reader),
0 = length(ets:tab2list(?METER_READER_TABLE)),
+ ok = ensure_meter_tick_exists(0),
+
ok = folsom_metrics:delete_metric(duration),
ok = folsom_metrics:delete_metric(spiral),
@@ -269,9 +288,9 @@
0 = Result.
-ensure_meter_tick_exists() ->
+ensure_meter_tick_exists(MeterCnt) ->
{state, State} = folsom_meter_timer_server:dump(),
- 2 = length(State),
+ MeterCnt = length(State),
ok.
%% internal function
@@ -349,25 +368,29 @@
for(N, LoopCount + 1, Counter).
cpu_topology() ->
- Test = [{node,[{processor,[{core,[{thread,{logical,1}},{thread,{logical,9}}]},
- {core,[{thread,{logical,3}},{thread,{logical,11}}]},
- {core,[{thread,{logical,5}},{thread,{logical,13}}]},
- {core,[{thread,{logical,7}},{thread,{logical,15}}]}]}]},
- {node,[{processor,[{core,[{thread,{logical,0}},{thread,{logical,8}}]},
- {core,[{thread,{logical,2}},{thread,{logical,10}}]},
- {core,[{thread,{logical,4}},{thread,{logical,12}}]},
- {core,[{thread,{logical,6}},{thread,{logical,14}}]}]}]}],
+ ?debugFmt("Testing various CPU topologies ...~n", []),
+ {ok, [Data]} = file:consult("../test/cpu_topo_data"),
+ [run_convert_and_jsonify(Item) || Item <- Data].
- ExpectedResult = [{node,[{processor,[{core,[{thread,[logical,1]},{thread,[logical,9]}]},
- {core,[{thread,[logical,3]},{thread,[logical,11]}]},
- {core,[{thread,[logical,5]},{thread,[logical,13]}]},
- {core,[{thread,[logical,7]},{thread,[logical,15]}]}]}]},
- {node,[{processor,[{core,[{thread,[logical,0]},{thread,[logical,8]}]},
- {core,[{thread,[logical,2]},{thread,[logical,10]}]},
- {core,[{thread,[logical,4]},{thread,[logical,12]}]},
- {core,[{thread,[logical,6]},{thread,[logical,14]}]}]}]}],
- ExpectedResult = folsom_vm_metrics:convert_cpu_topology(Test, []).
+run_convert_and_jsonify(Item) ->
+ ?debugFmt("Converting ... ~n~p~n", [Item]),
+ Result = folsom_vm_metrics:convert_system_info({cpu_topology, Item}),
+ %?debugFmt("~p~n", [mochijson2:encode(Result)]).
+ mochijson2:encode(Result).
+
+c_compiler_used() ->
+ Test = [{gnuc, {4,4,5}},
+ {gnuc, {4,4}},
+ {msc, 1600}],
+
+ Expected = [[{compiler, gnuc}, {version, <<"4.4.5">>}],
+ [{compiler, gnuc}, {version, <<"4.4">>}],
+ [{compiler, msc}, {version, <<"1600">>}]],
+
+ ?assertEqual(Expected, [folsom_vm_metrics:convert_system_info({c_compiler_used, {Compiler, Version}})
+ || {Compiler, Version} <- Test]).
+
duration_check(Duration) ->
[?assert(lists:keymember(Key, 1, Duration)) || Key <-
diff --git a/test/folsom_tests.erl b/test/folsom_tests.erl
index 04f0ba8..04f2d94 100644
--- a/test/folsom_tests.erl
+++ b/test/folsom_tests.erl
@@ -45,4 +45,55 @@
{"deleting metrics",
fun folsom_erlang_checks:delete_metrics/0},
{"cpu topology test",
- fun folsom_erlang_checks:cpu_topology/0}]}.
+ fun folsom_erlang_checks:cpu_topology/0},
+ {"c compiler test",
+ fun folsom_erlang_checks:c_compiler_used/0}]}.
+
+configure_test_() ->
+ {foreach, fun setup_app/0, fun cleanup_app/1,
+ [{"start with configured metrics",
+ fun() ->
+ ?assertMatch(ok, application:start(folsom)),
+ [counter, slide, <<"gauge">>, <<"uniform">>] =
+ lists:sort(folsom_metrics:get_metrics())
+ end}]}.
+
+setup_app() ->
+ application:unload(folsom),
+ Env = [{counter, counter},
+ {gauge, <<"gauge">>},
+ {histogram, [[<<"uniform">>, uniform, 5000],
+ [slide, slide_uniform, {60, 1028}]]}],
+ application:load({application, folsom, [{mod, {folsom, []}}, {env, Env}]}),
+ ok.
+
+cleanup_app(ok) ->
+ lists:foreach(fun folsom_metrics:delete_metric/1,
+ [counter, slide, <<"gauge">>, <<"uniform">>]),
+ application:stop(folsom),
+ application:unload(folsom),
+ ok.
+
+update_counter_test() ->
+ Tid = ets:new(sometable, [public, set]),
+ Workers = [spawn_monitor(fun() -> timer:sleep(100-N), folsom_utils:update_counter(Tid, hello, N) end) || N <- lists:seq(1, 100)],
+ wait_for_results(Workers),
+ ?assertEqual([{hello, 5050}], ets:lookup(Tid, hello)).
+
+wait_for_results([]) ->
+ ok;
+wait_for_results(Workers) ->
+ receive
+ {'DOWN', _, _, Pid, Reason} ->
+ case lists:keyfind(Pid, 1, Workers) of
+ false ->
+ wait_for_results(Workers);
+ _ ->
+ case Reason of
+ normal ->
+ wait_for_results(lists:keydelete(Pid, 1, Workers));
+ _ ->
+ erlang:error(Reason)
+ end
+ end
+ end.
diff --git a/test/mochijson2.erl b/test/mochijson2.erl
new file mode 100644
index 0000000..954a07d
--- /dev/null
+++ b/test/mochijson2.erl
@@ -0,0 +1,849 @@
+%% @author Bob Ippolito <bob@mochimedia.com>
+%% @copyright 2007 Mochi Media, Inc.
+
+%% @doc Yet another JSON (RFC 4627) library for Erlang. mochijson2 works
+%% with binaries as strings, arrays as lists (without an {array, _})
+%% wrapper and it only knows how to decode UTF-8 (and ASCII).
+%%
+%% JSON terms are decoded as follows (javascript -> erlang):
+%% <ul>
+%% <li>{"key": "value"} ->
+%% {struct, [{<<"key">>, <<"value">>}]}</li>
+%% <li>["array", 123, 12.34, true, false, null] ->
+%% [<<"array">>, 123, 12.34, true, false, null]
+%% </li>
+%% </ul>
+%% <ul>
+%% <li>Strings in JSON decode to UTF-8 binaries in Erlang</li>
+%% <li>Objects decode to {struct, PropList}</li>
+%% <li>Numbers decode to integer or float</li>
+%% <li>true, false, null decode to their respective terms.</li>
+%% </ul>
+%% The encoder will accept the same format that the decoder will produce,
+%% but will also allow additional cases for leniency:
+%% <ul>
+%% <li>atoms other than true, false, null will be considered UTF-8
+%% strings (even as a proplist key)
+%% </li>
+%% <li>{json, IoList} will insert IoList directly into the output
+%% with no validation
+%% </li>
+%% <li>{array, Array} will be encoded as Array
+%% (legacy mochijson style)
+%% </li>
+%% <li>A non-empty raw proplist will be encoded as an object as long
+%% as the first pair does not have an atom key of json, struct,
+%% or array
+%% </li>
+%% </ul>
+
+-module(mochijson2).
+-author('bob@mochimedia.com').
+-export([encoder/1, encode/1]).
+-export([decoder/1, decode/1]).
+
+% This is a macro to placate syntax highlighters..
+-define(Q, $\").
+-define(ADV_COL(S, N), S#decoder{offset=N+S#decoder.offset,
+ column=N+S#decoder.column}).
+-define(INC_COL(S), S#decoder{offset=1+S#decoder.offset,
+ column=1+S#decoder.column}).
+-define(INC_LINE(S), S#decoder{offset=1+S#decoder.offset,
+ column=1,
+ line=1+S#decoder.line}).
+-define(INC_CHAR(S, C),
+ case C of
+ $\n ->
+ S#decoder{column=1,
+ line=1+S#decoder.line,
+ offset=1+S#decoder.offset};
+ _ ->
+ S#decoder{column=1+S#decoder.column,
+ offset=1+S#decoder.offset}
+ end).
+-define(IS_WHITESPACE(C),
+ (C =:= $\s orelse C =:= $\t orelse C =:= $\r orelse C =:= $\n)).
+
+%% @type iolist() = [char() | binary() | iolist()]
+%% @type iodata() = iolist() | binary()
+%% @type json_string() = atom | binary()
+%% @type json_number() = integer() | float()
+%% @type json_array() = [json_term()]
+%% @type json_object() = {struct, [{json_string(), json_term()}]}
+%% @type json_iolist() = {json, iolist()}
+%% @type json_term() = json_string() | json_number() | json_array() |
+%% json_object() | json_iolist()
+
+-record(encoder, {handler=null,
+ utf8=false}).
+
+-record(decoder, {object_hook=null,
+ offset=0,
+ line=1,
+ column=1,
+ state=null}).
+
+%% @spec encoder([encoder_option()]) -> function()
+%% @doc Create an encoder/1 with the given options.
+%% @type encoder_option() = handler_option() | utf8_option()
+%% @type utf8_option() = boolean(). Emit unicode as utf8 (default - false)
+encoder(Options) ->
+ State = parse_encoder_options(Options, #encoder{}),
+ fun (O) -> json_encode(O, State) end.
+
+%% @spec encode(json_term()) -> iolist()
+%% @doc Encode the given as JSON to an iolist.
+encode(Any) ->
+ json_encode(Any, #encoder{}).
+
+%% @spec decoder([decoder_option()]) -> function()
+%% @doc Create a decoder/1 with the given options.
+decoder(Options) ->
+ State = parse_decoder_options(Options, #decoder{}),
+ fun (O) -> json_decode(O, State) end.
+
+%% @spec decode(iolist()) -> json_term()
+%% @doc Decode the given iolist to Erlang terms.
+decode(S) ->
+ json_decode(S, #decoder{}).
+
+%% Internal API
+
+parse_encoder_options([], State) ->
+ State;
+parse_encoder_options([{handler, Handler} | Rest], State) ->
+ parse_encoder_options(Rest, State#encoder{handler=Handler});
+parse_encoder_options([{utf8, Switch} | Rest], State) ->
+ parse_encoder_options(Rest, State#encoder{utf8=Switch}).
+
+parse_decoder_options([], State) ->
+ State;
+parse_decoder_options([{object_hook, Hook} | Rest], State) ->
+ parse_decoder_options(Rest, State#decoder{object_hook=Hook}).
+
+json_encode(true, _State) ->
+ <<"true">>;
+json_encode(false, _State) ->
+ <<"false">>;
+json_encode(null, _State) ->
+ <<"null">>;
+json_encode(I, _State) when is_integer(I) ->
+ integer_to_list(I);
+json_encode(F, _State) when is_float(F) ->
+ mochinum:digits(F);
+json_encode(S, State) when is_binary(S); is_atom(S) ->
+ json_encode_string(S, State);
+json_encode([{K, _}|_] = Props, State) when (K =/= struct andalso
+ K =/= array andalso
+ K =/= json) ->
+ json_encode_proplist(Props, State);
+json_encode({struct, Props}, State) when is_list(Props) ->
+ json_encode_proplist(Props, State);
+json_encode(Array, State) when is_list(Array) ->
+ json_encode_array(Array, State);
+json_encode({array, Array}, State) when is_list(Array) ->
+ json_encode_array(Array, State);
+json_encode({json, IoList}, _State) ->
+ IoList;
+json_encode(Bad, #encoder{handler=null}) ->
+ exit({json_encode, {bad_term, Bad}});
+json_encode(Bad, State=#encoder{handler=Handler}) ->
+ json_encode(Handler(Bad), State).
+
+json_encode_array([], _State) ->
+ <<"[]">>;
+json_encode_array(L, State) ->
+ F = fun (O, Acc) ->
+ [$,, json_encode(O, State) | Acc]
+ end,
+ [$, | Acc1] = lists:foldl(F, "[", L),
+ lists:reverse([$\] | Acc1]).
+
+json_encode_proplist([], _State) ->
+ <<"{}">>;
+json_encode_proplist(Props, State) ->
+ F = fun ({K, V}, Acc) ->
+ KS = json_encode_string(K, State),
+ VS = json_encode(V, State),
+ [$,, VS, $:, KS | Acc]
+ end,
+ [$, | Acc1] = lists:foldl(F, "{", Props),
+ lists:reverse([$\} | Acc1]).
+
+json_encode_string(A, State) when is_atom(A) ->
+ L = atom_to_list(A),
+ case json_string_is_safe(L) of
+ true ->
+ [?Q, L, ?Q];
+ false ->
+ json_encode_string_unicode(xmerl_ucs:from_utf8(L), State, [?Q])
+ end;
+json_encode_string(B, State) when is_binary(B) ->
+ case json_bin_is_safe(B) of
+ true ->
+ [?Q, B, ?Q];
+ false ->
+ json_encode_string_unicode(xmerl_ucs:from_utf8(B), State, [?Q])
+ end;
+json_encode_string(I, _State) when is_integer(I) ->
+ [?Q, integer_to_list(I), ?Q];
+json_encode_string(L, State) when is_list(L) ->
+ case json_string_is_safe(L) of
+ true ->
+ [?Q, L, ?Q];
+ false ->
+ json_encode_string_unicode(L, State, [?Q])
+ end.
+
+json_string_is_safe([]) ->
+ true;
+json_string_is_safe([C | Rest]) ->
+ case C of
+ ?Q ->
+ false;
+ $\\ ->
+ false;
+ $\b ->
+ false;
+ $\f ->
+ false;
+ $\n ->
+ false;
+ $\r ->
+ false;
+ $\t ->
+ false;
+ C when C >= 0, C < $\s; C >= 16#7f, C =< 16#10FFFF ->
+ false;
+ C when C < 16#7f ->
+ json_string_is_safe(Rest);
+ _ ->
+ false
+ end.
+
+json_bin_is_safe(<<>>) ->
+ true;
+json_bin_is_safe(<<C, Rest/binary>>) ->
+ case C of
+ ?Q ->
+ false;
+ $\\ ->
+ false;
+ $\b ->
+ false;
+ $\f ->
+ false;
+ $\n ->
+ false;
+ $\r ->
+ false;
+ $\t ->
+ false;
+ C when C >= 0, C < $\s; C >= 16#7f ->
+ false;
+ C when C < 16#7f ->
+ json_bin_is_safe(Rest)
+ end.
+
+json_encode_string_unicode([], _State, Acc) ->
+ lists:reverse([$\" | Acc]);
+json_encode_string_unicode([C | Cs], State, Acc) ->
+ Acc1 = case C of
+ ?Q ->
+ [?Q, $\\ | Acc];
+ %% Escaping solidus is only useful when trying to protect
+ %% against "</script>" injection attacks which are only
+ %% possible when JSON is inserted into a HTML document
+ %% in-line. mochijson2 does not protect you from this, so
+ %% if you do insert directly into HTML then you need to
+ %% uncomment the following case or escape the output of encode.
+ %%
+ %% $/ ->
+ %% [$/, $\\ | Acc];
+ %%
+ $\\ ->
+ [$\\, $\\ | Acc];
+ $\b ->
+ [$b, $\\ | Acc];
+ $\f ->
+ [$f, $\\ | Acc];
+ $\n ->
+ [$n, $\\ | Acc];
+ $\r ->
+ [$r, $\\ | Acc];
+ $\t ->
+ [$t, $\\ | Acc];
+ C when C >= 0, C < $\s ->
+ [unihex(C) | Acc];
+ C when C >= 16#7f, C =< 16#10FFFF, State#encoder.utf8 ->
+ [xmerl_ucs:to_utf8(C) | Acc];
+ C when C >= 16#7f, C =< 16#10FFFF, not State#encoder.utf8 ->
+ [unihex(C) | Acc];
+ C when C < 16#7f ->
+ [C | Acc];
+ _ ->
+ exit({json_encode, {bad_char, C}})
+ end,
+ json_encode_string_unicode(Cs, State, Acc1).
+
+hexdigit(C) when C >= 0, C =< 9 ->
+ C + $0;
+hexdigit(C) when C =< 15 ->
+ C + $a - 10.
+
+unihex(C) when C < 16#10000 ->
+ <<D3:4, D2:4, D1:4, D0:4>> = <<C:16>>,
+ Digits = [hexdigit(D) || D <- [D3, D2, D1, D0]],
+ [$\\, $u | Digits];
+unihex(C) when C =< 16#10FFFF ->
+ N = C - 16#10000,
+ S1 = 16#d800 bor ((N bsr 10) band 16#3ff),
+ S2 = 16#dc00 bor (N band 16#3ff),
+ [unihex(S1), unihex(S2)].
+
+json_decode(L, S) when is_list(L) ->
+ json_decode(iolist_to_binary(L), S);
+json_decode(B, S) ->
+ {Res, S1} = decode1(B, S),
+ {eof, _} = tokenize(B, S1#decoder{state=trim}),
+ Res.
+
+decode1(B, S=#decoder{state=null}) ->
+ case tokenize(B, S#decoder{state=any}) of
+ {{const, C}, S1} ->
+ {C, S1};
+ {start_array, S1} ->
+ decode_array(B, S1);
+ {start_object, S1} ->
+ decode_object(B, S1)
+ end.
+
+make_object(V, #decoder{object_hook=null}) ->
+ V;
+make_object(V, #decoder{object_hook=Hook}) ->
+ Hook(V).
+
+decode_object(B, S) ->
+ decode_object(B, S#decoder{state=key}, []).
+
+decode_object(B, S=#decoder{state=key}, Acc) ->
+ case tokenize(B, S) of
+ {end_object, S1} ->
+ V = make_object({struct, lists:reverse(Acc)}, S1),
+ {V, S1#decoder{state=null}};
+ {{const, K}, S1} ->
+ {colon, S2} = tokenize(B, S1),
+ {V, S3} = decode1(B, S2#decoder{state=null}),
+ decode_object(B, S3#decoder{state=comma}, [{K, V} | Acc])
+ end;
+decode_object(B, S=#decoder{state=comma}, Acc) ->
+ case tokenize(B, S) of
+ {end_object, S1} ->
+ V = make_object({struct, lists:reverse(Acc)}, S1),
+ {V, S1#decoder{state=null}};
+ {comma, S1} ->
+ decode_object(B, S1#decoder{state=key}, Acc)
+ end.
+
+decode_array(B, S) ->
+ decode_array(B, S#decoder{state=any}, []).
+
+decode_array(B, S=#decoder{state=any}, Acc) ->
+ case tokenize(B, S) of
+ {end_array, S1} ->
+ {lists:reverse(Acc), S1#decoder{state=null}};
+ {start_array, S1} ->
+ {Array, S2} = decode_array(B, S1),
+ decode_array(B, S2#decoder{state=comma}, [Array | Acc]);
+ {start_object, S1} ->
+ {Array, S2} = decode_object(B, S1),
+ decode_array(B, S2#decoder{state=comma}, [Array | Acc]);
+ {{const, Const}, S1} ->
+ decode_array(B, S1#decoder{state=comma}, [Const | Acc])
+ end;
+decode_array(B, S=#decoder{state=comma}, Acc) ->
+ case tokenize(B, S) of
+ {end_array, S1} ->
+ {lists:reverse(Acc), S1#decoder{state=null}};
+ {comma, S1} ->
+ decode_array(B, S1#decoder{state=any}, Acc)
+ end.
+
+tokenize_string(B, S=#decoder{offset=O}) ->
+ case tokenize_string_fast(B, O) of
+ {escape, O1} ->
+ Length = O1 - O,
+ S1 = ?ADV_COL(S, Length),
+ <<_:O/binary, Head:Length/binary, _/binary>> = B,
+ tokenize_string(B, S1, lists:reverse(binary_to_list(Head)));
+ O1 ->
+ Length = O1 - O,
+ <<_:O/binary, String:Length/binary, ?Q, _/binary>> = B,
+ {{const, String}, ?ADV_COL(S, Length + 1)}
+ end.
+
+tokenize_string_fast(B, O) ->
+ case B of
+ <<_:O/binary, ?Q, _/binary>> ->
+ O;
+ <<_:O/binary, $\\, _/binary>> ->
+ {escape, O};
+ <<_:O/binary, C1, _/binary>> when C1 < 128 ->
+ tokenize_string_fast(B, 1 + O);
+ <<_:O/binary, C1, C2, _/binary>> when C1 >= 194, C1 =< 223,
+ C2 >= 128, C2 =< 191 ->
+ tokenize_string_fast(B, 2 + O);
+ <<_:O/binary, C1, C2, C3, _/binary>> when C1 >= 224, C1 =< 239,
+ C2 >= 128, C2 =< 191,
+ C3 >= 128, C3 =< 191 ->
+ tokenize_string_fast(B, 3 + O);
+ <<_:O/binary, C1, C2, C3, C4, _/binary>> when C1 >= 240, C1 =< 244,
+ C2 >= 128, C2 =< 191,
+ C3 >= 128, C3 =< 191,
+ C4 >= 128, C4 =< 191 ->
+ tokenize_string_fast(B, 4 + O);
+ _ ->
+ throw(invalid_utf8)
+ end.
+
+tokenize_string(B, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, ?Q, _/binary>> ->
+ {{const, iolist_to_binary(lists:reverse(Acc))}, ?INC_COL(S)};
+ <<_:O/binary, "\\\"", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$\" | Acc]);
+ <<_:O/binary, "\\\\", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$\\ | Acc]);
+ <<_:O/binary, "\\/", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$/ | Acc]);
+ <<_:O/binary, "\\b", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$\b | Acc]);
+ <<_:O/binary, "\\f", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$\f | Acc]);
+ <<_:O/binary, "\\n", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$\n | Acc]);
+ <<_:O/binary, "\\r", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$\r | Acc]);
+ <<_:O/binary, "\\t", _/binary>> ->
+ tokenize_string(B, ?ADV_COL(S, 2), [$\t | Acc]);
+ <<_:O/binary, "\\u", C3, C2, C1, C0, Rest/binary>> ->
+ C = erlang:list_to_integer([C3, C2, C1, C0], 16),
+ if C > 16#D7FF, C < 16#DC00 ->
+ %% coalesce UTF-16 surrogate pair
+ <<"\\u", D3, D2, D1, D0, _/binary>> = Rest,
+ D = erlang:list_to_integer([D3,D2,D1,D0], 16),
+ [CodePoint] = xmerl_ucs:from_utf16be(<<C:16/big-unsigned-integer,
+ D:16/big-unsigned-integer>>),
+ Acc1 = lists:reverse(xmerl_ucs:to_utf8(CodePoint), Acc),
+ tokenize_string(B, ?ADV_COL(S, 12), Acc1);
+ true ->
+ Acc1 = lists:reverse(xmerl_ucs:to_utf8(C), Acc),
+ tokenize_string(B, ?ADV_COL(S, 6), Acc1)
+ end;
+ <<_:O/binary, C1, _/binary>> when C1 < 128 ->
+ tokenize_string(B, ?INC_CHAR(S, C1), [C1 | Acc]);
+ <<_:O/binary, C1, C2, _/binary>> when C1 >= 194, C1 =< 223,
+ C2 >= 128, C2 =< 191 ->
+ tokenize_string(B, ?ADV_COL(S, 2), [C2, C1 | Acc]);
+ <<_:O/binary, C1, C2, C3, _/binary>> when C1 >= 224, C1 =< 239,
+ C2 >= 128, C2 =< 191,
+ C3 >= 128, C3 =< 191 ->
+ tokenize_string(B, ?ADV_COL(S, 3), [C3, C2, C1 | Acc]);
+ <<_:O/binary, C1, C2, C3, C4, _/binary>> when C1 >= 240, C1 =< 244,
+ C2 >= 128, C2 =< 191,
+ C3 >= 128, C3 =< 191,
+ C4 >= 128, C4 =< 191 ->
+ tokenize_string(B, ?ADV_COL(S, 4), [C4, C3, C2, C1 | Acc]);
+ _ ->
+ throw(invalid_utf8)
+ end.
+
+tokenize_number(B, S) ->
+ case tokenize_number(B, sign, S, []) of
+ {{int, Int}, S1} ->
+ {{const, list_to_integer(Int)}, S1};
+ {{float, Float}, S1} ->
+ {{const, list_to_float(Float)}, S1}
+ end.
+
+tokenize_number(B, sign, S=#decoder{offset=O}, []) ->
+ case B of
+ <<_:O/binary, $-, _/binary>> ->
+ tokenize_number(B, int, ?INC_COL(S), [$-]);
+ _ ->
+ tokenize_number(B, int, S, [])
+ end;
+tokenize_number(B, int, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, $0, _/binary>> ->
+ tokenize_number(B, frac, ?INC_COL(S), [$0 | Acc]);
+ <<_:O/binary, C, _/binary>> when C >= $1 andalso C =< $9 ->
+ tokenize_number(B, int1, ?INC_COL(S), [C | Acc])
+ end;
+tokenize_number(B, int1, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 ->
+ tokenize_number(B, int1, ?INC_COL(S), [C | Acc]);
+ _ ->
+ tokenize_number(B, frac, S, Acc)
+ end;
+tokenize_number(B, frac, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, $., C, _/binary>> when C >= $0, C =< $9 ->
+ tokenize_number(B, frac1, ?ADV_COL(S, 2), [C, $. | Acc]);
+ <<_:O/binary, E, _/binary>> when E =:= $e orelse E =:= $E ->
+ tokenize_number(B, esign, ?INC_COL(S), [$e, $0, $. | Acc]);
+ _ ->
+ {{int, lists:reverse(Acc)}, S}
+ end;
+tokenize_number(B, frac1, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 ->
+ tokenize_number(B, frac1, ?INC_COL(S), [C | Acc]);
+ <<_:O/binary, E, _/binary>> when E =:= $e orelse E =:= $E ->
+ tokenize_number(B, esign, ?INC_COL(S), [$e | Acc]);
+ _ ->
+ {{float, lists:reverse(Acc)}, S}
+ end;
+tokenize_number(B, esign, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, C, _/binary>> when C =:= $- orelse C=:= $+ ->
+ tokenize_number(B, eint, ?INC_COL(S), [C | Acc]);
+ _ ->
+ tokenize_number(B, eint, S, Acc)
+ end;
+tokenize_number(B, eint, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 ->
+ tokenize_number(B, eint1, ?INC_COL(S), [C | Acc])
+ end;
+tokenize_number(B, eint1, S=#decoder{offset=O}, Acc) ->
+ case B of
+ <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 ->
+ tokenize_number(B, eint1, ?INC_COL(S), [C | Acc]);
+ _ ->
+ {{float, lists:reverse(Acc)}, S}
+ end.
+
+tokenize(B, S=#decoder{offset=O}) ->
+ case B of
+ <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) ->
+ tokenize(B, ?INC_CHAR(S, C));
+ <<_:O/binary, "{", _/binary>> ->
+ {start_object, ?INC_COL(S)};
+ <<_:O/binary, "}", _/binary>> ->
+ {end_object, ?INC_COL(S)};
+ <<_:O/binary, "[", _/binary>> ->
+ {start_array, ?INC_COL(S)};
+ <<_:O/binary, "]", _/binary>> ->
+ {end_array, ?INC_COL(S)};
+ <<_:O/binary, ",", _/binary>> ->
+ {comma, ?INC_COL(S)};
+ <<_:O/binary, ":", _/binary>> ->
+ {colon, ?INC_COL(S)};
+ <<_:O/binary, "null", _/binary>> ->
+ {{const, null}, ?ADV_COL(S, 4)};
+ <<_:O/binary, "true", _/binary>> ->
+ {{const, true}, ?ADV_COL(S, 4)};
+ <<_:O/binary, "false", _/binary>> ->
+ {{const, false}, ?ADV_COL(S, 5)};
+ <<_:O/binary, "\"", _/binary>> ->
+ tokenize_string(B, ?INC_COL(S));
+ <<_:O/binary, C, _/binary>> when (C >= $0 andalso C =< $9)
+ orelse C =:= $- ->
+ tokenize_number(B, S);
+ <<_:O/binary>> ->
+ trim = S#decoder.state,
+ {eof, S}
+ end.
+%%
+%% Tests
+%%
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+
+%% testing constructs borrowed from the Yaws JSON implementation.
+
+%% Create an object from a list of Key/Value pairs.
+
+obj_new() ->
+ {struct, []}.
+
+is_obj({struct, Props}) ->
+ F = fun ({K, _}) when is_binary(K) -> true end,
+ lists:all(F, Props).
+
+obj_from_list(Props) ->
+ Obj = {struct, Props},
+ ?assert(is_obj(Obj)),
+ Obj.
+
+%% Test for equivalence of Erlang terms.
+%% Due to arbitrary order of construction, equivalent objects might
+%% compare unequal as erlang terms, so we need to carefully recurse
+%% through aggregates (tuples and objects).
+
+equiv({struct, Props1}, {struct, Props2}) ->
+ equiv_object(Props1, Props2);
+equiv(L1, L2) when is_list(L1), is_list(L2) ->
+ equiv_list(L1, L2);
+equiv(N1, N2) when is_number(N1), is_number(N2) -> N1 == N2;
+equiv(B1, B2) when is_binary(B1), is_binary(B2) -> B1 == B2;
+equiv(A, A) when A =:= true orelse A =:= false orelse A =:= null -> true.
+
+%% Object representation and traversal order is unknown.
+%% Use the sledgehammer and sort property lists.
+
+equiv_object(Props1, Props2) ->
+ L1 = lists:keysort(1, Props1),
+ L2 = lists:keysort(1, Props2),
+ Pairs = lists:zip(L1, L2),
+ true = lists:all(fun({{K1, V1}, {K2, V2}}) ->
+ equiv(K1, K2) and equiv(V1, V2)
+ end, Pairs).
+
+%% Recursively compare tuple elements for equivalence.
+
+equiv_list([], []) ->
+ true;
+equiv_list([V1 | L1], [V2 | L2]) ->
+ equiv(V1, V2) andalso equiv_list(L1, L2).
+
+decode_test() ->
+ [1199344435545.0, 1] = decode(<<"[1199344435545.0,1]">>),
+ <<16#F0,16#9D,16#9C,16#95>> = decode([34,"\\ud835","\\udf15",34]).
+
+e2j_vec_test() ->
+ test_one(e2j_test_vec(utf8), 1).
+
+test_one([], _N) ->
+ %% io:format("~p tests passed~n", [N-1]),
+ ok;
+test_one([{E, J} | Rest], N) ->
+ %% io:format("[~p] ~p ~p~n", [N, E, J]),
+ true = equiv(E, decode(J)),
+ true = equiv(E, decode(encode(E))),
+ test_one(Rest, 1+N).
+
+e2j_test_vec(utf8) ->
+ [
+ {1, "1"},
+ {3.1416, "3.14160"}, %% text representation may truncate, trail zeroes
+ {-1, "-1"},
+ {-3.1416, "-3.14160"},
+ {12.0e10, "1.20000e+11"},
+ {1.234E+10, "1.23400e+10"},
+ {-1.234E-10, "-1.23400e-10"},
+ {10.0, "1.0e+01"},
+ {123.456, "1.23456E+2"},
+ {10.0, "1e1"},
+ {<<"foo">>, "\"foo\""},
+ {<<"foo", 5, "bar">>, "\"foo\\u0005bar\""},
+ {<<"">>, "\"\""},
+ {<<"\n\n\n">>, "\"\\n\\n\\n\""},
+ {<<"\" \b\f\r\n\t\"">>, "\"\\\" \\b\\f\\r\\n\\t\\\"\""},
+ {obj_new(), "{}"},
+ {obj_from_list([{<<"foo">>, <<"bar">>}]), "{\"foo\":\"bar\"}"},
+ {obj_from_list([{<<"foo">>, <<"bar">>}, {<<"baz">>, 123}]),
+ "{\"foo\":\"bar\",\"baz\":123}"},
+ {[], "[]"},
+ {[[]], "[[]]"},
+ {[1, <<"foo">>], "[1,\"foo\"]"},
+
+ %% json array in a json object
+ {obj_from_list([{<<"foo">>, [123]}]),
+ "{\"foo\":[123]}"},
+
+ %% json object in a json object
+ {obj_from_list([{<<"foo">>, obj_from_list([{<<"bar">>, true}])}]),
+ "{\"foo\":{\"bar\":true}}"},
+
+ %% fold evaluation order
+ {obj_from_list([{<<"foo">>, []},
+ {<<"bar">>, obj_from_list([{<<"baz">>, true}])},
+ {<<"alice">>, <<"bob">>}]),
+ "{\"foo\":[],\"bar\":{\"baz\":true},\"alice\":\"bob\"}"},
+
+ %% json object in a json array
+ {[-123, <<"foo">>, obj_from_list([{<<"bar">>, []}]), null],
+ "[-123,\"foo\",{\"bar\":[]},null]"}
+ ].
+
+%% test utf8 encoding
+encoder_utf8_test() ->
+ %% safe conversion case (default)
+ [34,"\\u0001","\\u0442","\\u0435","\\u0441","\\u0442",34] =
+ encode(<<1,"\321\202\320\265\321\201\321\202">>),
+
+ %% raw utf8 output (optional)
+ Enc = mochijson2:encoder([{utf8, true}]),
+ [34,"\\u0001",[209,130],[208,181],[209,129],[209,130],34] =
+ Enc(<<1,"\321\202\320\265\321\201\321\202">>).
+
+input_validation_test() ->
+ Good = [
+ {16#00A3, <<?Q, 16#C2, 16#A3, ?Q>>}, %% pound
+ {16#20AC, <<?Q, 16#E2, 16#82, 16#AC, ?Q>>}, %% euro
+ {16#10196, <<?Q, 16#F0, 16#90, 16#86, 16#96, ?Q>>} %% denarius
+ ],
+ lists:foreach(fun({CodePoint, UTF8}) ->
+ Expect = list_to_binary(xmerl_ucs:to_utf8(CodePoint)),
+ Expect = decode(UTF8)
+ end, Good),
+
+ Bad = [
+ %% 2nd, 3rd, or 4th byte of a multi-byte sequence w/o leading byte
+ <<?Q, 16#80, ?Q>>,
+ %% missing continuations, last byte in each should be 80-BF
+ <<?Q, 16#C2, 16#7F, ?Q>>,
+ <<?Q, 16#E0, 16#80,16#7F, ?Q>>,
+ <<?Q, 16#F0, 16#80, 16#80, 16#7F, ?Q>>,
+ %% we don't support code points > 10FFFF per RFC 3629
+ <<?Q, 16#F5, 16#80, 16#80, 16#80, ?Q>>,
+ %% escape characters trigger a different code path
+ <<?Q, $\\, $\n, 16#80, ?Q>>
+ ],
+ lists:foreach(
+ fun(X) ->
+ ok = try decode(X) catch invalid_utf8 -> ok end,
+ %% could be {ucs,{bad_utf8_character_code}} or
+ %% {json_encode,{bad_char,_}}
+ {'EXIT', _} = (catch encode(X))
+ end, Bad).
+
+inline_json_test() ->
+ ?assertEqual(<<"\"iodata iodata\"">>,
+ iolist_to_binary(
+ encode({json, [<<"\"iodata">>, " iodata\""]}))),
+ ?assertEqual({struct, [{<<"key">>, <<"iodata iodata">>}]},
+ decode(
+ encode({struct,
+ [{key, {json, [<<"\"iodata">>, " iodata\""]}}]}))),
+ ok.
+
+big_unicode_test() ->
+ UTF8Seq = list_to_binary(xmerl_ucs:to_utf8(16#0001d120)),
+ ?assertEqual(
+ <<"\"\\ud834\\udd20\"">>,
+ iolist_to_binary(encode(UTF8Seq))),
+ ?assertEqual(
+ UTF8Seq,
+ decode(iolist_to_binary(encode(UTF8Seq)))),
+ ok.
+
+custom_decoder_test() ->
+ ?assertEqual(
+ {struct, [{<<"key">>, <<"value">>}]},
+ (decoder([]))("{\"key\": \"value\"}")),
+ F = fun ({struct, [{<<"key">>, <<"value">>}]}) -> win end,
+ ?assertEqual(
+ win,
+ (decoder([{object_hook, F}]))("{\"key\": \"value\"}")),
+ ok.
+
+atom_test() ->
+ %% JSON native atoms
+ [begin
+ ?assertEqual(A, decode(atom_to_list(A))),
+ ?assertEqual(iolist_to_binary(atom_to_list(A)),
+ iolist_to_binary(encode(A)))
+ end || A <- [true, false, null]],
+ %% Atom to string
+ ?assertEqual(
+ <<"\"foo\"">>,
+ iolist_to_binary(encode(foo))),
+ ?assertEqual(
+ <<"\"\\ud834\\udd20\"">>,
+ iolist_to_binary(encode(list_to_atom(xmerl_ucs:to_utf8(16#0001d120))))),
+ ok.
+
+key_encode_test() ->
+ %% Some forms are accepted as keys that would not be strings in other
+ %% cases
+ ?assertEqual(
+ <<"{\"foo\":1}">>,
+ iolist_to_binary(encode({struct, [{foo, 1}]}))),
+ ?assertEqual(
+ <<"{\"foo\":1}">>,
+ iolist_to_binary(encode({struct, [{<<"foo">>, 1}]}))),
+ ?assertEqual(
+ <<"{\"foo\":1}">>,
+ iolist_to_binary(encode({struct, [{"foo", 1}]}))),
+ ?assertEqual(
+ <<"{\"foo\":1}">>,
+ iolist_to_binary(encode([{foo, 1}]))),
+ ?assertEqual(
+ <<"{\"foo\":1}">>,
+ iolist_to_binary(encode([{<<"foo">>, 1}]))),
+ ?assertEqual(
+ <<"{\"foo\":1}">>,
+ iolist_to_binary(encode([{"foo", 1}]))),
+ ?assertEqual(
+ <<"{\"\\ud834\\udd20\":1}">>,
+ iolist_to_binary(
+ encode({struct, [{[16#0001d120], 1}]}))),
+ ?assertEqual(
+ <<"{\"1\":1}">>,
+ iolist_to_binary(encode({struct, [{1, 1}]}))),
+ ok.
+
+unsafe_chars_test() ->
+ Chars = "\"\\\b\f\n\r\t",
+ [begin
+ ?assertEqual(false, json_string_is_safe([C])),
+ ?assertEqual(false, json_bin_is_safe(<<C>>)),
+ ?assertEqual(<<C>>, decode(encode(<<C>>)))
+ end || C <- Chars],
+ ?assertEqual(
+ false,
+ json_string_is_safe([16#0001d120])),
+ ?assertEqual(
+ false,
+ json_bin_is_safe(list_to_binary(xmerl_ucs:to_utf8(16#0001d120)))),
+ ?assertEqual(
+ [16#0001d120],
+ xmerl_ucs:from_utf8(
+ binary_to_list(
+ decode(encode(list_to_atom(xmerl_ucs:to_utf8(16#0001d120))))))),
+ ?assertEqual(
+ false,
+ json_string_is_safe([16#110000])),
+ ?assertEqual(
+ false,
+ json_bin_is_safe(list_to_binary(xmerl_ucs:to_utf8([16#110000])))),
+ %% solidus can be escaped but isn't unsafe by default
+ ?assertEqual(
+ <<"/">>,
+ decode(<<"\"\\/\"">>)),
+ ok.
+
+int_test() ->
+ ?assertEqual(0, decode("0")),
+ ?assertEqual(1, decode("1")),
+ ?assertEqual(11, decode("11")),
+ ok.
+
+large_int_test() ->
+ ?assertEqual(<<"-2147483649214748364921474836492147483649">>,
+ iolist_to_binary(encode(-2147483649214748364921474836492147483649))),
+ ?assertEqual(<<"2147483649214748364921474836492147483649">>,
+ iolist_to_binary(encode(2147483649214748364921474836492147483649))),
+ ok.
+
+float_test() ->
+ ?assertEqual(<<"-2147483649.0">>, iolist_to_binary(encode(-2147483649.0))),
+ ?assertEqual(<<"2147483648.0">>, iolist_to_binary(encode(2147483648.0))),
+ ok.
+
+handler_test() ->
+ ?assertEqual(
+ {'EXIT',{json_encode,{bad_term,{}}}},
+ catch encode({})),
+ F = fun ({}) -> [] end,
+ ?assertEqual(
+ <<"[]">>,
+ iolist_to_binary((encoder([{handler, F}]))({}))),
+ ok.
+
+-endif.
diff --git a/test/mochinum.erl b/test/mochinum.erl
new file mode 100644
index 0000000..c52b15c
--- /dev/null
+++ b/test/mochinum.erl
@@ -0,0 +1,354 @@
+%% @copyright 2007 Mochi Media, Inc.
+%% @author Bob Ippolito <bob@mochimedia.com>
+
+%% @doc Useful numeric algorithms for floats that cover some deficiencies
+%% in the math module. More interesting is digits/1, which implements
+%% the algorithm from:
+%% http://www.cs.indiana.edu/~burger/fp/index.html
+%% See also "Printing Floating-Point Numbers Quickly and Accurately"
+%% in Proceedings of the SIGPLAN '96 Conference on Programming Language
+%% Design and Implementation.
+
+-module(mochinum).
+-author("Bob Ippolito <bob@mochimedia.com>").
+-export([digits/1, frexp/1, int_pow/2, int_ceil/1]).
+
+%% IEEE 754 Float exponent bias
+-define(FLOAT_BIAS, 1022).
+-define(MIN_EXP, -1074).
+-define(BIG_POW, 4503599627370496).
+
+%% External API
+
+%% @spec digits(number()) -> string()
+%% @doc Returns a string that accurately represents the given integer or float
+%% using a conservative amount of digits. Great for generating
+%% human-readable output, or compact ASCII serializations for floats.
+digits(N) when is_integer(N) ->
+ integer_to_list(N);
+digits(0.0) ->
+ "0.0";
+digits(Float) ->
+ {Frac1, Exp1} = frexp_int(Float),
+ [Place0 | Digits0] = digits1(Float, Exp1, Frac1),
+ {Place, Digits} = transform_digits(Place0, Digits0),
+ R = insert_decimal(Place, Digits),
+ case Float < 0 of
+ true ->
+ [$- | R];
+ _ ->
+ R
+ end.
+
+%% @spec frexp(F::float()) -> {Frac::float(), Exp::float()}
+%% @doc Return the fractional and exponent part of an IEEE 754 double,
+%% equivalent to the libc function of the same name.
+%% F = Frac * pow(2, Exp).
+frexp(F) ->
+ frexp1(unpack(F)).
+
+%% @spec int_pow(X::integer(), N::integer()) -> Y::integer()
+%% @doc Moderately efficient way to exponentiate integers.
+%% int_pow(10, 2) = 100.
+int_pow(_X, 0) ->
+ 1;
+int_pow(X, N) when N > 0 ->
+ int_pow(X, N, 1).
+
+%% @spec int_ceil(F::float()) -> integer()
+%% @doc Return the ceiling of F as an integer. The ceiling is defined as
+%% F when F == trunc(F);
+%% trunc(F) when F < 0;
+%% trunc(F) + 1 when F > 0.
+int_ceil(X) ->
+ T = trunc(X),
+ case (X - T) of
+ Pos when Pos > 0 -> T + 1;
+ _ -> T
+ end.
+
+
+%% Internal API
+
+int_pow(X, N, R) when N < 2 ->
+ R * X;
+int_pow(X, N, R) ->
+ int_pow(X * X, N bsr 1, case N band 1 of 1 -> R * X; 0 -> R end).
+
+insert_decimal(0, S) ->
+ "0." ++ S;
+insert_decimal(Place, S) when Place > 0 ->
+ L = length(S),
+ case Place - L of
+ 0 ->
+ S ++ ".0";
+ N when N < 0 ->
+ {S0, S1} = lists:split(L + N, S),
+ S0 ++ "." ++ S1;
+ N when N < 6 ->
+ %% More places than digits
+ S ++ lists:duplicate(N, $0) ++ ".0";
+ _ ->
+ insert_decimal_exp(Place, S)
+ end;
+insert_decimal(Place, S) when Place > -6 ->
+ "0." ++ lists:duplicate(abs(Place), $0) ++ S;
+insert_decimal(Place, S) ->
+ insert_decimal_exp(Place, S).
+
+insert_decimal_exp(Place, S) ->
+ [C | S0] = S,
+ S1 = case S0 of
+ [] ->
+ "0";
+ _ ->
+ S0
+ end,
+ Exp = case Place < 0 of
+ true ->
+ "e-";
+ false ->
+ "e+"
+ end,
+ [C] ++ "." ++ S1 ++ Exp ++ integer_to_list(abs(Place - 1)).
+
+
+digits1(Float, Exp, Frac) ->
+ Round = ((Frac band 1) =:= 0),
+ case Exp >= 0 of
+ true ->
+ BExp = 1 bsl Exp,
+ case (Frac =/= ?BIG_POW) of
+ true ->
+ scale((Frac * BExp * 2), 2, BExp, BExp,
+ Round, Round, Float);
+ false ->
+ scale((Frac * BExp * 4), 4, (BExp * 2), BExp,
+ Round, Round, Float)
+ end;
+ false ->
+ case (Exp =:= ?MIN_EXP) orelse (Frac =/= ?BIG_POW) of
+ true ->
+ scale((Frac * 2), 1 bsl (1 - Exp), 1, 1,
+ Round, Round, Float);
+ false ->
+ scale((Frac * 4), 1 bsl (2 - Exp), 2, 1,
+ Round, Round, Float)
+ end
+ end.
+
+scale(R, S, MPlus, MMinus, LowOk, HighOk, Float) ->
+ Est = int_ceil(math:log10(abs(Float)) - 1.0e-10),
+ %% Note that the scheme implementation uses a 326 element look-up table
+ %% for int_pow(10, N) where we do not.
+ case Est >= 0 of
+ true ->
+ fixup(R, S * int_pow(10, Est), MPlus, MMinus, Est,
+ LowOk, HighOk);
+ false ->
+ Scale = int_pow(10, -Est),
+ fixup(R * Scale, S, MPlus * Scale, MMinus * Scale, Est,
+ LowOk, HighOk)
+ end.
+
+fixup(R, S, MPlus, MMinus, K, LowOk, HighOk) ->
+ TooLow = case HighOk of
+ true ->
+ (R + MPlus) >= S;
+ false ->
+ (R + MPlus) > S
+ end,
+ case TooLow of
+ true ->
+ [(K + 1) | generate(R, S, MPlus, MMinus, LowOk, HighOk)];
+ false ->
+ [K | generate(R * 10, S, MPlus * 10, MMinus * 10, LowOk, HighOk)]
+ end.
+
+generate(R0, S, MPlus, MMinus, LowOk, HighOk) ->
+ D = R0 div S,
+ R = R0 rem S,
+ TC1 = case LowOk of
+ true ->
+ R =< MMinus;
+ false ->
+ R < MMinus
+ end,
+ TC2 = case HighOk of
+ true ->
+ (R + MPlus) >= S;
+ false ->
+ (R + MPlus) > S
+ end,
+ case TC1 of
+ false ->
+ case TC2 of
+ false ->
+ [D | generate(R * 10, S, MPlus * 10, MMinus * 10,
+ LowOk, HighOk)];
+ true ->
+ [D + 1]
+ end;
+ true ->
+ case TC2 of
+ false ->
+ [D];
+ true ->
+ case R * 2 < S of
+ true ->
+ [D];
+ false ->
+ [D + 1]
+ end
+ end
+ end.
+
+unpack(Float) ->
+ <<Sign:1, Exp:11, Frac:52>> = <<Float:64/float>>,
+ {Sign, Exp, Frac}.
+
+frexp1({_Sign, 0, 0}) ->
+ {0.0, 0};
+frexp1({Sign, 0, Frac}) ->
+ Exp = log2floor(Frac),
+ <<Frac1:64/float>> = <<Sign:1, ?FLOAT_BIAS:11, (Frac-1):52>>,
+ {Frac1, -(?FLOAT_BIAS) - 52 + Exp};
+frexp1({Sign, Exp, Frac}) ->
+ <<Frac1:64/float>> = <<Sign:1, ?FLOAT_BIAS:11, Frac:52>>,
+ {Frac1, Exp - ?FLOAT_BIAS}.
+
+log2floor(Int) ->
+ log2floor(Int, 0).
+
+log2floor(0, N) ->
+ N;
+log2floor(Int, N) ->
+ log2floor(Int bsr 1, 1 + N).
+
+
+transform_digits(Place, [0 | Rest]) ->
+ transform_digits(Place, Rest);
+transform_digits(Place, Digits) ->
+ {Place, [$0 + D || D <- Digits]}.
+
+
+frexp_int(F) ->
+ case unpack(F) of
+ {_Sign, 0, Frac} ->
+ {Frac, ?MIN_EXP};
+ {_Sign, Exp, Frac} ->
+ {Frac + (1 bsl 52), Exp - 53 - ?FLOAT_BIAS}
+ end.
+
+%%
+%% Tests
+%%
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+int_ceil_test() ->
+ ?assertEqual(1, int_ceil(0.0001)),
+ ?assertEqual(0, int_ceil(0.0)),
+ ?assertEqual(1, int_ceil(0.99)),
+ ?assertEqual(1, int_ceil(1.0)),
+ ?assertEqual(-1, int_ceil(-1.5)),
+ ?assertEqual(-2, int_ceil(-2.0)),
+ ok.
+
+int_pow_test() ->
+ ?assertEqual(1, int_pow(1, 1)),
+ ?assertEqual(1, int_pow(1, 0)),
+ ?assertEqual(1, int_pow(10, 0)),
+ ?assertEqual(10, int_pow(10, 1)),
+ ?assertEqual(100, int_pow(10, 2)),
+ ?assertEqual(1000, int_pow(10, 3)),
+ ok.
+
+digits_test() ->
+ ?assertEqual("0",
+ digits(0)),
+ ?assertEqual("0.0",
+ digits(0.0)),
+ ?assertEqual("1.0",
+ digits(1.0)),
+ ?assertEqual("-1.0",
+ digits(-1.0)),
+ ?assertEqual("0.1",
+ digits(0.1)),
+ ?assertEqual("0.01",
+ digits(0.01)),
+ ?assertEqual("0.001",
+ digits(0.001)),
+ ?assertEqual("1.0e+6",
+ digits(1000000.0)),
+ ?assertEqual("0.5",
+ digits(0.5)),
+ ?assertEqual("4503599627370496.0",
+ digits(4503599627370496.0)),
+ %% small denormalized number
+ %% 4.94065645841246544177e-324 =:= 5.0e-324
+ <<SmallDenorm/float>> = <<0,0,0,0,0,0,0,1>>,
+ ?assertEqual("5.0e-324",
+ digits(SmallDenorm)),
+ ?assertEqual(SmallDenorm,
+ list_to_float(digits(SmallDenorm))),
+ %% large denormalized number
+ %% 2.22507385850720088902e-308
+ <<BigDenorm/float>> = <<0,15,255,255,255,255,255,255>>,
+ ?assertEqual("2.225073858507201e-308",
+ digits(BigDenorm)),
+ ?assertEqual(BigDenorm,
+ list_to_float(digits(BigDenorm))),
+ %% small normalized number
+ %% 2.22507385850720138309e-308
+ <<SmallNorm/float>> = <<0,16,0,0,0,0,0,0>>,
+ ?assertEqual("2.2250738585072014e-308",
+ digits(SmallNorm)),
+ ?assertEqual(SmallNorm,
+ list_to_float(digits(SmallNorm))),
+ %% large normalized number
+ %% 1.79769313486231570815e+308
+ <<LargeNorm/float>> = <<127,239,255,255,255,255,255,255>>,
+ ?assertEqual("1.7976931348623157e+308",
+ digits(LargeNorm)),
+ ?assertEqual(LargeNorm,
+ list_to_float(digits(LargeNorm))),
+ %% issue #10 - mochinum:frexp(math:pow(2, -1074)).
+ ?assertEqual("5.0e-324",
+ digits(math:pow(2, -1074))),
+ ok.
+
+frexp_test() ->
+ %% zero
+ ?assertEqual({0.0, 0}, frexp(0.0)),
+ %% one
+ ?assertEqual({0.5, 1}, frexp(1.0)),
+ %% negative one
+ ?assertEqual({-0.5, 1}, frexp(-1.0)),
+ %% small denormalized number
+ %% 4.94065645841246544177e-324
+ <<SmallDenorm/float>> = <<0,0,0,0,0,0,0,1>>,
+ ?assertEqual({0.5, -1073}, frexp(SmallDenorm)),
+ %% large denormalized number
+ %% 2.22507385850720088902e-308
+ <<BigDenorm/float>> = <<0,15,255,255,255,255,255,255>>,
+ ?assertEqual(
+ {0.99999999999999978, -1022},
+ frexp(BigDenorm)),
+ %% small normalized number
+ %% 2.22507385850720138309e-308
+ <<SmallNorm/float>> = <<0,16,0,0,0,0,0,0>>,
+ ?assertEqual({0.5, -1021}, frexp(SmallNorm)),
+ %% large normalized number
+ %% 1.79769313486231570815e+308
+ <<LargeNorm/float>> = <<127,239,255,255,255,255,255,255>>,
+ ?assertEqual(
+ {0.99999999999999989, 1024},
+ frexp(LargeNorm)),
+ %% issue #10 - mochinum:frexp(math:pow(2, -1074)).
+ ?assertEqual(
+ {0.5, -1073},
+ frexp(math:pow(2, -1074))),
+ ok.
+
+-endif.
diff --git a/test/slide_uniform_eqc.erl b/test/slide_uniform_eqc.erl
index 99deb90..12f6cb0 100644
--- a/test/slide_uniform_eqc.erl
+++ b/test/slide_uniform_eqc.erl
@@ -44,9 +44,11 @@
-record(state, {moment=1000,
sample,
name,
+ count=orddict:new(),
values=[]}).
initial_state() ->
+ meck:expect(folsom_utils, now_epoch, fun(_Now) -> 1000 end),
meck:expect(folsom_utils, now_epoch, fun() -> 1000 end),
#state{}.
@@ -64,8 +66,9 @@
S#state{name={call, erlang, element, [1, V]}, sample={call, erlang, element, [2, V]}};
next_state(S, V, {call, ?MODULE, tick, [_Moment]}) ->
S#state{moment=V};
-next_state(#state{moment=Moment, values=Values0, sample=Sample}=S, NewSample, {call, ?MODULE, update, [_, Val]}) ->
- S#state{values={call, slide_uniform_eqc, new_state_values, [Sample, Moment, Values0, Val]},
+next_state(#state{moment=Moment, values=Values0, sample=Sample, count=Count}=S, NewSample, {call, ?MODULE, update, [_, Val]}) ->
+ S#state{values={call, slide_uniform_eqc, new_state_values, [Sample, Moment, Values0, Val, Count]},
+ count={call, orddict, update_counter, [Moment, 1, Count]},
sample=NewSample};
next_state(#state{values=Values, moment=Moment}=S, _V, {call, ?MODULE, trim, _}) ->
%% trim the model
@@ -107,11 +110,14 @@
prop_window_test_() ->
{setup, fun() -> ok end, fun(_X) -> (catch meck:unload(folsom_utils)), folsom:stop() end,
fun(_X) ->
- ?_assert(eqc:quickcheck(eqc:numtests(?NUMTESTS, ?QC_OUT(prop_window())))) end}.
+ {timeout, 30,
+ ?_assert(eqc:quickcheck(eqc:numtests(?NUMTESTS, ?QC_OUT(prop_window()))))} end}.
prop_window() ->
folsom:start(),
(catch meck:new(folsom_utils)),
+ (catch meck:expect(folsom_utils, update_counter, fun(Tid, Key, Value) -> meck:passthrough([Tid, Key, Value]) end)),
+ (catch meck:expect(folsom_utils, timestamp, fun() -> Res = os:timestamp(), put(timestamp, Res), Res end)),
?FORALL(Cmds, commands(?MODULE),
aggregate(command_names(Cmds),
begin
@@ -144,6 +150,7 @@
tick(Moment) ->
IncrBy = trunc(random:uniform(10)),
meck:expect(folsom_utils, now_epoch, fun() -> Moment + IncrBy end),
+ meck:expect(folsom_utils, now_epoch, fun(_Now) -> Moment + IncrBy end),
Moment+IncrBy.
update(Sample, Val) ->
@@ -159,16 +166,28 @@
trim(L, Moment, Window) ->
[{K, V} || {{M, _C}=K, V} <- L, M >= Moment - Window].
-new_state_values(Sample, Moment, Values, Val) ->
- Cnt = length([true || {{M, _C}, _V} <- Values, M == Moment]),
- case Cnt >= ?SIZE of
+new_state_values(_Sample, Moment, Values, Val, Count) ->
+ %Cnt = length([true || {{M, _C}, _V} <- Values, M == Moment]),
+ Cnt =
+ case orddict:find(Moment, Count) of
+ error ->
+ 1;
+ {ok, V} ->
+ V+1
+ end,
+ case Cnt > ?SIZE of
true ->
%% replace
- {Rnd, _} = random:uniform_s(?SIZE, Sample#slide_uniform.seed),
- lists:keyreplace({Moment, Rnd}, 1, Values, {{Moment, Rnd}, Val});
+ {Rnd, _} = random:uniform_s(Cnt, get(timestamp)),
+ case Rnd =< ?SIZE of
+ true ->
+ lists:keyreplace({Moment, Rnd}, 1, Values, {{Moment, Rnd}, Val});
+ false ->
+ Values
+ end;
false ->
%% insert
- Values ++ [{{Moment, Cnt+1}, Val}]
+ Values ++ [{{Moment, Cnt}, Val}]
end.
-endif.
