diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index 037271a9a..aaaa205c1 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -2,9 +2,15 @@ name: Erlang CI on: push: - branches: [ nhse-develop ] + branches: + - nhse-develop + - nhse-develop-3.2 + - nhse-develop-3.4 pull_request: - branches: [ nhse-develop ] + branches: + - nhse-develop + - nhse-develop-3.2 + - nhse-develop-3.4 jobs: @@ -17,25 +23,12 @@ jobs: strategy: fail-fast: false matrix: - otp: [22, 24, 25] + otp: [24, 26] os: [ubuntu-latest] - # OTP lower than 23 does not run on ubuntu-latest (22.04), see - # https://github.com/erlef/setup-beam#compatibility-between-operating-system-and-erlangotp - exclude: - - otp: 22 - os: ubuntu-latest - include: - - otp: 22 - os: ubuntu-20.04 steps: - uses: lukka/get-cmake@latest - - uses: actions/checkout@v2 - - name: Install dependencies (Ubuntu) - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt-get -qq update - sudo apt-get -qq install libsnappy-dev libc6-dev + - uses: actions/checkout@v4 - name: Install Erlang/OTP uses: erlef/setup-beam@v1 with: diff --git a/priv/tracers/tracer_accumulating_time.erl b/priv/tracers/tracer_accumulating_time.erl deleted file mode 100644 index f66acda00..000000000 --- a/priv/tracers/tracer_accumulating_time.erl +++ /dev/null @@ -1,79 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_accumulating_time). - --compile(export_all). --compile(nowarn_export_all). - -start(Pid_list, MFA_list, IntervalMS) -> - dbg:tracer(process, {fun trace/2, new_stats()}), - [dbg:p(Pid, [call, timestamp, arity]) || Pid <- Pid_list], - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], [{return_trace}]}]) || - {Mod, Func, Arity} <- MFA_list], - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - timer:send_interval(IntervalMS, TPid, print_report), - {started, TPid}. - -stop() -> - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - stopped. - -trace({trace_ts, Pid, call, {Mod, Func, Arity}, TS}, {Dict}) -> - MFA = {Mod, Func, Arity}, - DKey = {Pid, MFA}, - {dict:store(DKey, TS, Dict)}; -trace({trace_ts, Pid, return_from, {Mod, Func, Arity}, _Res, TS}, {Dict}) -> - MFA = {Mod, Func, Arity}, - DKey = {Pid, MFA}, - Start = case dict:find(DKey, Dict) of - {ok, StTime} -> StTime; - error -> os:timestamp() - end, - Elapsed = timer:now_diff(TS, Start), - SumKey = {sum, MFA}, - {OldCount, OldTime} = case dict:find(SumKey, Dict) of - error -> - {0, 0}; - {ok, Else} -> - Else - end, - Dict2 = dict:erase(DKey, Dict), - {dict:store(SumKey, {OldCount+1, OldTime+Elapsed}, Dict2)}; -trace(print_report, {Dict}) -> - print_stats(Dict), - {dict:from_list([X || {K, _V} = X <- dict:to_list(Dict), - element(1, K) /= sum])}; -trace(Unknown, {Dict}) -> - erlang:display(wha), - io:format("Unknown! ~P\n", [Unknown, 20]), - {Dict}. - -new_stats() -> - {dict:new()}. - -print_stats(Dict) -> - Reports = lists:sort([{MFA, X} || {{sum, MFA}, X} <- dict:to_list(Dict)]), - [io:format("~p MFA ~p count ~p elapsed_msec ~p\n", - [time(), MFA, Count, Sum div 1000]) || - {MFA, {Count, Sum}} <- Reports]. diff --git a/priv/tracers/tracer_backend_latency.erl b/priv/tracers/tracer_backend_latency.erl deleted file mode 100644 index 408df78f2..000000000 --- a/priv/tracers/tracer_backend_latency.erl +++ /dev/null @@ -1,199 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_backend_latency). - --compile(export_all). --compile(nowarn_export_all). - -start() -> - start(500). - -start(LatencyMS) -> - start(LatencyMS, [get_fsm, put_fsm, - bitcask, eleveldb, file, prim_file, riak_kv_fs2_backend]). - -start(LatencyMS, Modules) -> - %% catch folsom_metrics:delete_metric(foo), - %% folsom_metrics:new_histogram(foo, uniform, 9981239823), - - dbg:tracer(process, {fun trace/2, new_stats(LatencyMS)}), - dbg:p(all, [call, timestamp, arity]), - - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], []}]) || - lists:member(put_fsm, Modules), - Mod <- [riak_kv_put_fsm], - {Func, Arity} <- [{init, 1}, {finish, 2}]], - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], []}]) || - lists:member(get_fsm, Modules), - Mod <- [riak_kv_get_fsm], - {Func, Arity} <- [{init, 1}, {finalize, 1}]], - - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], [{return_trace}]}]) || - lists:member(bitcask, Modules), - Mod <- [bitcask], - {Func, Arity} <- [ - {open,1}, {open,2}, - {close,1}, - {close_write_file,1}, - {get,2}, - {put,3}, - {delete,2}, - {sync,1}, - {iterator,3}, {iterator_next,1}, {iterator_release,1}, - {needs_merge,1}, - {is_empty_estimate,1}, - {status,1}]], - - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], [{return_trace}]}]) || - lists:member(eleveldb, Modules), - Mod <- [eleveldb], - {Func, Arity} <- [ - {open,2}, - {close,1}, - {get,3}, - {put,4}, - {delete,3}, - {write,3}, - {status,2}, - {destroy,2}, - {is_empty,1}, - {iterator,2}, - {iterator,3}, - {iterator_move,2}, - {iterator_close,1}]], - - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], [{return_trace}]}]) || - lists:member(file, Modules), - Mod <- [file], - {Func, Arity} <- [ - {open,2}, - {close,1}, - {pread,2}, - {pread,3}, - {read,2}, - {write,2}, - {pwrite,2}, - {pwrite,3}, - {truncate,1}, - {delete,1}, - {position,2}, - {sync,1} - ]], - - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], [{return_trace}]}]) || - lists:member(prim_file, Modules), - Mod <- [prim_file], - {Func, Arity} <- [ - {list_dir,2}, - {read_file_info,1}, - {write_file_info,1} - ]], - - [catch dbg:tpl(Mod, Func, Arity, [{'_', [], [{return_trace}]}]) || - lists:member(riak_kv_fs2_backend, Modules), - Mod <- [riak_kv_fs2_backend], - {Func, Arity} <- [ - {get_object,4}, - {put_object,5}, - {delete,4}]], - - %% Don't need return_trace events for this use case, but here's - %% how to do it if needed. - %%dbg:tpl(bitcask, merge_single_entry, 6, [{'_', [], [{return_trace}]}]). - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - %% timer:send_interval(Interval, TPid, print_report), - io:format("Not using timer:send_interval...\n"), - {started, TPid}. - -stop() -> - %% io:format("Histogram stats:\n~p\n", [catch folsom_metrics:get_histogram_statistics(foo)]), - %% catch folsom_metrics:delete_metric(foo), - - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - stopped. - -trace({trace_ts, Pid, call, {riak_kv_put_fsm, init, _}, TS}, {Dict, LMS}) -> - {dict:store({put, Pid}, TS, Dict), LMS}; -trace({trace_ts, Pid, call, {riak_kv_put_fsm, finish, _}, TS}, {Dict, LatencyMS}) -> - Start = case dict:find({put, Pid}, Dict) of - {ok, StTime} -> StTime; - error -> os:timestamp() - end, - case timer:now_diff(TS, Start) div 1000 of - Elapsed when Elapsed > LatencyMS -> - io:format("~p ~p: put_fsm: ~p msec @ ~p ~p\n", [date(), time(), Elapsed, node(), Pid]); - _Elapsed -> - ok - end, - {dict:erase(Pid, Dict), LatencyMS}; -trace({trace_ts, Pid, call, {riak_kv_get_fsm, init, _}, TS}, {Dict, LMS}) -> - {dict:store({get, Pid}, TS, Dict), LMS}; -trace({trace_ts, Pid, call, {riak_kv_get_fsm, finalize, _}, TS}, {Dict, LatencyMS}) -> - Start = case dict:find({get, Pid}, Dict) of - {ok, StTime} -> StTime; - error -> os:timestamp() - end, - case timer:now_diff(TS, Start) div 1000 of - Elapsed when Elapsed > LatencyMS -> - io:format("~p ~p: get_fsm: ~p msec @ ~p ~p\n", [date(), time(), Elapsed, node(), Pid]); - _Elapsed -> - ok - end, - {dict:erase(Pid, Dict), LatencyMS}; -trace({trace_ts, Pid, call, {Mod, _, _}, TS}, {Dict, LMS}) -> - {dict:store({Mod, Pid}, TS, Dict), LMS}; -trace({trace_ts, Pid, return_from, {Mod, Func, _}, _Res, TS}, {Dict, LatencyMS}) -> - DKey = {Mod, Pid}, - Start = case dict:find(DKey, Dict) of - {ok, StTime} -> StTime; - error -> os:timestamp() - end, - case timer:now_diff(TS, Start) div 1000 of - Elapsed when Elapsed > LatencyMS -> - io:format("~p ~p: ~p ~p: ~p msec\n", [date(), time(), Mod, - Func, Elapsed]); - _Elapsed -> - ok - end, - %% if Mod == file, Func == pread -> - %% folsom_metrics_histogram:update(foo, Elapsed); - %% true -> - %% ok - %% end, - {dict:erase(DKey, Dict), LatencyMS}; -trace(print_report, DictStats) -> - %% print_stats(DictStats), - %% new_stats(); - DictStats; -trace(Unknown, DictStats) -> - erlang:display(wha), - io:format("Unknown! ~P\n", [Unknown, 20]), - DictStats. - -new_stats(LatencyMS) -> - {dict:new(), LatencyMS}. - -print_stats(_DictStats) -> - ok. - diff --git a/priv/tracers/tracer_eleveldb_put_size.erl b/priv/tracers/tracer_eleveldb_put_size.erl deleted file mode 100644 index c776e7a47..000000000 --- a/priv/tracers/tracer_eleveldb_put_size.erl +++ /dev/null @@ -1,77 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_eleveldb_put_size). - --compile(nowarn_export_all). --compile(export_all). - -start() -> - start(10*1000). - -start(Interval) -> - Stats = {StatName, _} = new_stats(), - reset_metric(StatName), - - dbg:tracer(process, {fun trace/2, Stats}), - dbg:p(all, [call]), - dbg:tpl(eleveldb, write, 3, [{'_', [], []}]), - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - timer:send_interval(Interval, TPid, print_report), - {started, TPid}. - -stop() -> - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - stopped. - -trace({trace, _Pid, call, {eleveldb, write, [_, PutList, _]}}, - {StatName, SumBytes}) -> - Bs = [begin - Bs = size(K) + size(V), - folsom_metrics_histogram:update(StatName, Bs), - Bs - end || {put, K, V} <- PutList], - {StatName, SumBytes + lists:sum(Bs)}; -trace(print_report, Stats = {StatName, _}) -> - print_stats(Stats), - reset_metric(StatName), - new_stats(); -trace(_Unknown, Stats) -> - erlang:display(wha), - %% io:format("Unknown! ~P\n", [Unknown, 20]), - Stats. - -new_stats() -> - {foo, 0}. - -print_stats({StatName, SumBytes}) -> - if SumBytes == 0 -> - io:format("~p ~p: 0 bytes\n", [date(), time()]); - true -> - Ps = folsom_metrics:get_histogram_statistics(StatName), - io:format("~p ~p: ~p bytes\n ~p\n", [date(), time(), SumBytes, Ps]) - end. - -reset_metric(Stats) -> - catch folsom_metrics:delete_metric(Stats), - folsom_metrics:new_histogram(Stats, uniform, 9981239823). diff --git a/priv/tracers/tracer_fsm_init.erl b/priv/tracers/tracer_fsm_init.erl deleted file mode 100644 index ba9c23647..000000000 --- a/priv/tracers/tracer_fsm_init.erl +++ /dev/null @@ -1,92 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_fsm_init). - --compile(nowarn_export_all). --compile(export_all). - -start() -> - start(1*1000). - -start(Interval) -> - %%% Count the get, put, buckets, keys, exchange, and index FSM init() calls - dbg:tracer(process, {fun trace/2, new_stats()}), - dbg:p(all, call), - [dbg:tpl(Mod, init, 1, [{'_', [], []}]) || - Mod <- [riak_kv_buckets_fsm, riak_kv_exchange_fsm, riak_kv_get_fsm, riak_kv_index_fsm, riak_kv_keys_fsm, riak_kv_put_fsm]], - dbg:tpl(riak_kv_put_fsm, start_link, 3, [{'_', [], []}]), - - %% Don't need return_trace events for this use case, but here's - %% how to do it if needed. - %%dbg:tpl(bitcask, merge_single_entry, 6, [{'_', [], [{return_trace}]}]). - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - timer:send_interval(Interval, TPid, print_report), - {started, TPid}. - -stop() -> - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - stopped. - -trace({trace, _Pid, call, {riak_kv_put_fsm, start_link, _}}, - {Pstart_link, R, P, B, E, I, K}) -> - {Pstart_link+1, R, P, B, E, I, K}; -trace({trace, _Pid, call, {riak_kv_get_fsm, init, _}}, - {Pstart_link, R, P, B, E, I, K}) -> - {Pstart_link, R+1, P, B, E, I, K}; -trace({trace, _Pid, call, {riak_kv_put_fsm, init, _}}, - {Pstart_link, R, P, B, E, I, K}) -> - {Pstart_link, R, P+1, B, E, I, K}; -trace({trace, _Pid, call, {riak_kv_buckets_fsm, init, _}}, - {Pstart_link, R, P, B, E, I, K}) -> - {Pstart_link, R, P, B+1, E, I, K}; -trace({trace, _Pid, call, {riak_kv_exchange_fsm, init, _}}, - {Pstart_link, R, P, B, E, I, K}) -> - {Pstart_link, R, P, B, E+1, I, K}; -trace({trace, _Pid, call, {riak_kv_index_fsm, init, _}}, - {Pstart_link, R, P, B, E, I, K}) -> - {Pstart_link, R, P, B, E, I+1, K}; -trace({trace, _Pid, call, {riak_kv_keys_fsm, init, _}}, - {Pstart_link, R, P, B, E, I, K}) -> - {Pstart_link, R, P, B, E, I, K+1}; -trace(print_report, Stats) -> - print_stats(Stats), - new_stats(); -trace(Unknown, Stats) -> - erlang:display(wha), - io:format("Unknown! ~P\n", [Unknown, 20]), - Stats. - -new_stats() -> - {0, - 0, 0, 0, 0, 0, 0}. - -print_stats({Pstart_link, Get, Put, Buckets, Exchange, Index, Keys}) -> - Stats = [{put_start, Pstart_link}, - {get, Get}, - {put, Put}, - {buckets, Buckets}, - {exchange, Exchange}, - {index, Index}, - {keys, Keys}], - io:format("~p ~p: ~p\n", [date(), time(), Stats]). diff --git a/priv/tracers/tracer_func_args.erl b/priv/tracers/tracer_func_args.erl deleted file mode 100644 index c4d51bcb8..000000000 --- a/priv/tracers/tracer_func_args.erl +++ /dev/null @@ -1,104 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% For example: what ETS tables are being called the most by ets:lookup/2? -%% The 1st arg of ets:lookup/2 is the table name. -%% Watch for 10 seconds. -%% -%% > func_args_tracer:start(ets, lookup, 2, 10, fun(Args) -> hd(Args) end). -%% -%% Tracer pid: <0.16102.15>, use func_args_tracer:stop() to stop -%% Otherwise, tracing stops in 10 seconds -%% Current date & time: {2013,9,19} {18,5,48} -%% {started,<0.16102.15>} -%% Total calls: 373476 -%% Call stats: -%% [{folsom_histograms,114065}, -%% {ac_tab,69689}, -%% {ets_riak_core_ring_manager,67147}, -%% {folsom_spirals,57076}, -%% {riak_capability_ets,48862}, -%% {riak_core_node_watcher,8149}, -%% {riak_api_pb_registrations,8144}, -%% {folsom,243}, -%% {folsom_meters,43}, -%% {folsom_durations,20}, -%% {timer_tab,18}, -%% {folsom_gauges,8}, -%% {riak_core_stat_cache,5}, -%% {sys_dist,3}, -%% {inet_db,1}, -%% {21495958,1}, -%% {3145765,1}, -%% {3407910,1}] -%% - --module(tracer_func_args). - --compile(nowarn_export_all). --compile(export_all). - -start(Mod, Func, Arity, RunSeconds) -> - start(Mod, Func, Arity, RunSeconds, fun(Args) -> Args end). - -start(Mod, Func, Arity, RunSeconds, ArgMangler) -> - catch ets:delete(foo), - ets:new(foo, [named_table, public, set]), - dbg:tracer(process, {fun trace/2, new_stats({foo, ArgMangler})}), - dbg:p(all, call), - dbg:tpl(Mod, Func, Arity, [{'_', [], []}]), - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - io:format("Otherwise, tracing stops in ~p seconds\n", [RunSeconds]), - io:format("Current date & time: ~p ~p\n", [date(), time()]), - spawn(fun() -> timer:sleep(RunSeconds * 1000), stop() end), - {started, TPid}. - -stop() -> - Sort = fun({_,A}, {_, B}) -> A > B end, - Res = ets:tab2list(foo), - TotalCalls = lists:sum([Count || {_Arg, Count} <- Res]), - io:format("Total calls: ~p\n", [TotalCalls]), - io:format("Call stats:\n~p\n", [catch lists:sort(Sort, Res)]), - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - timer:sleep(100), - stopped. - -trace({trace, _Pid, call, {_, _, Args}}, {Tab, ArgMangler} = Acc) -> - Args2 = ArgMangler(Args), - try - ets:update_counter(Tab, Args2, {2, 1}) - catch _:_ -> - ets:insert(Tab, {Args2, 1}) - end, - Acc; -trace(Unknown, DictStats) -> - io:format("Unknown! ~P\n", [Unknown, 20]), - DictStats. - -new_stats({Tab, _ArgMangler} = Acc) -> - ets:delete_all_objects(Tab), - Acc. - -print_stats(_DictStats) -> - ok. - diff --git a/priv/tracers/tracer_gc_latency.erl b/priv/tracers/tracer_gc_latency.erl deleted file mode 100644 index 680d17aa5..000000000 --- a/priv/tracers/tracer_gc_latency.erl +++ /dev/null @@ -1,82 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_gc_latency). - --compile(nowarn_export_all). --compile(export_all). - -start(LatencyMS) -> - catch folsom_metrics:delete_metric(foo), - folsom_metrics:new_histogram(foo, uniform, 50*1000*1000), - dbg:tracer(process, {fun trace/2, new_stats(LatencyMS)}), - {ok, _} = dbg:p(all, [timestamp, garbage_collection, running]), - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - io:format("Current date & time: ~p ~p local time\n", [date(), time()]), - {started, TPid}. - -stop() -> - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - timer:sleep(100), - catch folsom_metrics:delete_metric(foo), - stopped. - -trace({trace_ts, Pid, gc_start, _Stats, TS}, {Dict, LMS}) -> - {dict:store(Pid, TS, Dict), LMS}; -trace({trace_ts, Pid, gc_end, _Stats, TS}, {Dict, LMS}=Acc) -> - DKey = Pid, - case dict:find(DKey, Dict) of - {ok, GcStart} -> - Elapsed = erlang:max(-1, (timer:now_diff(TS, GcStart) div 1000)), - if Elapsed > LMS -> - io:format("~p: GC of ~p elapsed time ~p > threshold ~p\n", - [time(), Pid, Elapsed, LMS]), - io:format(" ~w,~w\n", [process_info(Pid, message_queue_len), _Stats]); - true -> - ok - end, - {dict:erase(DKey, Dict), LMS}; - error -> - Acc - end; -trace({trace_ts, Pid, InOrOut, _MFA, TS}, {Dict, _LMS}=Acc) -> - DKey = Pid, - case dict:find(DKey, Dict) of - {ok, GcStart} -> - io:format("Hey, pid ~p scheduled ~p but started GC ~p msec ago\n", - [Pid, InOrOut, timer:now_diff(TS, GcStart)]); - _ -> - ok - end, - Acc; -trace(Unknown, DictStats) -> - erlang:display(wha), - io:format("Unknown! ~P\n\t~P", [Unknown, 20, DictStats,7]), - DictStats. - -new_stats(LatencyMS) -> - {dict:new(), LatencyMS}. - -print_stats(_DictStats) -> - ok. - diff --git a/priv/tracers/tracer_large4.erl b/priv/tracers/tracer_large4.erl deleted file mode 100644 index 93eb301c0..000000000 --- a/priv/tracers/tracer_large4.erl +++ /dev/null @@ -1,107 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_large4). - --compile(nowarn_export_all). --compile(export_all). - --record(r_object, {bucket = '_', - key = '_', - contents = '_', - vclock = '_', - updatemetadata = '_', - updatevalue = '_' - }). - -go(Time, Count, Size) -> - ss(), - %% gets - GetMS = [{['_', - #r_object{bucket='$1', - key='$2'}], - [], - [{message,{{'$1','$2'}}}, {return_trace}]}], - erlang:trace_pattern({riak_kv_get_fsm, calculate_objsize, 2}, GetMS, [local]), - - %% puts - PutMS = [{['$1','$2','_','$3','_'], - [{'>',{size,'$3'},Size}], - [{message,{{'$1','$2',{size,'$3'}}}}]}], - erlang:trace_pattern({riak_kv_eleveldb_backend, put, 5}, PutMS, [local]), - erlang:trace_pattern({riak_kv_bitcask_backend, put, 5}, PutMS, [local]), - erlang:trace_pattern({riak_kv_memory_backend, put, 5}, PutMS, [local]), - - {Tracer, _} = spawn_monitor(?MODULE, tracer, [0, Count, Size, dict:new()]), - erlang:trace(all, true, [call, arity, {tracer, Tracer}]), - receive - {'DOWN', _, process, Tracer, _} -> - ok - after Time -> - exit(Tracer, kill), - receive - {'DOWN', _, process, Tracer, _} -> - ok - end - end, - ss(), - io:format("object trace stopped~n"). - -tracer(Limit, Limit, _, _) -> - ok; -tracer(Count, Limit, Threshold, Objs) -> - receive - {trace,Pid,call,{riak_kv_get_fsm,calculate_objsize,2},{Bucket,Key}} -> - Objs2 = dict:store(Pid, {Bucket,Key}, Objs), - tracer(Count+1, Limit, Threshold, Objs2); - {trace,Pid,return_from,{riak_kv_get_fsm,calculate_objsize,2},Size} -> - case Size >= Threshold of - true -> - case dict:find(Pid, Objs) of - {ok, {Bucket, Key}} -> - io:format("~p: get: ~p~n", [ts(), {Bucket, Key, Size}]); - _ -> - ok - end; - false -> - ok - end, - Objs2 = dict:erase(Pid, Objs), - tracer(Count+1, Limit, Threshold, Objs2); - {trace,_Pid,call,{riak_kv_eleveldb_backend,put,5},{Bucket,Key,Size}} -> - io:format("~p: put(l): ~p~n", [ts(), {Bucket, Key, Size}]), - tracer(Count+1, Limit, Threshold, Objs); - {trace,_Pid,call,{riak_kv_bitcask_backend,put,5},{Bucket,Key,Size}} -> - io:format("~p: put(b): ~p~n", [ts(), {Bucket, Key, Size}]), - tracer(Count+1, Limit, Threshold, Objs); - {trace,_Pid,call,{riak_kv_memory_backend,put,5},{Bucket,Key,Size}} -> - io:format("~p: put(m): ~p~n", [ts(), {Bucket, Key, Size}]), - tracer(Count+1, Limit, Threshold, Objs); - Msg -> - io:format("tracer: ~p~n", [Msg]), - tracer(Count+1, Limit, Threshold, Objs) - end. - -ts() -> - calendar:now_to_datetime(os:timestamp()). - -ss() -> - erlang:trace_pattern({'_','_','_'}, false, [local]), - erlang:trace(all, false, [call, arity]). diff --git a/priv/tracers/tracer_latency_histogram.erl b/priv/tracers/tracer_latency_histogram.erl deleted file mode 100644 index d7f513b58..000000000 --- a/priv/tracers/tracer_latency_histogram.erl +++ /dev/null @@ -1,127 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% For example: create a histogram of call latencies for bitcask:put/3. -%% Watch for 10 seconds. -%% -%% > latency_histogram_tracer:start(bitcask, put, 3, 10). -%% -%% Tracer pid: <0.2108.18>, use latency_histogram_tracer:stop() to stop -%% Otherwise, tracing stops in 10 seconds -%% Current date & time: {2013,9,19} {18,14,13} -%% {started,<0.2108.18>} -%% Histogram stats: -%% [{min,0}, -%% {max,48}, -%% {arithmetic_mean,2.765411819271055}, -%% {geometric_mean,2.527103493663478}, -%% {harmonic_mean,2.2674039086593973}, -%% {median,3}, -%% {variance,3.5629207473971585}, -%% {standard_deviation,1.8875700642352746}, -%% {skewness,2.0360354571500774}, -%% {kurtosis,18.529695846728423}, -%% {percentile,[{50,3},{75,4},{90,5},{95,6},{99,8},{999,14}]}, -%% {histogram,[{1,13436}, -%% {2,12304}, -%% {3,10789}, -%% {4,7397}, -%% {5,4191}, -%% {6,1929}, -%% {7,873}, -%% {8,420}, -%% {9,163}, -%% {10,79}, -%% {11,42}, -%% {12,47}, -%% {13,11}, -%% {14,16}, -%% {15,7}, -%% {16,5}, -%% {17,3}, -%% {18,4}, -%% {19,2}, -%% {20,4}, -%% {21,1}, -%% {22,11}, -%% {23,2}, -%% {24,1}, -%% {25,2}, -%% {26,1}, -%% {27,0}, -%% {28,1}, -%% {29,2}, -%% {30,0}, -%% {31,0}, -%% {40,2}, -%% {50,1}]}, -%% {n,51746}] - --module(tracer_latency_histogram). - --compile(nowarn_export_all). --compile(export_all). - -start(Mod, Func, Arity, RunSeconds) -> - catch folsom_metrics:delete_metric(foo), - folsom_metrics:new_histogram(foo, uniform, 50*1000*1000), - dbg:tracer(process, {fun trace/2, new_stats(0)}), - dbg:p(all, [call, timestamp, arity]), - dbg:tpl(Mod, Func, Arity, [{'_', [], [{return_trace}]}]), - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - io:format("Otherwise, tracing stops in ~p seconds\n", [RunSeconds]), - io:format("Current date & time: ~p ~p\n", [date(), time()]), - spawn(fun() -> timer:sleep(RunSeconds * 1000), stop() end), - {started, TPid}. - -stop() -> - io:format("Histogram stats:\n~p\n", [catch folsom_metrics:get_histogram_statistics(foo)]), - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - timer:sleep(100), - catch folsom_metrics:delete_metric(foo), - stopped. - -trace({trace_ts, Pid, call, {_, _, _}, TS}, {Dict, LMS}) -> - {dict:store(Pid, TS, Dict), LMS}; -trace({trace_ts, Pid, return_from, {_, _, _}, _Res, TS}, {Dict, LatencyMS}) -> - DKey = Pid, - Start = case dict:find(DKey, Dict) of - {ok, StTime} -> StTime; - error -> os:timestamp() - end, - Elapsed = timer:now_diff(TS, Start) div 1000, - folsom_metrics_histogram:update(foo, Elapsed), - {dict:erase(DKey, Dict), LatencyMS}; -trace(print_report, DictStats) -> - DictStats; -trace(Unknown, DictStats) -> - erlang:display(wha), - io:format("Unknown! ~P\n", [Unknown, 20]), - DictStats. - -new_stats(LatencyMS) -> - {dict:new(), LatencyMS}. - -print_stats(_DictStats) -> - ok. - diff --git a/priv/tracers/tracer_merge_and_and_handoff.erl b/priv/tracers/tracer_merge_and_and_handoff.erl deleted file mode 100644 index d4b57f375..000000000 --- a/priv/tracers/tracer_merge_and_and_handoff.erl +++ /dev/null @@ -1,95 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_merge_and_and_handoff). - --compile(nowarn_export_all). --compile(export_all). - -start() -> - start(1*1000). - -start(Interval) -> - dbg:tracer(process, {fun trace/2, {orddict:new(), orddict:new()}}), - dbg:p(all, call), - dbg:tpl(bitcask, merge_single_entry, 6, [{'_', [], []}]), - dbg:tpl(riak_kv_vnode, encode_handoff_item, 2, [{'_', [], []}]), - dbg:tpl(riak_core_handoff_receiver, process_message, 3, [{'_', [], []}]), - - %% Don't need return_trace events for this use case, but here's - %% how to do it if needed. - %%dbg:tpl(bitcask, merge_single_entry, 6, [{'_', [], [{return_trace}]}]). - - {ok, TPid} = dbg:get_tracer(), - io:format("Tracer pid: ~p, use ~p:stop() to stop\n", [TPid, ?MODULE]), - timer:send_interval(Interval, TPid, print_report), - {started, TPid}. - -stop() -> - dbg:stop_clear(), - catch exit(element(2,dbg:get_tracer()), kill), - stopped. - -trace({trace, _Pid, call, {bitcask, merge_single_entry, - [K, V, _TS, _FId, {File,_,_,_}, _State]}}, - {MDict, HDict}) -> - Dir = re:replace(File, "/[^/]*\$", "", [{return, binary}]), - Bytes = size(K) + size(V), - MDict2 = increment_cbdict(MDict, Dir, Bytes), - {MDict2, HDict}; -trace({trace, _Pid, call, {riak_kv_vnode, encode_handoff_item, - [{B, K}, V]}}, - {MDict, HDict}) -> - Bytes = size(B) + size(K) + size(V), - Key = "all-sending-handoff", - HDict2 = increment_cbdict(HDict, Key, Bytes), - {MDict, HDict2}; -trace({trace, _Pid, call, {riak_core_handoff_receiver, process_message, - [_Type, Msg, State]}}, - {MDict, HDict}) -> - Bytes = size(Msg), - Partition = element(5, State), % ugly hack - Key = Partition, - HDict2 = increment_cbdict(HDict, Key, Bytes), - {MDict, HDict2}; -trace(print_report, {MDict, HDict}) -> - print_stats(MDict, merge), - print_stats(HDict, handoff), - {orddict:new(), orddict:new()}. - -%% "cb" = count + bytes -increment_cbdict(Dict, Key, Bytes) -> - orddict:update(Key, fun({Count, Bs}) -> {Count + 1, Bs + Bytes} end, - {1, Bytes}, Dict). - -print_stats(Dict, Type) -> - F = fun(Key, {Count, Bytes}, {SumC, SumB}) when Count > 0 -> - io:format("~p ~p: ~p items ~p bytes ~p avg-size ~p\n", - [date(), time(), Count, Bytes, Bytes div Count, Key]), - {SumC + Count, SumB + Bytes}; - (_, _, Acc) -> - Acc - end, - {Count, Bytes} = orddict:fold(F, {0, 0}, Dict), - Avg = if Count > 0 -> Bytes div Count; - true -> 0 - end, - io:format("~p ~p: ~p total: ~p items ~p bytes ~p avg-size\n", - [date(), time(), Type, Count, Bytes, Avg]). diff --git a/priv/tracers/tracer_read_bin_trace_file.erl b/priv/tracers/tracer_read_bin_trace_file.erl deleted file mode 100644 index cfe3a2efe..000000000 --- a/priv/tracers/tracer_read_bin_trace_file.erl +++ /dev/null @@ -1,70 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_read_bin_trace_file). - --compile(nowarn_export_all). --compile(export_all). - -read(Path) -> - read(Path, 1). - -read(Path, LatencyMS) -> - {ok, FH} = file:open(Path, [read, binary, raw]), - read(file:read(FH, 5), FH, LatencyMS, []). - -read(eof, _FH, _, _) -> - ok; -read({ok, <>}, FH, LatencyMS, Hist) -> - {ok, Bin} = file:read(FH, Size), - case binary_to_term(Bin) of - {trace_ts, _, call, {M,F,A}, Time} -> - %%io:format("call MFA = ~p:~p/~p, ", [M, F, length(A)]), - read(file:read(FH, 5), FH, LatencyMS, [{{M,F,length(A)}, Time, A}|Hist]); - {trace_ts, _, return_from, MFA, Res, EndTime} -> - %%io:format("MFA ~p Hist ~p\n", [MFA, Hist]), - try - {value, {_, StartTime, A}, NewHist} = lists:keytake(MFA, 1, Hist), - MSec = timer:now_diff(EndTime, StartTime)/1000, - if MSec > LatencyMS -> - io:format("~p ~p msec\nArgs: (~p/~p) ~P\nRes: ~P\n\n", - [MFA, MSec, - erts_debug:flat_size(A), erts_debug:size(A), - A, 20, Res, 20]); - true -> - ok - end, - read(file:read(FH, 5), FH, LatencyMS, NewHist) - catch - error:{badmatch,false} -> - read(file:read(FH, 5), FH, LatencyMS, Hist); - Class:Reason:Stacktrace -> - io:format("ERR ~p ~p @ ~p\n", [Class, Reason, Stacktrace]), - read(file:read(FH, 5), FH, LatencyMS, Hist) - end - end. - -%% read(eof, _FH) -> -%% ok; -%% read({ok, <>}, FH) -> -%% {ok, Bin} = file:read(FH, Size), -%% io:format("~P\n", [binary_to_term(Bin), 15]), -%% read(file:read(FH, 5), FH). - diff --git a/priv/tracers/tracer_timeit.erl b/priv/tracers/tracer_timeit.erl deleted file mode 100644 index 6ddcf0fa0..000000000 --- a/priv/tracers/tracer_timeit.erl +++ /dev/null @@ -1,118 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(tracer_timeit). - --compile(nowarn_export_all). --compile(export_all). - -%% @doc Dynamically add timing to MFA. There are various types of -%% timing. -%% -%% all - time latency of all calls to MFA -%% -%% {sample, N, Max} - sample every N calls and stop sampling after Max -%% -%% {threshold, Millis, Max} - count # of calls where latency is > Millis -%% and count # of calls total, thus percentage of calls over threshold -timeit(Mod, Fun, Arity, Type) -> - Type2 = case Type of - {sample, N, Max} -> {sample, {N, Max}, {0, 0, 0}}; - {threshold, Millis, Max} -> {threshold, {Millis, Max}, {0, 0}}; - {all, Max} -> {all, {0, Max}} - end, - dbg:tracer(process, {fun trace/2, {orddict:new(), Type2}}), - dbg:p(all, call), - dbg:tpl(Mod, Fun, Arity, [{'_', [], [{return_trace}]}]). - -stop() -> dbg:stop_clear(). - -trace({trace, Pid, call, {Mod, Fun, _}}, {D, {all, {Count, Max}}}) -> - D2 = orddict:store({Pid, Mod, Fun}, os:timestamp(), D), - {D2, {all, {Count, Max}}}; -trace({trace, Pid, call, {Mod, Fun, _}}, - {D, {sample, {N, Max}, {M, K, Total}}}) -> - M2 = M+1, - Total2 = Total+1, - if N == M2 -> - D2 = orddict:store({Pid, Mod, Fun}, os:timestamp(), D), - {D2, {sample, {N, Max}, {0, K, Total2}}}; - true -> - {D, {sample, {N, Max}, {M2, K, Total2}}} - end; -trace({trace, Pid, call, {Mod, Fun, _}}, - {D, {threshold, {Millis, Max}, {Over, Total}}}) -> - D2 = orddict:store({Pid, Mod, Fun}, os:timestamp(), D), - {D2, {threshold, {Millis, Max}, {Over, Total+1}}}; - -trace({trace, Pid, return_from, {Mod, Fun, _}, _Result}, - Acc={D, {all, {Count, Max}}}) -> - Key = {Pid, Mod, Fun}, - case orddict:find(Key, D) of - {ok, StartTime} -> - Count2 = Count+1, - ElapsedUs = timer:now_diff(os:timestamp(), StartTime), - ElapsedMs = ElapsedUs/1000, - io:format(user, "~p:~p:~p: ~p ms\n", [Pid, Mod, Fun, ElapsedMs]), - if Count2 == Max -> stop(); - true -> - D2 = orddict:erase(Key, D), - {D2, {all, {Count2, Max}}} - end; - error -> Acc - end; -trace({trace, Pid, return_from, {Mod, Fun, _}, _Result}, - Acc={D, {sample, {N, Max}, {M, K, Total}}}) -> - Key = {Pid, Mod, Fun}, - case orddict:find(Key, D) of - {ok, StartTime} -> - K2 = K+1, - ElapsedUs = timer:now_diff(os:timestamp(), StartTime), - ElapsedMs = ElapsedUs/1000, - io:format(user, "[sample ~p/~p] ~p:~p:~p: ~p ms\n", - [K2, Total, Pid, Mod, Fun, ElapsedMs]), - if K2 == Max -> stop(); - true -> - D2 = orddict:erase(Key, D), - {D2, {sample, {N, Max}, {M, K2, Total}}} - end; - error -> Acc - end; -trace({trace, Pid, return_from, {Mod, Fun, _}, _Result}, - Acc={D, {threshold, {Millis, Max}, {Over, Total}}}) -> - Key = {Pid, Mod, Fun}, - case orddict:find(Key, D) of - {ok, StartTime} -> - ElapsedUs = timer:now_diff(os:timestamp(), StartTime), - ElapsedMs = ElapsedUs / 1000, - if ElapsedMs > Millis -> - Over2 = Over+1, - io:format(user, "[over threshold ~p, ~p/~p] ~p:~p:~p: ~p ms\n", - [Millis, Over2, Total, Pid, Mod, Fun, ElapsedMs]); - true -> - Over2 = Over - end, - if Max == Over -> stop(); - true -> - D2 = orddict:erase(Key, D), - {D2, {threshold, {Millis, Max}, {Over2, Total}}} - end; - error -> Acc - end. diff --git a/rebar.config b/rebar.config index d39d80a53..212880c0e 100644 --- a/rebar.config +++ b/rebar.config @@ -1,6 +1,5 @@ {minimum_otp_vsn, "22.0"}. -{src_dirs, ["./priv/tracers", "./src"]}. {cover_enabled, false}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warnings_as_errors, @@ -42,16 +41,16 @@ ]}. {deps, [ - {riak_core, {git, "https://github.com/nhs-riak/riak_core.git", {branch, "nhse-develop"}}}, + {riak_core, {git, "https://github.com/nhs-riak/riak_core.git", {branch, "nhse-develop-3.4"}}}, {sidejob, {git, "https://github.com/nhs-riak/sidejob.git", {branch, "nhse-develop"}}}, {bitcask, {git, "https://github.com/nhs-riak/bitcask.git", {branch, "nhse-develop"}}}, {redbug, {git, "https://github.com/massemanet/redbug", {branch, "master"}}}, - {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.2"}}}, + {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.5"}}}, {sext, {git, "https://github.com/uwiger/sext.git", {tag, "1.8.0"}}}, {riak_pipe, {git, "https://github.com/nhs-riak/riak_pipe.git", {branch, "nhse-develop"}}}, {riak_dt, {git, "https://github.com/nhs-riak/riak_dt.git", {branch, "nhse-develop"}}}, - {riak_api, {git, "https://github.com/nhs-riak/riak_api.git", {branch, "nhse-develop"}}}, + {riak_api, {git, "https://github.com/nhs-riak/riak_api.git", {branch, "nhse-d34-otp26"}}}, {hyper, {git, "https://github.com/nhs-riak/hyper", {branch, "nhse-develop"}}}, {kv_index_tictactree, {git, "https://github.com/nhs-riak/kv_index_tictactree.git", {branch, "nhse-develop"}}}, - {rhc, {git, "https://github.com/nhs-riak/riak-erlang-http-client", {branch, "nhse-develop"}}} + {rhc, {git, "https://github.com/nhs-riak/riak-erlang-http-client", {branch, "nhse-d34-otp26"}}} ]}. diff --git a/src/riak_kv_bitcask_backend.erl b/src/riak_kv_bitcask_backend.erl index 0820838a1..e95a05481 100644 --- a/src/riak_kv_bitcask_backend.erl +++ b/src/riak_kv_bitcask_backend.erl @@ -230,7 +230,7 @@ get(Bucket, Key, #state{ref=Ref, key_vsn=KVers}=State) -> %% NOTE: The bitcask backend does not currently support %% secondary indexing and the_IndexSpecs parameter %% is ignored. --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-type index_spec() :: riak_object:index_spec(). -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> {ok, state()} | {error, term(), state()}. diff --git a/src/riak_kv_eleveldb_backend.erl b/src/riak_kv_eleveldb_backend.erl index ee927be45..9cdb12b28 100644 --- a/src/riak_kv_eleveldb_backend.erl +++ b/src/riak_kv_eleveldb_backend.erl @@ -177,7 +177,7 @@ get(Bucket, Key, #state{read_opts=ReadOpts, {error, Reason, State} end. --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-type index_spec() :: riak_object:index_spec(). %% @doc Normal put, use existing option, do not modify write options -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> diff --git a/src/riak_kv_leveled_backend.erl b/src/riak_kv_leveled_backend.erl index c50b109fb..aade73f39 100644 --- a/src/riak_kv_leveled_backend.erl +++ b/src/riak_kv_leveled_backend.erl @@ -234,12 +234,9 @@ head(Bucket, Key, #state{bookie=Bookie}=State) -> end. %% @doc Insert an object into the leveled backend. --type index_spec() :: {add, Index, SecondaryKey} | - {remove, Index, SecondaryKey}. - -spec flush_put(riak_object:bucket(), riak_object:key(), - [index_spec()], + [riak_object:index_spec()], binary(), state()) -> {ok, state()} | @@ -250,7 +247,7 @@ flush_put(Bucket, Key, IndexSpecs, Val, State) -> -spec put(riak_object:bucket(), riak_object:key(), - [index_spec()], + [riak_object:index_spec()], binary(), state()) -> {ok, state()} | @@ -262,7 +259,7 @@ put(Bucket, Key, IndexSpecs, Val, State) -> %% @doc Delete an object from the leveled backend -spec delete(riak_object:bucket(), riak_object:key(), - [index_spec()], + [riak_object:index_spec()], state()) -> {ok, state()} | {error, term(), state()}. @@ -671,10 +668,10 @@ callback(Ref, UnexpectedCallback, State) -> %% =================================================================== %% Internal functions %% =================================================================== - +-type regex() :: {re_pattern, term(), term(), term(), term()}|undefined. -spec dollarkey_foldfun( - riak_kv_backend:fold_keys_fun(), boolean(), re:mp()|undefined) + riak_kv_backend:fold_keys_fun(), boolean(), regex()) -> riak_kv_backend:fold_objects_fun(). dollarkey_foldfun(FoldKeysFun, ReadTombs, TermRegex) -> FilteredFoldKeysFun = @@ -738,7 +735,7 @@ log_fragmentation(Allocator) -> %% flush_put or put has been called -spec do_put(riak_object:bucket(), riak_object:key(), - [index_spec()], + [riak_object:index_spec()], binary(), boolean(), state()) -> diff --git a/src/riak_kv_memory_backend.erl b/src/riak_kv_memory_backend.erl index 193aecb52..84b73f8cd 100644 --- a/src/riak_kv_memory_backend.erl +++ b/src/riak_kv_memory_backend.erl @@ -208,7 +208,7 @@ get(Bucket, Key, State=#state{data_ref=DataRef, end. %% @doc Insert an object into the memory backend. --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-type index_spec() :: riak_object:index_spec(). -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> {ok, state()}. put(Bucket, PrimaryKey, IndexSpecs, Val, State=#state{data_ref=DataRef, diff --git a/src/riak_kv_multi_backend.erl b/src/riak_kv_multi_backend.erl index b82df3d8d..f6e103132 100644 --- a/src/riak_kv_multi_backend.erl +++ b/src/riak_kv_multi_backend.erl @@ -257,7 +257,7 @@ head(Bucket, Key, State) -> %% @doc Insert an object with secondary index %% information into the kv backend --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-type index_spec() :: riak_object:index_spec(). -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> {ok, state()} | {error, term(), state()}. diff --git a/src/riak_kv_multi_prefix_backend.erl b/src/riak_kv_multi_prefix_backend.erl index b195b1af2..c6ce4f284 100644 --- a/src/riak_kv_multi_prefix_backend.erl +++ b/src/riak_kv_multi_prefix_backend.erl @@ -294,7 +294,7 @@ head(Bucket, Key, State) -> %% @doc Insert an object with secondary index %% information into the kv backend --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-type index_spec() :: riak_object:index_spec(). -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> {ok, state()} | {error, term(), state()}. diff --git a/src/riak_kv_test_util.erl b/src/riak_kv_test_util.erl index 1fadbb89d..6de3931c9 100644 --- a/src/riak_kv_test_util.erl +++ b/src/riak_kv_test_util.erl @@ -294,7 +294,7 @@ dep_apps(Test, Extra) -> (_) -> ok end, - [sasl, Silencer, folsom, exometer_core, runtime_tools, + [sasl, Silencer, exometer_core, runtime_tools, mochiweb, webmachine, sidejob, poolboy, basho_stats, bitcask, eleveldb, riak_core, riak_pipe, riak_api, riak_dt, riak_pb, riak_kv, DefaultSetupFun, Extra]. diff --git a/src/riak_kv_yessir_backend.erl b/src/riak_kv_yessir_backend.erl index 1f8a39e6a..63f089da7 100644 --- a/src/riak_kv_yessir_backend.erl +++ b/src/riak_kv_yessir_backend.erl @@ -272,7 +272,7 @@ make_get_return_val(RObj, false = _WantsBinary, #state{op_get = Gets} = S) -> {ok, RObj, S#state{op_get = Gets + 1}}. %% @doc Store an object, yes, sir! --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-type index_spec() :: riak_object:index_spec(). -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> {ok, state()}. put(_Bucket, _PKey, _IndexSpecs, _Val, #state{op_put = Puts} = S) -> diff --git a/src/riak_object.erl b/src/riak_object.erl index e32c84899..e348d304e 100644 --- a/src/riak_object.erl +++ b/src/riak_object.erl @@ -73,8 +73,11 @@ -type r_content() :: #r_content{}. -type index_op() :: add | remove. -type index_value() :: integer() | binary(). +-type index_spec() :: {index_op(), binary(), index_value()}. -type binary_version() :: v0 | v1. +-export_type([index_spec/0]). + -define(MAX_KEY_SIZE, 65536). -define(LASTMOD_LEN, 29). %% static length of rfc1123_date() type. Hard-coded in Erlang. @@ -984,8 +987,7 @@ dvv_enabled(Bucket) -> %% the case where there is no existing object %% stored for a key and therefore no existing %% index data. --spec index_specs(riak_object()) -> - [{index_op(), binary(), index_value()}]. +-spec index_specs(riak_object()) -> [{index_op(), binary(), index_value()}]. index_specs(Obj) -> Indexes = index_data(Obj), assemble_index_specs(Indexes, add).