Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change EXIT_PROCESSOR_SLA_MARGIN to EXIT_PROCESSOR_SLA_SECONDS #1491

Open
wants to merge 34 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
deab148
change EXIT_PROCESSOR_SLA_MARGIN to EXIT_PROCESSOR_SLA_SECONDS
souradeep-das Apr 27, 2020
6174271
remove eth-block-time and credo fix
souradeep-das Apr 27, 2020
db4f1a2
update- change cabbage sla_sec
souradeep-das Apr 27, 2020
85fc0f5
fixed- configuration_api_test
souradeep-das Apr 28, 2020
da7964a
fix barebone test
souradeep-das Apr 28, 2020
8e9a537
dummy
souradeep-das Apr 28, 2020
a250c93
add ETHEREUM_BLOCK_TIME_SECONDS to variables_test_barebone
souradeep-das Apr 29, 2020
a0c7d4f
revert mix lock
souradeep-das Apr 29, 2020
e074347
temp assigning ethereum_block_time_seconds which wont be used later
souradeep-das May 4, 2020
b2157c9
credo fix
souradeep-das May 4, 2020
dc32c7d
lint fix
souradeep-das May 4, 2020
9e42630
update sla_seconds for test.exs
souradeep-das May 5, 2020
43bd892
change: @system_env_name_margin to @system_env_name and @app_env_name…
souradeep-das May 5, 2020
b4c5342
remove single pipeline
souradeep-das May 5, 2020
545034d
revert mix lock
souradeep-das May 5, 2020
3d60044
merge master
souradeep-das May 17, 2020
c8ea7aa
merge changes from master
souradeep-das May 17, 2020
b25e4dc
get timestamp from eth_height
souradeep-das May 18, 2020
1ba1a3d
revert to eth_height_now
souradeep-das May 18, 2020
872f1b5
fix: change eth_height to eth_timestamp and fix tests
souradeep-das May 28, 2020
67d08db
fix: update eth_timestamp_now in finalization and core tests
souradeep-das May 28, 2020
eac77e0
Merge branch 'master' into souradeep/margin_to_seconds
souradeep-das May 28, 2020
84b4472
fix: get_block_timestamp in exit_processor
souradeep-das May 28, 2020
4a98cda
refactor: remove unix time usage
souradeep-das May 29, 2020
2b663c2
fix: get block_timestamp from opts for tests
souradeep-das May 29, 2020
99cf072
style: lint fix
souradeep-das May 29, 2020
8c072f9
style: remove unused vars
souradeep-das Jun 1, 2020
719e7b2
merge master
souradeep-das Jun 1, 2020
34cd035
style: fix lint
souradeep-das Jun 1, 2020
d11ba3e
Merge branch 'master' into souradeep/margin_to_seconds
souradeep-das Jun 8, 2020
49587d8
Merge branch 'master' into souradeep/margin_to_seconds
unnawut Jun 10, 2020
96d9e31
refactor: remove irrelevant guards
souradeep-das Jun 12, 2020
b64b5eb
fix: typo
souradeep-das Jun 12, 2020
c1a0770
style: lint fix
souradeep-das Jun 12, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ defmodule OMG.Status.Monitor.MemoryMonitorTest do
assert_receive :got_clear_alarm
end

test "works with :buffered_memory and :cached_memory values are not provided", context do
test "works with :buffered_memory and :cached_memory values are not provided" do
unnawut marked this conversation as resolved.
Show resolved Hide resolved
set_memsup(total_memory: 1000, free_memory: 100)
assert_receive :got_raise_alarm
end
Expand Down
2 changes: 1 addition & 1 deletion apps/omg_watcher/lib/omg_watcher/api/configuration.ex
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ defmodule OMG.Watcher.API.Configuration do
@spec get_configuration() :: {:ok, map()}
def get_configuration() do
configuration = %{
exit_processor_sla_margin: Configuration.exit_processor_sla_margin(),
exit_processor_sla_seconds: Configuration.exit_processor_sla_seconds(),
deposit_finality_margin: OMG.Configuration.deposit_finality_margin(),
contract_semver: OMG.Eth.Configuration.contract_semver(),
network: OMG.Eth.Configuration.network()
Expand Down
4 changes: 2 additions & 2 deletions apps/omg_watcher/lib/omg_watcher/configuration.ex
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ defmodule OMG.Watcher.Configuration do
Provides access to applications configuration
"""
@app :omg_watcher
def exit_processor_sla_margin() do
Application.fetch_env!(@app, :exit_processor_sla_margin)
def exit_processor_sla_seconds() do
Application.fetch_env!(@app, :exit_processor_sla_seconds)
end

def exit_processor_sla_margin_forced() do
Expand Down
24 changes: 11 additions & 13 deletions apps/omg_watcher/lib/omg_watcher/exit_processor.ex
Original file line number Diff line number Diff line change
Expand Up @@ -251,32 +251,29 @@ defmodule OMG.Watcher.ExitProcessor do
Reads the exit data from `OMG.DB`.

Options:
- `exit_processor_sla_margin`: number of blocks after exit start before it's considered late (and potentially:
- `exit_processor_sla_seconds`: seconds after exit start before it's considered late (and potentially:
unchallenged)
- `exit_processor_sla_margin_forced`: if `true` will override the check of `exit_processor_sla_margin` against
- `exit_processor_sla_margin_forced`: if `true` will override the check of `exit_processor_sla_seconds` against
`min_exit_period_seconds`
- `min_exit_period_seconds`: should reflect the value of this parameter for the specific child chain watched,
- `ethereum_block_time_seconds`: just to relate blocks to seconds for the `exit_processor_sla_margin` check
- `metrics_collection_interval`: how often are the metrics sent to `telemetry` (in milliseconds)
"""
def init(
exit_processor_sla_margin: exit_processor_sla_margin,
exit_processor_sla_seconds: exit_processor_sla_seconds,
exit_processor_sla_margin_forced: exit_processor_sla_margin_forced,
metrics_collection_interval: metrics_collection_interval,
min_exit_period_seconds: min_exit_period_seconds,
ethereum_block_time_seconds: ethereum_block_time_seconds,
child_block_interval: child_block_interval
) do
{:ok, db_exits} = PaymentExitInfo.all_exit_infos()
{:ok, db_ifes} = PaymentExitInfo.all_in_flight_exits_infos()
{:ok, db_competitors} = DB.competitors_info()

:ok =
Core.check_sla_margin(
exit_processor_sla_margin,
Core.check_sla_seconds(
exit_processor_sla_seconds,
exit_processor_sla_margin_forced,
min_exit_period_seconds,
ethereum_block_time_seconds
min_exit_period_seconds
)

{:ok, processor} =
Expand All @@ -286,7 +283,7 @@ defmodule OMG.Watcher.ExitProcessor do
db_competitors,
min_exit_period_seconds,
child_block_interval,
exit_processor_sla_margin
exit_processor_sla_seconds
)

{:ok, _} = :timer.send_interval(metrics_collection_interval, self(), :send_metrics)
Expand Down Expand Up @@ -630,12 +627,13 @@ defmodule OMG.Watcher.ExitProcessor do
Core.find_ifes_in_blocks(state, prepared_request)
end

defp run_status_gets(%ExitProcessor.Request{eth_height_now: nil, blknum_now: nil} = request) do
defp run_status_gets(%ExitProcessor.Request{eth_timestamp_now: nil, blknum_now: nil} = request) do
{:ok, eth_height_now} = EthereumHeight.get()
{:ok, eth_timestamp_now} = Eth.get_block_timestamp_by_number(eth_height_now)
{blknum_now, _} = State.get_status()

_ = Logger.debug("eth_height_now: #{inspect(eth_height_now)}, blknum_now: #{inspect(blknum_now)}")
%{request | eth_height_now: eth_height_now, blknum_now: blknum_now}
_ = Logger.debug("eth_timestamp_now: #{inspect(eth_timestamp_now)}, blknum_now: #{inspect(blknum_now)}")
%{request | eth_timestamp_now: eth_timestamp_now, blknum_now: blknum_now}
end

defp get_utxo_existence(%ExitProcessor.Request{utxos_to_check: positions} = request),
Expand Down
10 changes: 5 additions & 5 deletions apps/omg_watcher/lib/omg_watcher/exit_processor/canonicity.ex
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,11 @@ defmodule OMG.Watcher.ExitProcessor.Canonicity do

@doc """
Returns a tuple with byzantine events: first element is a list of events for ifes with competitor
and the second is the same list filtered for late ifes past sla margin
and the second is the same list filtered for late ifes past sla seconds
"""
@spec get_ife_txs_with_competitors(Core.t(), KnownTx.known_txs_by_input_t(), pos_integer()) ::
{list(Event.NonCanonicalIFE.t()), list(Event.UnchallengedNonCanonicalIFE.t())}
def get_ife_txs_with_competitors(state, known_txs_by_input, eth_height_now) do
def get_ife_txs_with_competitors(state, known_txs_by_input, eth_timestamp_now) do
non_canonical_ifes =
state.in_flight_exits
|> Map.values()
Expand All @@ -87,13 +87,13 @@ defmodule OMG.Watcher.ExitProcessor.Canonicity do
|> Enum.uniq()
|> Enum.map(fn txbytes -> %Event.NonCanonicalIFE{txbytes: txbytes} end)

past_sla_margin = fn {ife, _double_spend} ->
ife.eth_height + state.sla_margin <= eth_height_now
past_sla_seconds = fn {ife, _double_spend} ->
ife.timestamp + state.sla_seconds <= eth_timestamp_now
end

late_non_canonical_ife_events =
non_canonical_ifes
|> Stream.filter(past_sla_margin)
|> Stream.filter(past_sla_seconds)
|> Stream.map(fn {ife, _double_spend} -> Transaction.raw_txbytes(ife.tx) end)
|> Enum.uniq()
|> Enum.map(fn txbytes -> %Event.UnchallengedNonCanonicalIFE{txbytes: txbytes} end)
Expand Down
47 changes: 23 additions & 24 deletions apps/omg_watcher/lib/omg_watcher/exit_processor/core.ex
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,7 @@ defmodule OMG.Watcher.ExitProcessor.Core do

use OMG.Utils.LoggerExt

@default_sla_margin 10

@default_sla_seconds 10
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure should the value reflect the original value, which I think should be 10 block * 15s per block = 150?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this only takes place on tests. @souradeep-das can we confirm this - raise a value and whether tests fails?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

true, I think these are for tests. Fails when sla_seconds is >10

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if this is for tests then let's remove it. we should also remove the default setting below:
sla_seconds \\ @default_sla_seconds

@zero_address OMG.Eth.zero_address()

@max_inputs Transaction.Payment.max_inputs()
Expand All @@ -70,7 +69,7 @@ defmodule OMG.Watcher.ExitProcessor.Core do
@type new_piggyback_event_t() :: new_piggyback_input_event_t() | new_piggyback_output_event_t()

defstruct [
:sla_margin,
:sla_seconds,
:min_exit_period_seconds,
:child_block_interval,
exits: %{},
Expand All @@ -80,7 +79,7 @@ defmodule OMG.Watcher.ExitProcessor.Core do
]

@type t :: %__MODULE__{
sla_margin: non_neg_integer(),
sla_seconds: non_neg_integer(),
exits: %{Utxo.Position.t() => ExitInfo.t()},
in_flight_exits: %{Transaction.tx_hash() => InFlightExitInfo.t()},
# NOTE: maps only standard exit_ids to the natural keys of standard exits (input pointers/utxo_pos)
Expand Down Expand Up @@ -114,15 +113,15 @@ defmodule OMG.Watcher.ExitProcessor.Core do
db_competitors :: [{Transaction.tx_hash(), CompetitorInfo.t()}],
min_exit_period_seconds :: non_neg_integer(),
child_block_interval :: non_neg_integer,
sla_margin :: non_neg_integer
sla_seconds :: non_neg_integer
) :: {:ok, t()}
def init(
db_exits,
db_in_flight_exits,
db_competitors,
min_exit_period_seconds,
child_block_interval,
sla_margin \\ @default_sla_margin
sla_seconds \\ @default_sla_seconds
) do
exits = db_exits |> Enum.map(&ExitInfo.from_db_kv/1) |> Map.new()

Expand All @@ -134,32 +133,32 @@ defmodule OMG.Watcher.ExitProcessor.Core do
in_flight_exits: db_in_flight_exits |> Enum.map(&InFlightExitInfo.from_db_kv/1) |> Map.new(),
exit_ids: exit_ids,
competitors: db_competitors |> Enum.map(&CompetitorInfo.from_db_kv/1) |> Map.new(),
sla_margin: sla_margin,
sla_seconds: sla_seconds,
min_exit_period_seconds: min_exit_period_seconds,
child_block_interval: child_block_interval
}}
end

@doc """
Use to check if the settings regarding the `:exit_processor_sla_margin` config of `:omg_watcher` are OK.
Use to check if the settings regarding the `:exit_processor_sla_seconds` config of `:omg_watcher` are OK.

Since there are combinations of our configuration that may lead to a dangerous setup of the Watcher
(in particular - muting the reports of unchallenged_exits), we're enforcing that the `exit_processor_sla_margin`
(in particular - muting the reports of unchallenged_exits), we're enforcing that the `exit_processor_sla_seconds`
be not larger than `min_exit_period`.
"""
@spec check_sla_margin(pos_integer(), boolean(), pos_integer(), pos_integer()) :: :ok | {:error, :sla_margin_too_big}
def check_sla_margin(sla_margin, sla_margin_forced, min_exit_period_seconds, ethereum_block_time_seconds)
@spec check_sla_seconds(pos_integer(), boolean(), pos_integer()) :: :ok | {:error, :sla_margin_too_big}
def check_sla_seconds(sla_seconds, sla_margin_forced, min_exit_period_seconds)

def check_sla_margin(sla_margin, true, min_exit_period_seconds, ethereum_block_time_seconds) do
def check_sla_seconds(sla_seconds, true, min_exit_period_seconds) do
_ =
if !sla_margin_safe?(sla_margin, min_exit_period_seconds, ethereum_block_time_seconds),
do: Logger.warn("Allowing unsafe sla margin of #{sla_margin} blocks")
if !sla_seconds_safe?(sla_seconds, min_exit_period_seconds),
do: Logger.warn("Allowing unsafe sla margin of #{sla_seconds} seconds")

:ok
end

def check_sla_margin(sla_margin, false, min_exit_period_seconds, ethereum_block_time_seconds) do
if sla_margin_safe?(sla_margin, min_exit_period_seconds, ethereum_block_time_seconds),
def check_sla_seconds(sla_seconds, false, min_exit_period_seconds) do
if sla_seconds_safe?(sla_seconds, min_exit_period_seconds),
do: :ok,
else: {:error, :sla_margin_too_big}
end
Expand Down Expand Up @@ -460,17 +459,16 @@ defmodule OMG.Watcher.ExitProcessor.Core do
@spec check_validity(ExitProcessor.Request.t(), t()) :: check_validity_result_t()
def check_validity(
%ExitProcessor.Request{
eth_height_now: eth_height_now,
eth_timestamp_now: eth_timestamp_now,
utxos_to_check: utxos_to_check,
utxo_exists_result: utxo_exists_result,
blocks_result: blocks
},
%__MODULE__{} = state
)
when not is_nil(eth_height_now) do
) do
utxo_exists? = Enum.zip(utxos_to_check, utxo_exists_result) |> Map.new()

{invalid_exits, late_invalid_exits} = StandardExit.get_invalid(state, utxo_exists?, eth_height_now)
{invalid_exits, late_invalid_exits} = StandardExit.get_invalid(state, utxo_exists?, eth_timestamp_now)

invalid_exit_events =
invalid_exits
Expand All @@ -483,12 +481,12 @@ defmodule OMG.Watcher.ExitProcessor.Core do
known_txs_by_input = KnownTx.get_all_from_blocks_appendix(blocks, state)

{non_canonical_ife_events, late_non_canonical_ife_events} =
ExitProcessor.Canonicity.get_ife_txs_with_competitors(state, known_txs_by_input, eth_height_now)
ExitProcessor.Canonicity.get_ife_txs_with_competitors(state, known_txs_by_input, eth_timestamp_now)

invalid_ife_challenges_events = ExitProcessor.Canonicity.get_invalid_ife_challenges(state)

{invalid_piggybacks_events, late_invalid_piggybacks_events} =
ExitProcessor.Piggyback.get_invalid_piggybacks_events(state, known_txs_by_input, eth_height_now)
ExitProcessor.Piggyback.get_invalid_piggybacks_events(state, known_txs_by_input, eth_timestamp_now)

available_piggybacks_events =
state
Expand Down Expand Up @@ -627,6 +625,7 @@ defmodule OMG.Watcher.ExitProcessor.Core do
address != @zero_address
end

defp sla_margin_safe?(exit_processor_sla_margin, min_exit_period_seconds, ethereum_block_time_seconds),
do: exit_processor_sla_margin * ethereum_block_time_seconds < min_exit_period_seconds
defp sla_seconds_safe?(exit_processor_sla_seconds, min_exit_period_seconds) do
exit_processor_sla_seconds < min_exit_period_seconds
end
end
14 changes: 7 additions & 7 deletions apps/omg_watcher/lib/omg_watcher/exit_processor/piggyback.ex
Original file line number Diff line number Diff line change
Expand Up @@ -87,15 +87,15 @@ defmodule OMG.Watcher.ExitProcessor.Piggyback do
end

@doc """
Returns a tuple of ivalid piggybacks and invalid piggybacks that are past SLA margin.
This is inclusive, invalid piggybacks past SLA margin are included in the invalid piggybacks list.
Returns a tuple of ivalid piggybacks and invalid piggybacks that are past SLA seconds.
This is inclusive, invalid piggybacks past SLA seconds are included in the invalid piggybacks list.
"""
@spec get_invalid_piggybacks_events(Core.t(), KnownTx.known_txs_by_input_t(), pos_integer()) ::
{list(Event.InvalidPiggyback.t()), list(Event.UnchallengedPiggyback.t())}
def get_invalid_piggybacks_events(
%Core{sla_margin: sla_margin, in_flight_exits: ifes},
%Core{sla_seconds: sla_seconds, in_flight_exits: ifes},
known_txs_by_input,
eth_height_now
eth_timestamp_now
) do
invalid_piggybacks_by_ife =
ifes
Expand All @@ -104,13 +104,13 @@ defmodule OMG.Watcher.ExitProcessor.Piggyback do

invalid_piggybacks_events = to_events(invalid_piggybacks_by_ife, &to_invalid_piggyback_event/1)

past_sla_margin = fn {ife, _type, _materials} ->
ife.eth_height + sla_margin <= eth_height_now
past_sla_seconds = fn {ife, _type, _materials} ->
ife.timestamp + sla_seconds <= eth_timestamp_now
end

unchallenged_piggybacks_events =
invalid_piggybacks_by_ife
|> Enum.filter(past_sla_margin)
|> Enum.filter(past_sla_seconds)
|> to_events(&to_unchallenged_piggyback_event/1)

{invalid_piggybacks_events, unchallenged_piggybacks_events}
Expand Down
4 changes: 2 additions & 2 deletions apps/omg_watcher/lib/omg_watcher/exit_processor/request.ex
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ defmodule OMG.Watcher.ExitProcessor.Request do
alias OMG.Utxo

defstruct [
:eth_height_now,
:eth_timestamp_now,
:blknum_now,
utxos_to_check: [],
spends_to_get: [],
Expand All @@ -41,7 +41,7 @@ defmodule OMG.Watcher.ExitProcessor.Request do
]

@type t :: %__MODULE__{
eth_height_now: nil | pos_integer,
eth_timestamp_now: nil | pos_integer,
blknum_now: nil | pos_integer,
utxos_to_check: list(Utxo.Position.t()),
spends_to_get: list(Utxo.Position.t()),
Expand Down
10 changes: 7 additions & 3 deletions apps/omg_watcher/lib/omg_watcher/exit_processor/standard_exit.ex
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,11 @@ defmodule OMG.Watcher.ExitProcessor.StandardExit do
"""
@spec get_invalid(Core.t(), %{Utxo.Position.t() => boolean}, pos_integer()) ::
{%{Utxo.Position.t() => ExitInfo.t()}, %{Utxo.Position.t() => ExitInfo.t()}}
def get_invalid(%Core{sla_margin: sla_margin} = state, utxo_exists?, eth_height_now) do
def get_invalid(
state,
utxo_exists?,
eth_timestamp_now
) do
active_exits = active_exits(state)

exits_invalid_by_ife =
Expand All @@ -93,8 +97,8 @@ defmodule OMG.Watcher.ExitProcessor.StandardExit do

# get exits which are still invalid and after the SLA margin
late_invalid_exits =
Enum.filter(invalid_exits, fn {_, %ExitInfo{eth_height: eth_height}} ->
eth_height + sla_margin <= eth_height_now
Enum.filter(invalid_exits, fn {_, %ExitInfo{block_timestamp: block_timestamp}} ->
block_timestamp + state.sla_seconds <= eth_timestamp_now
end)

{Map.new(invalid_exits), Map.new(late_invalid_exits)}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ defmodule OMG.Watcher.ReleaseTasks.SetExitProcessorSLAMargin do
require Logger
@app :omg_watcher

@system_env_name_margin "EXIT_PROCESSOR_SLA_MARGIN"
@app_env_name_margin :exit_processor_sla_margin
@system_env_name "EXIT_PROCESSOR_SLA_SECONDS"
@app_env_name :exit_processor_sla_seconds

@system_env_name_force "EXIT_PROCESSOR_SLA_MARGIN_FORCED"
@app_env_name_force :exit_processor_sla_margin_forced
Expand All @@ -33,15 +33,15 @@ defmodule OMG.Watcher.ReleaseTasks.SetExitProcessorSLAMargin do

Config.Reader.merge(config,
omg_watcher: [
exit_processor_sla_margin: get_exit_processor_sla_margin(),
exit_processor_sla_seconds: get_exit_processor_sla_seconds(),
exit_processor_sla_margin_forced: get_exit_processor_sla_forced()
]
)
end

defp get_exit_processor_sla_margin() do
config_value = validate_int(get_env(@system_env_name_margin), Application.get_env(@app, @app_env_name_margin))
_ = Logger.info("CONFIGURATION: App: #{@app} Key: #{@system_env_name_margin} Value: #{inspect(config_value)}.")
defp get_exit_processor_sla_seconds() do
config_value = validate_int(get_env(@system_env_name), Application.get_env(@app, @app_env_name))
_ = Logger.info("CONFIGURATION: App: #{@app} Key: #{@system_env_name} Value: #{inspect(config_value)}.")
config_value
end

Expand All @@ -66,7 +66,7 @@ defmodule OMG.Watcher.ReleaseTasks.SetExitProcessorSLAMargin do
defp to_int(value) do
case Integer.parse(value) do
{result, ""} -> result
_ -> exit("#{@system_env_name_margin} must be an integer.")
_ -> exit("#{@system_env_name} must be an integer.")
end
end
end
Loading