From 6b1d4839d846c786ab0e864165aab1dbe5ee3b67 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 7 Jul 2020 20:17:36 +0200 Subject: [PATCH 01/19] update block operations generator --- tests/generators/operations/main.py | 61 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 6d4f6d1395..b8493a21bc 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,25 +1,16 @@ from typing import Iterable -from eth2spec.test.phase0.block_processing import ( - test_process_attestation, - test_process_attester_slashing, - test_process_block_header, - test_process_deposit, - test_process_proposer_slashing, - test_process_voluntary_exit, -) - +from importlib import import_module from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests from importlib import reload from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0 - +from eth2spec.test.context import PHASE0, PHASE1 -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -27,28 +18,46 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='operations', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("operations", [ - create_provider('attestation', test_process_attestation, 'minimal'), - create_provider('attestation', test_process_attestation, 'mainnet'), - create_provider('attester_slashing', test_process_attester_slashing, 'minimal'), - create_provider('attester_slashing', test_process_attester_slashing, 'mainnet'), - create_provider('block_header', test_process_block_header, 'minimal'), - create_provider('block_header', test_process_block_header, 'mainnet'), - create_provider('deposit', test_process_deposit, 'minimal'), - create_provider('deposit', test_process_deposit, 'mainnet'), - create_provider('proposer_slashing', test_process_proposer_slashing, 'minimal'), - create_provider('proposer_slashing', test_process_proposer_slashing, 'mainnet'), - create_provider('voluntary_exit', test_process_voluntary_exit, 'minimal'), - create_provider('voluntary_exit', test_process_voluntary_exit, 'mainnet'), + phase_0_mods = [(key, 'eth2spec.test.phase0.block_processing.test_process_' + key) for key in [ + 'attestation', + 'attester_slashing', + 'block_header', + 'deposit', + 'proposer_slashing', + 'voluntary_exit', + ]] + phase_1_mods = [(key, 'eth2spec.test.phase1.block_processing.test_process_' + key) for key in [ + 'attestation', + 'chunk_challenge', + 'custody_key_reveal', + 'custody_slashing', + 'early_derived_secret_reveal', + 'shard_transition', + ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) + + gen_runner.run_generator(f"phase0/operations", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods + ]) + # TODO: disabled for testing + # gen_runner.run_generator(f"phase0/operations", [ + # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + # ]) + gen_runner.run_generator(f"phase1/operations", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) + # Disabled for now + # gen_runner.run_generator(f"phase1/operations", [ + # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + # ]) From 3c374b1e1664a6bb349e91ab7c05a70818792ee6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 7 Jul 2020 21:21:43 +0200 Subject: [PATCH 02/19] Update test gen to output phase 1 test vectors. Mainnet disabled for now. --- tests/core/gen_helpers/gen_base/gen_runner.py | 2 + .../pyspec/eth2spec/debug/random_value.py | 5 +- .../test/phase1/epoch_processing/__init__.py | 0 tests/generators/epoch_processing/main.py | 59 +++++++++++-------- tests/generators/operations/main.py | 11 ++-- tests/generators/rewards/main.py | 53 ++++++++++------- tests/generators/sanity/main.py | 40 +++++++++---- tests/generators/ssz_static/main.py | 40 +++++++------ 8 files changed, 126 insertions(+), 84 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase1/epoch_processing/__init__.py diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/gen_helpers/gen_base/gen_runner.py index 32f3594b35..dbe2b406a7 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/gen_helpers/gen_base/gen_runner.py @@ -2,6 +2,7 @@ from pathlib import Path import sys from typing import Iterable, AnyStr, Any, Callable +import traceback from ruamel.yaml import ( YAML, @@ -152,6 +153,7 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): except Exception as e: print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") + traceback.print_exc() print(f"completed {generator_name}") diff --git a/tests/core/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py index 9fc3be9788..33eb25c0c6 100644 --- a/tests/core/pyspec/eth2spec/debug/random_value.py +++ b/tests/core/pyspec/eth2spec/debug/random_value.py @@ -68,9 +68,8 @@ def get_random_ssz_object(rng: Random, else: return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.limit())))) if issubclass(typ, ByteVector): - # Sanity, don't generate absurdly big random values - # If a client is aiming to performance-test, they should create a benchmark suite. - assert typ.type_byte_length() <= max_bytes_length + # Random byte vectors can be bigger than max bytes size, e.g. custody chunk data. + # No max-bytes-length limitation here. if mode == RandomizationMode.mode_zero: return typ(b'\x00' * typ.type_byte_length()) elif mode == RandomizationMode.mode_max: diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/__init__.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 13ac76f419..c47f63fc13 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,23 +1,15 @@ from typing import Iterable -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.phase0.epoch_processing import ( - test_process_final_updates, - test_process_justification_and_finalization, - test_process_registry_updates, - test_process_rewards_and_penalties, - test_process_slashings -) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from importlib import reload +from importlib import reload, import_module from eth2spec.config import config_util -from eth2spec.test.context import PHASE0 - +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -25,27 +17,42 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='epoch_processing', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('final_updates', test_process_final_updates, 'minimal'), - create_provider('final_updates', test_process_final_updates, 'mainnet'), - create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), - create_provider('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'), - create_provider('registry_updates', test_process_registry_updates, 'minimal'), - create_provider('registry_updates', test_process_registry_updates, 'mainnet'), - create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'minimal'), - # runs full epochs filled with data, with uncached ssz. Disabled for now. - # create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'mainnet'), - create_provider('slashings', test_process_slashings, 'minimal'), - create_provider('slashings', test_process_slashings, 'mainnet'), + phase_0_mods = [(key, 'eth2spec.test.phase0.epoch_processing.test_process_' + key) for key in [ + 'final_updates', + 'justification_and_finalization', + 'registry_updates', + 'rewards_and_penalties', + 'slashings', + ]] + phase_1_mods = [(key, 'eth2spec.test.phase1.epoch_processing.test_process_' + key) for key in [ + 'challenge_deadlines', + 'custody_final_updates', + 'reveal_deadlines', + ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) + + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods + ]) + # TODO: disabled for testing + # gen_runner.run_generator(f"epoch_processing", [ + # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + # ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) + # Disabled for now + # gen_runner.run_generator(f"epoch_processing", [ + # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + # ]) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index b8493a21bc..165bb66c99 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,9 +1,8 @@ from typing import Iterable -from importlib import import_module from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from importlib import reload +from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 @@ -47,17 +46,17 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: 'shard_transition', ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"phase0/operations", [ + gen_runner.run_generator(f"operations", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) # TODO: disabled for testing - # gen_runner.run_generator(f"phase0/operations", [ + # gen_runner.run_generator(f"operations", [ # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods # ]) - gen_runner.run_generator(f"phase1/operations", [ + gen_runner.run_generator(f"operations", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) # Disabled for now - # gen_runner.run_generator(f"phase1/operations", [ + # gen_runner.run_generator(f"operations", [ # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods # ]) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 5d063c4340..0e6f6dfe71 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,21 +1,15 @@ from typing import Iterable -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.phase0.rewards import ( - test_basic, - test_leak, - test_random, -) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from importlib import reload +from importlib import reload, import_module from eth2spec.config import config_util -from eth2spec.test.context import PHASE0 - +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 -def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -23,22 +17,41 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='rewards', - handler_name='core', + handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("rewards", [ - create_provider(test_basic, 'minimal'), - create_provider(test_basic, 'mainnet'), - create_provider(test_leak, 'minimal'), - create_provider(test_leak, 'mainnet'), - create_provider(test_random, 'minimal'), - create_provider(test_random, 'mainnet'), + phase_0_mods = [(key, 'eth2spec.test.phase0.rewards.test_'+key) for key in [ + 'basic', + 'basic', + 'leak', + 'leak', + 'random', + 'random', + ]] + # No additional phase 1 specific rewards tests, yet. + phase_1_mods = phase_0_mods + + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods + ]) + # TODO: disabled for testing + # gen_runner.run_generator(f"rewards", [ + # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + # ]) + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) + # Disabled for now + # gen_runner.run_generator(f"rewards", [ + # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + # ]) + diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 45a1c8c4f5..147467e916 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,18 +1,15 @@ from typing import Iterable -from importlib import reload from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests - -from eth2spec.test.context import PHASE0 -from eth2spec.test.phase0.sanity import test_blocks, test_slots +from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: - +def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -20,20 +17,39 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='sanity', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("sanity", [ - create_provider('blocks', test_blocks, 'minimal'), - create_provider('blocks', test_blocks, 'mainnet'), - create_provider('slots', test_slots, 'minimal'), - create_provider('slots', test_slots, 'mainnet'), + phase_0_mods = [(key, 'eth2spec.test.phase0.sanity.test_' + key) for key in [ + 'blocks', + 'slots', + ]] + phase_1_mods = [(key, 'eth2spec.test.phase1.sanity.test_' + key) for key in [ + 'blocks', # more phase 1 specific block tests + 'shard_blocks', + ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) + + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) + # TODO: disabled for testing + # gen_runner.run_generator(f"sanity", [ + # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + # ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods + ]) + # Disabled for now + # gen_runner.run_generator(f"sanity", [ + # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + # ]) + diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index b9cb51db08..8dfd0cc86e 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -7,8 +7,9 @@ from eth2spec.debug import random_value, encode from eth2spec.config import config_util -from eth2spec.phase0 import spec -from eth2spec.test.context import PHASE0 +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -16,7 +17,7 @@ ) -MAX_BYTES_LENGTH = 100 +MAX_BYTES_LENGTH = 1000 MAX_LIST_LENGTH = 10 @@ -30,14 +31,14 @@ def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, cha yield "roots", "data", roots_data -def get_spec_ssz_types(): +def get_spec_ssz_types(spec): return [ (name, value) for (name, value) in getmembers(spec, isclass) if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class ] -def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int): +def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int): random_mode_name = mode.to_name() # Reproducible RNG @@ -45,7 +46,7 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization for i in range(count): yield gen_typing.TestCase( - fork_name=PHASE0, + fork_name=fork_name, runner_name='ssz_static', handler_name=name, suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}", @@ -54,19 +55,23 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization ) -def create_provider(config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, +def create_provider(fork_name, config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: # Apply changes to presets, this affects some of the vector types. config_util.prepare_config(configs_path, config_name) - reload(spec) + reload(spec_phase0) + reload(spec_phase1) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: count = cases_if_random if chaos or mode.is_changing() else 1 + spec = spec_phase0 + if fork_name == PHASE1: + spec = spec_phase1 - for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types()): - yield from ssz_static_cases(seed * 1000 + i, name, ssz_type, mode, chaos, count) + for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)): + yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -80,10 +85,11 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: seed += 1 settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30)) seed += 1 - settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) - seed += 1 - - gen_runner.run_generator("ssz_static", [ - create_provider(config_name, seed, mode, chaos, cases_if_random) - for (seed, config_name, mode, chaos, cases_if_random) in settings - ]) + # settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) + # seed += 1 + + for fork in [PHASE0, PHASE1]: + gen_runner.run_generator("ssz_static", [ + create_provider(fork, config_name, seed, mode, chaos, cases_if_random) + for (seed, config_name, mode, chaos, cases_if_random) in settings + ]) From 60f4bd228865258965010ed734c409f9e42284c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 Jul 2020 21:59:01 +0800 Subject: [PATCH 03/19] The AttesterSlashing tests for phase1 can be enabled now after #1889 --- .../test_process_attester_slashing.py | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py index 192e0390d1..82d4903117 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py @@ -1,6 +1,5 @@ from eth2spec.test.context import ( - PHASE0, - spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases + spec_state_test, expect_assertion_error, always_bls, with_all_phases ) from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ @@ -194,10 +193,7 @@ def test_participants_already_slashed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -# Some of the following tests are phase0 only: phase 1 lists participants with bitfields instead of index list. - - -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att1_high_index(spec, state): @@ -210,7 +206,7 @@ def test_att1_high_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att2_high_index(spec, state): @@ -223,7 +219,7 @@ def test_att2_high_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att1_empty_indices(spec, state): @@ -235,7 +231,7 @@ def test_att1_empty_indices(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att2_empty_indices(spec, state): @@ -247,7 +243,7 @@ def test_att2_empty_indices(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_all_empty_indices(spec, state): @@ -262,7 +258,7 @@ def test_all_empty_indices(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att1_bad_extra_index(spec, state): @@ -278,7 +274,7 @@ def test_att1_bad_extra_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att1_bad_replaced_index(spec, state): @@ -294,7 +290,7 @@ def test_att1_bad_replaced_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att2_bad_extra_index(spec, state): @@ -310,7 +306,7 @@ def test_att2_bad_extra_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att2_bad_replaced_index(spec, state): @@ -326,7 +322,7 @@ def test_att2_bad_replaced_index(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att1_duplicate_index_normal_signed(spec, state): @@ -346,7 +342,7 @@ def test_att1_duplicate_index_normal_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att2_duplicate_index_normal_signed(spec, state): @@ -366,7 +362,7 @@ def test_att2_duplicate_index_normal_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att1_duplicate_index_double_signed(spec, state): @@ -381,7 +377,7 @@ def test_att1_duplicate_index_double_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test @always_bls def test_att2_duplicate_index_double_signed(spec, state): @@ -396,7 +392,7 @@ def test_att2_duplicate_index_double_signed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test def test_unsorted_att_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) @@ -409,7 +405,7 @@ def test_unsorted_att_1(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases([PHASE0]) +@with_all_phases @spec_state_test def test_unsorted_att_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) From 52ba90d7195cb3892cf0c9396518495237835d8c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 11 Aug 2020 16:19:20 +0800 Subject: [PATCH 04/19] Skip the testcases that don't support the fork or the config --- tests/core/gen_helpers/gen_base/gen_runner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/gen_helpers/gen_base/gen_runner.py index dbe2b406a7..da60567f90 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/gen_helpers/gen_base/gen_runner.py @@ -135,7 +135,13 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): written_part = False meta = dict() - for (name, out_kind, data) in test_case.case_fn(): + + output = test_case.case_fn() + # If the output is `None`, that means the testcase doesn't support this fork or this config. + if output is None: + continue + + for (name, out_kind, data) in output: written_part = True if out_kind == "meta": meta[name] = data From 83760b15ac1dc7833a77e03559f666cc75945b19 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 10 Aug 2020 20:54:22 +0800 Subject: [PATCH 05/19] Use v-guide's `get_custody_secret` --- tests/core/pyspec/eth2spec/test/helpers/custody.py | 9 --------- .../block_processing/test_process_custody_slashing.py | 9 +++++++-- .../pyspec/eth2spec/test/phase1/sanity/test_blocks.py | 9 +++++++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 162ca32639..b3a8c0a957 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -179,15 +179,6 @@ def get_sample_shard_transition(spec, start_slot, block_lengths): return shard_transition -def get_custody_secret(spec, state, validator_index, epoch=None): - period = spec.get_custody_period_for_validator(validator_index, epoch if epoch is not None - else spec.get_current_epoch(state)) - epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, validator_index) - domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) - signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) - return bls.Sign(privkeys[validator_index], signing_root) - - def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=True): test_vector = get_custody_test_vector(length) offset = 0 diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index b08b654130..f8c92cb44f 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -1,11 +1,11 @@ from eth2spec.test.helpers.custody import ( get_valid_custody_slashing, - get_custody_secret, get_custody_slashable_shard_transition, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, ) +from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.ssz.ssz_typing import ByteList from eth2spec.test.helpers.state import get_balance, transition_to from eth2spec.test.context import ( @@ -77,7 +77,12 @@ def run_standard_custody_slashing_test(spec, if block_lengths is None: block_lengths = [2**15 // 3] * len(offset_slots) - custody_secret = get_custody_secret(spec, state, validator_index) + custody_secret = spec.get_custody_secret( + state, + validator_index, + privkeys[validator_index], + spec.get_current_epoch(state), + ) shard_transition, slashable_test_vector = get_custody_slashable_shard_transition( spec, state.slot, diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py index 5fe3394db0..1ddbf1c6bd 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py @@ -8,7 +8,6 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.custody import ( - get_custody_secret, get_custody_slashable_test_vector, get_valid_chunk_challenge, get_valid_custody_chunk_response, @@ -16,6 +15,7 @@ get_valid_custody_slashing, get_valid_early_derived_secret_reveal, ) +from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.shard_block import ( build_shard_block, get_committee_index_of_shard, @@ -215,7 +215,12 @@ def test_custody_slashing(spec, state): committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) # Create slashable shard block body validator_index = spec.get_beacon_committee(state, state.slot, committee_index)[0] - custody_secret = get_custody_secret(spec, state, validator_index) + custody_secret = spec.get_custody_secret( + state, + validator_index, + privkeys[validator_index], + spec.get_current_epoch(state), + ) slashable_body = get_custody_slashable_test_vector(spec, custody_secret, length=100, slashable=True) shard_block = build_shard_block(spec, state, shard, body=slashable_body, slot=state.slot, signed=True) shard_block_dict: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} From 62896642609fc78a0b02173f6b8f215a6634c4a8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 12 Aug 2020 20:48:52 +0800 Subject: [PATCH 06/19] Update testing context 1. Add `with_configs` decorator to assign available configs 2. Add `only_full_crosslink` decorator to detect if the configuation can do full crosslinking 3. Add `context.is_pytest` flag: True if calling via pytest. False if calling from test generator. --- configs/mainnet/phase0.yaml | 1 + configs/mainnet/phase1.yaml | 1 + configs/minimal/phase0.yaml | 1 + configs/minimal/phase1.yaml | 1 + setup.py | 4 ++ tests/core/gen_helpers/gen_base/gen_runner.py | 29 ++++++---- tests/core/gen_helpers/requirements.txt | 1 + tests/core/gen_helpers/setup.py | 3 +- .../pyspec/eth2spec/config/config_util.py | 2 + tests/core/pyspec/eth2spec/test/context.py | 50 +++++++++++++++- tests/core/pyspec/eth2spec/test/exceptions.py | 2 + .../test/helpers/shard_transitions.py | 5 -- .../test_process_chunk_challenge.py | 4 +- .../test_process_shard_transition.py | 22 ++----- .../test/phase1/sanity/test_blocks.py | 30 +++------- .../test/phase1/sanity/test_shard_blocks.py | 58 ++++--------------- .../fork_choice/test_on_shard_block.py | 13 +---- tests/generators/sanity/main.py | 1 - 18 files changed, 111 insertions(+), 117 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/exceptions.py diff --git a/configs/mainnet/phase0.yaml b/configs/mainnet/phase0.yaml index 616baaf345..ad05db0128 100644 --- a/configs/mainnet/phase0.yaml +++ b/configs/mainnet/phase0.yaml @@ -2,6 +2,7 @@ # Note: the intention of this file (for now) is to illustrate what a mainnet configuration could look like. # Some of these constants may still change before the launch of Phase 0. +CONFIG_NAME: "mainnet" # Misc # --------------------------------------------------------------- diff --git a/configs/mainnet/phase1.yaml b/configs/mainnet/phase1.yaml index 0e986079e2..1d689f2b56 100644 --- a/configs/mainnet/phase1.yaml +++ b/configs/mainnet/phase1.yaml @@ -1,5 +1,6 @@ # Mainnet preset - phase 1 +CONFIG_NAME: "mainnet" # phase1-fork # --------------------------------------------------------------- diff --git a/configs/minimal/phase0.yaml b/configs/minimal/phase0.yaml index 78cd601770..38b5c85d60 100644 --- a/configs/minimal/phase0.yaml +++ b/configs/minimal/phase0.yaml @@ -1,5 +1,6 @@ # Minimal preset +CONFIG_NAME: "minimal" # Misc # --------------------------------------------------------------- diff --git a/configs/minimal/phase1.yaml b/configs/minimal/phase1.yaml index b9fce9dda9..cf17ba52a7 100644 --- a/configs/minimal/phase1.yaml +++ b/configs/minimal/phase1.yaml @@ -1,5 +1,6 @@ # Minimal preset - phase 1 +CONFIG_NAME: "minimal" # phase1-fork # --------------------------------------------------------------- diff --git a/setup.py b/setup.py index 427f08bc63..fc28c3ea04 100644 --- a/setup.py +++ b/setup.py @@ -119,6 +119,8 @@ def get_spec(file_name: str) -> SpecObject: from eth2spec.utils.hash_function import hash SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' ''' PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config @@ -150,6 +152,8 @@ def get_spec(file_name: str) -> SpecObject: SSZVariableName = str GeneralizedIndex = NewType('GeneralizedIndex', int) SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' ''' SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: uint64) -> int: diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/gen_helpers/gen_base/gen_runner.py index da60567f90..a22073c00a 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/gen_helpers/gen_base/gen_runner.py @@ -10,6 +10,13 @@ from gen_base.gen_typing import TestProvider +from eth2spec.test import context +from eth2spec.test.exceptions import SkippedTest + + +# Flag that the runner does NOT run test via pytest +context.is_pytest = False + def validate_output_dir(path_str): path = Path(path_str) @@ -136,19 +143,19 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): written_part = False meta = dict() - output = test_case.case_fn() - # If the output is `None`, that means the testcase doesn't support this fork or this config. - if output is None: + try: + for (name, out_kind, data) in test_case.case_fn(): + written_part = True + if out_kind == "meta": + meta[name] = data + if out_kind == "data": + output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml)) + if out_kind == "ssz": + output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) + except SkippedTest as e: + print(e) continue - for (name, out_kind, data) in output: - written_part = True - if out_kind == "meta": - meta[name] = data - if out_kind == "data": - output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml)) - if out_kind == "ssz": - output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) # Once all meta data is collected (if any), write it to a meta data file. if len(meta) != 0: written_part = True diff --git a/tests/core/gen_helpers/requirements.txt b/tests/core/gen_helpers/requirements.txt index dc3f619042..c972f9b7bb 100644 --- a/tests/core/gen_helpers/requirements.txt +++ b/tests/core/gen_helpers/requirements.txt @@ -1,2 +1,3 @@ ruamel.yaml==0.16.5 eth-utils==1.6.0 +pytest==3.6.1 diff --git a/tests/core/gen_helpers/setup.py b/tests/core/gen_helpers/setup.py index b674dbfb6b..fa7315a1f3 100644 --- a/tests/core/gen_helpers/setup.py +++ b/tests/core/gen_helpers/setup.py @@ -5,6 +5,7 @@ packages=['gen_base', 'gen_from_tests'], install_requires=[ "ruamel.yaml==0.16.5", - "eth-utils==1.6.0" + "eth-utils==1.6.0", + "pytest==3.6.1", ] ) diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 1977e5b054..3f6764e98d 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -54,6 +54,8 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: out[k] = [int(item) if item.isdigit() else item for item in v] elif isinstance(v, str) and v.startswith("0x"): out[k] = bytes.fromhex(v[2:]) + elif k == "CONFIG_NAME": + out[k] = str(v) else: out[k] = int(v) return out diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 68696ea0a4..0d8f472db0 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -1,9 +1,11 @@ +import pytest + from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.utils import bls +from .exceptions import SkippedTest from .helpers.genesis import create_genesis_state - from .utils import vector_test, with_meta_tags from random import Random @@ -22,11 +24,16 @@ def reload_specs(): # Some of the Spec module functionality is exposed here to deal with phase-specific changes. SpecForkName = NewType("SpecForkName", str) +ConfigName = NewType("ConfigName", str) PHASE0 = SpecForkName('phase0') PHASE1 = SpecForkName('phase1') ALL_PHASES = (PHASE0, PHASE1) +MAINNET = ConfigName('mainnet') +MINIMAL = ConfigName('minimal') + + # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. @@ -175,6 +182,17 @@ def entry(*args, **kw): DEFAULT_BLS_ACTIVE = True +is_pytest = True + + +def dump_skipping_message(reason: str) -> None: + message = f"[Skipped test] {reason}" + if is_pytest: + pytest.skip(message) + else: + raise SkippedTest(message) + + def spec_test(fn): # Bls switch must be wrapped by vector_test, # to fully go through the yielded bls switch data, before setting back the BLS setting. @@ -275,7 +293,8 @@ def wrapper(*args, **kw): if 'phase' in kw: phase = kw.pop('phase') if phase not in phases: - return + dump_skipping_message(f"doesn't support this fork: {phase}") + return None run_phases = [phase] available_phases = set(run_phases) @@ -300,3 +319,30 @@ def wrapper(*args, **kw): return ret return wrapper return decorator + + +def with_configs(configs): + def decorator(fn): + def wrapper(*args, spec: Spec, **kw): + available_configs = set(configs) + if spec.CONFIG_NAME not in available_configs: + dump_skipping_message(f"doesn't support this config: {spec.CONFIG_NAME}") + return None + + return fn(*args, spec=spec, **kw) + return wrapper + return decorator + + +def only_full_crosslink(fn): + def is_full_crosslink(spec, state): + epoch = spec.compute_epoch_at_slot(state.slot) + return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state) + + def wrapper(*args, spec: Spec, state: Any, **kw): + # TODO: update condition to "phase1+" if we have phase2 + if spec.fork == PHASE1 and not is_full_crosslink(spec, state): + dump_skipping_message("only for full crosslink") + return None + return fn(*args, spec=spec, state=state, **kw) + return wrapper diff --git a/tests/core/pyspec/eth2spec/test/exceptions.py b/tests/core/pyspec/eth2spec/test/exceptions.py new file mode 100644 index 0000000000..c553ec3744 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/exceptions.py @@ -0,0 +1,2 @@ +class SkippedTest(Exception): + ... diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 6e508e3970..d10d1ee7bc 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -35,8 +35,3 @@ def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition - - -def is_full_crosslink(spec, state): - epoch = spec.compute_epoch_at_slot(state.slot) - return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index 9e67c0b480..bfc6cd799c 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -9,9 +9,9 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, - with_all_phases_except, - spec_state_test, expect_assertion_error, + spec_state_test, + with_all_phases_except, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py index 2bb569eac2..b0a51557af 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py @@ -1,6 +1,7 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, + only_full_crosslink, spec_state_test, ) from eth2spec.test.helpers.attestations import ( @@ -10,7 +11,6 @@ ) from eth2spec.test.helpers.shard_transitions import ( run_shard_transitions_processing, - is_full_crosslink, ) from eth2spec.test.helpers.shard_block import ( build_shard_block, @@ -92,31 +92,22 @@ def run_successful_crosslink_tests(spec, state, target_len_offset_slot): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_basic_crosslinks(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=1) @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_multiple_offset_slots(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=2) @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_no_winning_root(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1) init_slot = state.slot @@ -163,11 +154,8 @@ def test_no_winning_root(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_wrong_shard_transition_root(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1) init_slot = state.slot diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py index 1ddbf1c6bd..922b604ad4 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py @@ -1,9 +1,11 @@ from typing import Dict, Sequence from eth2spec.test.context import ( - PHASE0, + PHASE0, MINIMAL, with_all_phases_except, spec_state_test, + only_full_crosslink, + with_configs, ) from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block @@ -22,7 +24,6 @@ get_sample_shard_block_body, get_shard_transitions, ) -from eth2spec.test.helpers.shard_transitions import is_full_crosslink from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to @@ -99,12 +100,8 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_process_beacon_block_with_normal_shard_transition(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -117,12 +114,8 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_process_beacon_block_with_empty_proposal_transition(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -140,12 +133,8 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_with_shard_transition_with_custody_challenge_and_response(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) # build shard block @@ -178,6 +167,7 @@ def test_with_shard_transition_with_custody_challenge_and_response(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL]) def test_custody_key_reveal(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) @@ -202,12 +192,8 @@ def test_early_derived_secret_reveal(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_custody_slashing(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) # Build shard block diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py index 440d0dd8d8..ab66314e5b 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py @@ -4,12 +4,12 @@ expect_assertion_error, spec_state_test, with_all_phases_except, + only_full_crosslink, ) from eth2spec.test.helpers.shard_block import ( build_shard_block, sign_shard_block, ) -from eth2spec.test.helpers.shard_transitions import is_full_crosslink from eth2spec.test.helpers.state import next_slot, transition_to_valid_shard_slot, transition_to @@ -46,11 +46,8 @@ def run_shard_blocks(spec, shard_state, signed_shard_block, beacon_parent_state, @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_valid_shard_block(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) @@ -68,11 +65,8 @@ def test_valid_shard_block(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_shard_parent_root(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) @@ -87,11 +81,8 @@ def test_invalid_shard_parent_root(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_beacon_parent_root(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -105,11 +96,8 @@ def test_invalid_beacon_parent_root(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_slot(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -124,11 +112,8 @@ def test_invalid_slot(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_proposer_index(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -148,12 +133,8 @@ def test_invalid_proposer_index(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_out_of_bound_offset(spec, state): - # TODO: Handle this edge case properly - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -173,11 +154,8 @@ def test_out_of_bound_offset(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_invalid_offset(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) # 4 is not in `SHARD_BLOCK_OFFSETS` @@ -195,11 +173,8 @@ def test_invalid_offset(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_empty_block_body(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -217,11 +192,8 @@ def test_empty_block_body(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_invalid_signature(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -239,11 +211,8 @@ def test_invalid_signature(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_max_offset(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -259,11 +228,8 @@ def test_max_offset(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_pending_shard_parent_block(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - # Block N beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) diff --git a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py index de47aaa390..0d1cd1fc16 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py +++ b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls, only_full_crosslink from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, @@ -8,7 +8,6 @@ get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root -from eth2spec.test.helpers.shard_transitions import is_full_crosslink from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block @@ -209,11 +208,8 @@ def create_simple_fork(spec, state, store, shard): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_shard_simple_fork(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) shard = spec.Shard(1) @@ -237,11 +233,8 @@ def test_shard_simple_fork(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_shard_latest_messages_for_different_shards(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) shard_0 = spec.Shard(0) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index f1e4b4483a..0323399703 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -54,4 +54,3 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: # gen_runner.run_generator(f"sanity", [ # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods # ]) - From 7eeb754f7dd1739821c5906733af7c9876aaf297 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 12 Aug 2020 21:50:36 +0800 Subject: [PATCH 07/19] Add `disable_process_reveal_deadlines` decorator --- tests/core/pyspec/eth2spec/test/context.py | 8 ++++++++ .../pyspec/eth2spec/test/phase0/sanity/test_blocks.py | 7 +++++-- tests/formats/README.md | 3 +++ tests/formats/sanity/blocks.md | 7 ++++--- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 68696ea0a4..bc1ebe0923 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -300,3 +300,11 @@ def wrapper(*args, **kw): return ret return wrapper return decorator + + +def disable_process_reveal_deadlines(fn): + def entry(*args, spec: Spec, **kw): + if hasattr(spec, 'process_reveal_deadlines'): + spec.process_reveal_deadlines = lambda state: None + return fn(*args, spec=spec, **kw) + return with_meta_tags({'reveal_deadlines_setting': 1})(entry) diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 6d7ff096af..973dd3ae44 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -23,6 +23,7 @@ from eth2spec.test.context import ( PHASE0, PHASE1, spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, + disable_process_reveal_deadlines, ) @@ -762,8 +763,9 @@ def create_signed_exit(index): # exceeding the minimal-config randao mixes memory size. # Applies to all voluntary-exit sanity block tests. -@with_phases([PHASE0]) +@with_all_phases @spec_state_test +@disable_process_reveal_deadlines def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -811,8 +813,9 @@ def test_double_validator_exit_same_block(spec, state): yield 'post', None -@with_phases([PHASE0]) +@with_all_phases @spec_state_test +@disable_process_reveal_deadlines def test_multiple_different_validator_exits_same_block(spec, state): validator_indices = [ spec.get_active_validator_indices(state, spec.get_current_epoch(state))[i] diff --git a/tests/formats/README.md b/tests/formats/README.md index 63b9a53903..36a5ec21b2 100644 --- a/tests/formats/README.md +++ b/tests/formats/README.md @@ -165,6 +165,9 @@ bls_setting: int -- optional, can have 3 different values: but there is no change of outcome when running the test if BLS is ON or OFF. 1: known as "BLS required" - if the test validity is strictly dependent on BLS being ON 2: known as "BLS ignored" - if the test validity is strictly dependent on BLS being OFF +reveal_deadlines_setting: -- optional, can have 2 different values: + 0: default, `process_reveal_deadlines` is ON. + 1: `process_reveal_deadlines` is OFF. ``` diff --git a/tests/formats/sanity/blocks.md b/tests/formats/sanity/blocks.md index 991bc35d22..44b37ed5ee 100644 --- a/tests/formats/sanity/blocks.md +++ b/tests/formats/sanity/blocks.md @@ -7,9 +7,10 @@ Sanity tests to cover a series of one or more blocks being processed, aiming to ### `meta.yaml` ```yaml -description: string -- Optional. Description of test case, purely for debugging purposes. -bls_setting: int -- see general test-format spec. -blocks_count: int -- the number of blocks processed in this test. +description: string -- Optional. Description of test case, purely for debugging purposes. +bls_setting: int -- see general test-format spec. +reveal_deadlines_setting: int -- see general test-format spec. +blocks_count: int -- the number of blocks processed in this test. ``` From 42f368c45c57f911a32ce8fa8705a559e38dd751 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 25 Aug 2020 13:46:29 +0800 Subject: [PATCH 08/19] Disable `process_reveal_deadlines` in the custody tests that have `EPOCHS_PER_CUSTODY_PERIOD`-long transition --- .../block_processing/test_process_chunk_challenge.py | 12 ++++++++++++ .../test_process_custody_slashing.py | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index 9e67c0b480..b541a1321c 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -12,6 +12,7 @@ with_all_phases_except, spec_state_test, expect_assertion_error, + disable_process_reveal_deadlines, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -67,6 +68,7 @@ def run_custody_chunk_response_processing(spec, state, custody_response, valid=T @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_challenge_appended(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -89,6 +91,7 @@ def test_challenge_appended(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_challenge_empty_element_replaced(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -113,6 +116,7 @@ def test_challenge_empty_element_replaced(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_duplicate_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -137,6 +141,7 @@ def test_duplicate_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_second_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -163,6 +168,7 @@ def test_second_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_multiple_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -186,6 +192,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_many_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) @@ -209,6 +216,7 @@ def test_many_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_off_chain_attestation(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -228,6 +236,7 @@ def test_off_chain_attestation(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_custody_response(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -258,6 +267,7 @@ def test_custody_response(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_custody_response_chunk_index_2(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -287,6 +297,7 @@ def test_custody_response_chunk_index_2(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_custody_response_multiple_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -317,6 +328,7 @@ def test_custody_response_multiple_epochs(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_custody_response_many_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index b08b654130..9fdfc3d898 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -13,6 +13,7 @@ with_all_phases_except, spec_state_test, expect_assertion_error, + disable_process_reveal_deadlines, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -106,30 +107,35 @@ def run_standard_custody_slashing_test(spec, @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_incorrect_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state, correct=False) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_multiple_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_many_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines def test_invalid_custody_slashing(spec, state): yield from run_standard_custody_slashing_test( spec, From 458166af8312ac4c65a5c05b7ce4c7eb895c116a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 27 Aug 2020 16:12:31 +0800 Subject: [PATCH 09/19] Update the pytest version to >=4.4 as eth2spec --- tests/core/gen_helpers/requirements.txt | 2 +- tests/core/gen_helpers/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/gen_helpers/requirements.txt b/tests/core/gen_helpers/requirements.txt index c972f9b7bb..e7cdd30ea2 100644 --- a/tests/core/gen_helpers/requirements.txt +++ b/tests/core/gen_helpers/requirements.txt @@ -1,3 +1,3 @@ ruamel.yaml==0.16.5 eth-utils==1.6.0 -pytest==3.6.1 +pytest>=4.4 diff --git a/tests/core/gen_helpers/setup.py b/tests/core/gen_helpers/setup.py index fa7315a1f3..e9fc1a7877 100644 --- a/tests/core/gen_helpers/setup.py +++ b/tests/core/gen_helpers/setup.py @@ -6,6 +6,6 @@ install_requires=[ "ruamel.yaml==0.16.5", "eth-utils==1.6.0", - "pytest==3.6.1", + "pytest>=4.4", ] ) From b4583aba5daf3e79e650b51b13ea2d9709d9a63d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Aug 2020 14:52:41 +0800 Subject: [PATCH 10/19] Add skipping messages to some old test cases --- .../test/phase0/sanity/test_blocks.py | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 6d7ff096af..c712ded70e 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -22,7 +22,7 @@ from eth2spec.test.context import ( PHASE0, PHASE1, - spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, + spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, dump_skipping_message, ) @@ -291,10 +291,9 @@ def test_empty_epoch_transition(spec, state): @with_all_phases @spec_state_test def test_empty_epoch_transition_not_finalizing(spec, state): - # Don't run for non-minimal configs, it takes very long, and the effect - # of calling finalization/justification is just the same as with the minimal configuration. if spec.SLOTS_PER_EPOCH > 8: - return + return dump_skipping_message("Skip mainnet config for saving time." + " Minimal config suffice to cover the target-of-test.") # copy for later balance lookups. pre_balances = list(state.balances) @@ -480,9 +479,8 @@ def test_attester_slashing(spec, state): @with_all_phases @spec_state_test def test_duplicate_attester_slashing(spec, state): - # Skip test if config cannot handle multiple AttesterSlashings per block if spec.MAX_ATTESTER_SLASHINGS < 2: - return + return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) attester_slashings = [attester_slashing, attester_slashing.copy()] @@ -509,9 +507,8 @@ def test_duplicate_attester_slashing(spec, state): @with_all_phases @spec_state_test def test_multiple_attester_slashings_no_overlap(spec, state): - # Skip test if config cannot handle multiple AttesterSlashings per block if spec.MAX_ATTESTER_SLASHINGS < 2: - return + return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") # copy for later balance lookups. pre_state = state.copy() @@ -550,9 +547,8 @@ def test_multiple_attester_slashings_no_overlap(spec, state): @with_all_phases @spec_state_test def test_multiple_attester_slashings_partial_overlap(spec, state): - # Skip test if config cannot handle multiple AttesterSlashings per block if spec.MAX_ATTESTER_SLASHINGS < 2: - return + return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") # copy for later balance lookups. pre_state = state.copy() @@ -888,9 +884,9 @@ def test_historical_batch(spec, state): @with_all_phases @spec_state_test def test_eth1_data_votes_consensus(spec, state): - # Don't run when it will take very, very long to simulate. Minimal configuration suffices. if spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 2: - return + return dump_skipping_message("Skip test if config with longer `EPOCHS_PER_ETH1_VOTING_PERIOD` for saving time." + " Minimal config suffice to cover the target-of-test.") voting_period_slots = spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH @@ -932,9 +928,9 @@ def test_eth1_data_votes_consensus(spec, state): @with_all_phases @spec_state_test def test_eth1_data_votes_no_consensus(spec, state): - # Don't run when it will take very, very long to simulate. Minimal configuration suffices. if spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 2: - return + return dump_skipping_message("Skip test if config with longer `EPOCHS_PER_ETH1_VOTING_PERIOD` for saving time." + " Minimal config suffice to cover the target-of-test.") voting_period_slots = spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH From 74a2a1c0d4a7706fd77da2b85769c7b355c20562 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 4 Sep 2020 02:40:36 +0800 Subject: [PATCH 11/19] Enable all tests --- tests/generators/epoch_processing/main.py | 14 ++++++-------- tests/generators/operations/main.py | 14 ++++++-------- tests/generators/sanity/main.py | 14 ++++++-------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index cf87e74513..d4114b93b6 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -47,14 +47,12 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: gen_runner.run_generator(f"epoch_processing", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"epoch_processing", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"epoch_processing", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"epoch_processing", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ]) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index c04ad26716..0ecc63f468 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -51,14 +51,12 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: gen_runner.run_generator(f"operations", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"operations", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"operations", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"operations", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"operations", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) + gen_runner.run_generator(f"operations", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ]) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 0323399703..c2669ffdc1 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -43,14 +43,12 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: gen_runner.run_generator(f"sanity", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"sanity", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"sanity", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"sanity", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ]) From a0d646e1e06ca431c24b4045d0b3485845407609 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 4 Sep 2020 02:41:09 +0800 Subject: [PATCH 12/19] Add decorators to diable testing with mainnet config --- tests/core/pyspec/eth2spec/test/context.py | 7 +++++-- .../test_process_chunk_challenge.py | 13 +++++++++++++ .../test_process_custody_slashing.py | 7 +++++++ .../test_process_challenge_deadlines.py | 5 ++++- .../test_process_reveal_deadlines.py | 4 ++++ 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index c2bbff3412..7f58c66108 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -329,12 +329,15 @@ def entry(*args, spec: Spec, **kw): return with_meta_tags({'reveal_deadlines_setting': 1})(entry) -def with_configs(configs): +def with_configs(configs, reason=None): def decorator(fn): def wrapper(*args, spec: Spec, **kw): available_configs = set(configs) if spec.CONFIG_NAME not in available_configs: - dump_skipping_message(f"doesn't support this config: {spec.CONFIG_NAME}") + message = f"doesn't support this config: {spec.CONFIG_NAME}." + if reason is not None: + message = f"{message} Reason: {reason}" + dump_skipping_message(message) return None return fn(*args, spec=spec, **kw) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index ecc7041846..e916010b25 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -9,10 +9,12 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, + MINIMAL, expect_assertion_error, disable_process_reveal_deadlines, spec_state_test, with_all_phases_except, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -68,6 +70,7 @@ def run_custody_chunk_response_processing(spec, state, custody_response, valid=T @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") @disable_process_reveal_deadlines def test_challenge_appended(spec, state): transition_to_valid_shard_slot(spec, state) @@ -92,6 +95,7 @@ def test_challenge_appended(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_challenge_empty_element_replaced(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -117,6 +121,7 @@ def test_challenge_empty_element_replaced(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_duplicate_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -142,6 +147,7 @@ def test_duplicate_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_second_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -169,6 +175,7 @@ def test_second_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_multiple_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -193,6 +200,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_many_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) @@ -217,6 +225,7 @@ def test_many_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_off_chain_attestation(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -237,6 +246,7 @@ def test_off_chain_attestation(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -268,6 +278,7 @@ def test_custody_response(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_chunk_index_2(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -298,6 +309,7 @@ def test_custody_response_chunk_index_2(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_multiple_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -329,6 +341,7 @@ def test_custody_response_multiple_epochs(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_many_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index 6836685b81..fc7efa5bc9 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -10,10 +10,12 @@ from eth2spec.test.helpers.state import get_balance, transition_to from eth2spec.test.context import ( PHASE0, + MINIMAL, with_all_phases_except, spec_state_test, expect_assertion_error, disable_process_reveal_deadlines, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -113,6 +115,7 @@ def run_standard_custody_slashing_test(spec, @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state) @@ -120,6 +123,7 @@ def test_custody_slashing(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_incorrect_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state, correct=False) @@ -127,6 +131,7 @@ def test_incorrect_custody_slashing(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_multiple_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) @@ -134,6 +139,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_many_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5) @@ -141,6 +147,7 @@ def test_many_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_invalid_custody_slashing(spec, state): yield from run_standard_custody_slashing_test( spec, diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py index 828ace91cd..e270ff615e 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py @@ -8,8 +8,10 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, - with_all_phases_except, + MINIMAL, spec_state_test, + with_all_phases_except, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -25,6 +27,7 @@ def run_process_challenge_deadlines(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_chunk_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py index da1a604469..5777e184aa 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py @@ -4,7 +4,9 @@ from eth2spec.test.helpers.state import transition_to from eth2spec.test.context import ( PHASE0, + MINIMAL, with_all_phases_except, + with_configs, spec_state_test, ) from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -17,6 +19,7 @@ def run_process_challenge_deadlines(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_reveal_deadline(spec, state): assert state.validators[0].slashed == 0 transition_to(spec, state, spec.get_randao_epoch_for_custody_period(0, 0) * spec.SLOTS_PER_EPOCH) @@ -36,6 +39,7 @@ def test_validator_slashed_after_reveal_deadline(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_not_slashed_after_reveal(spec, state): transition_to(spec, state, spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) custody_key_reveal = get_valid_custody_key_reveal(spec, state) From 916ae9e309f0c08faade851cda361715ff08d198 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 4 Sep 2020 21:26:16 +0800 Subject: [PATCH 13/19] Enable more rewards and finality tests --- tests/generators/finality/main.py | 14 +++++++++----- tests/generators/rewards/main.py | 15 ++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index dca0ecb8dc..ef2d8293f4 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -4,7 +4,7 @@ from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from eth2spec.test.context import PHASE0 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.test.phase0.finality import test_finality from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 @@ -12,7 +12,7 @@ from eth2spec.utils import bls -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -26,14 +26,18 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: runner_name='finality', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": + # No additional phase 1 specific rewards tests, yet. + key = 'finality' gen_runner.run_generator("finality", [ - create_provider('finality', test_finality, 'minimal'), - create_provider('finality', test_finality, 'mainnet'), + create_provider(PHASE0, 'finality', test_finality, 'minimal'), + create_provider(PHASE0, 'finality', test_finality, 'mainnet'), + create_provider(PHASE1, 'finality', test_finality, 'minimal'), + create_provider(PHASE1, 'finality', test_finality, 'mainnet'), ]) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 30554358bd..b07092d687 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -45,15 +45,12 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: gen_runner.run_generator(f"rewards", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"rewards", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"rewards", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"rewards", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) - + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ]) From fd4e7dde943c2805b2c09657a838ce0e860cfeef Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Sep 2020 13:34:23 +0800 Subject: [PATCH 14/19] Fix `disable_process_reveal_deadlines` decorator: should have set it back to the spec function for other test cases afterwards --- tests/core/pyspec/eth2spec/test/context.py | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 7f58c66108..12fb931528 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -264,6 +264,24 @@ def entry(*args, **kw): return entry +def disable_process_reveal_deadlines(fn): + """ + Decorator to make a function execute with `process_reveal_deadlines` OFF. + This is for testing long-range epochs transition without considering the reveal-deadline slashing effect. + """ + def entry(*args, spec: Spec, **kw): + if hasattr(spec, 'process_reveal_deadlines'): + old_state = spec.process_reveal_deadlines + spec.process_reveal_deadlines = lambda state: None + + yield from fn(*args, spec=spec, **kw) + + if hasattr(spec, 'process_reveal_deadlines'): + spec.process_reveal_deadlines = old_state + + return with_meta_tags({'reveal_deadlines_setting': 1})(entry) + + def with_all_phases(fn): """ A decorator for running a test with every phase @@ -321,14 +339,6 @@ def wrapper(*args, **kw): return decorator -def disable_process_reveal_deadlines(fn): - def entry(*args, spec: Spec, **kw): - if hasattr(spec, 'process_reveal_deadlines'): - spec.process_reveal_deadlines = lambda state: None - return fn(*args, spec=spec, **kw) - return with_meta_tags({'reveal_deadlines_setting': 1})(entry) - - def with_configs(configs, reason=None): def decorator(fn): def wrapper(*args, spec: Spec, **kw): From c0c2461d2f581f0d0f921746624b80dab4ee28ec Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 15 Sep 2020 11:00:36 +0800 Subject: [PATCH 15/19] Remove redundant keys and use Dict for `phase_0_mods`, `phase_1_mods` instead of List --- tests/generators/epoch_processing/main.py | 16 ++++++++-------- tests/generators/operations/main.py | 16 ++++++++-------- tests/generators/rewards/main.py | 15 ++++++--------- tests/generators/sanity/main.py | 16 ++++++++-------- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index d4114b93b6..d13213b1e8 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -31,28 +31,28 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: if __name__ == "__main__": - phase_0_mods = [(key, 'eth2spec.test.phase0.epoch_processing.test_process_' + key) for key in [ + phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [ 'final_updates', 'justification_and_finalization', 'registry_updates', 'rewards_and_penalties', 'slashings', - ]] - phase_1_mods = [(key, 'eth2spec.test.phase1.epoch_processing.test_process_' + key) for key in [ + ]} + phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ 'challenge_deadlines', 'custody_final_updates', 'reveal_deadlines', - ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() ]) gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 0ecc63f468..24fbe768f6 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -31,32 +31,32 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: if __name__ == "__main__": - phase_0_mods = [(key, 'eth2spec.test.phase0.block_processing.test_process_' + key) for key in [ + phase_0_mods = {key: 'eth2spec.test.phase0.block_processing.test_process_' + key for key in [ 'attestation', 'attester_slashing', 'block_header', 'deposit', 'proposer_slashing', 'voluntary_exit', - ]] - phase_1_mods = [(key, 'eth2spec.test.phase1.block_processing.test_process_' + key) for key in [ + ]} + phase_1_mods = {**{key: 'eth2spec.test.phase1.block_processing.test_process_' + key for key in [ 'attestation', 'chunk_challenge', 'custody_key_reveal', 'custody_slashing', 'early_derived_secret_reveal', 'shard_transition', - ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() ]) gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index b07092d687..2e44ac5ffc 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -31,26 +31,23 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: if __name__ == "__main__": - phase_0_mods = [(key, 'eth2spec.test.phase0.rewards.test_'+key) for key in [ + phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_'+key for key in [ 'basic', - 'basic', - 'leak', 'leak', 'random', - 'random', - ]] + ]} # No additional phase 1 specific rewards tests, yet. phase_1_mods = phase_0_mods gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() ]) gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index c2669ffdc1..95ab52473d 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -31,24 +31,24 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: if __name__ == "__main__": - phase_0_mods = [(key, 'eth2spec.test.phase0.sanity.test_' + key) for key in [ + phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [ 'blocks', 'slots', - ]] - phase_1_mods = [(key, 'eth2spec.test.phase1.sanity.test_' + key) for key in [ + ]} + phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ 'blocks', # more phase 1 specific block tests 'shard_blocks', - ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() ]) gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() ]) gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) From 20bd00cef027fac37e0628a13e37da0b38475580 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 15 Sep 2020 11:19:50 +0800 Subject: [PATCH 16/19] Add `make lint_generators` linter check for test generators --- .circleci/config.yml | 5 ++++- Makefile | 5 +++++ tests/generators/epoch_processing/main.py | 3 ++- tests/generators/operations/main.py | 3 ++- tests/generators/rewards/main.py | 5 +++-- tests/generators/sanity/main.py | 5 +++-- tests/generators/ssz_generic/ssz_boolean.py | 1 - tests/generators/ssz_generic/ssz_container.py | 2 +- tests/generators/ssz_static/main.py | 6 ++++-- 9 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df415f2ebd..2767bec704 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,8 +130,11 @@ jobs: key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: - name: Run linter + name: Run linter for pyspec command: make lint + - run: + name: Run linter for test generators + command: make lint_generators build_deposit_contract: docker: - image: ethereum/solc:0.6.11-alpine diff --git a/Makefile b/Makefile index 3914b7b275..8fa1044445 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ SPEC_DIR = ./specs SSZ_DIR = ./ssz TEST_LIBS_DIR = ./tests/core +TEST_GENERATORS_DIR = ./tests/generators PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec TEST_VECTOR_DIR = ../eth2.0-spec-tests/tests GENERATOR_DIR = ./tests/generators @@ -113,6 +114,10 @@ lint: pyspec flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 +lint_generators: pyspec + . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ + flake8 --config $(LINTER_CONFIG_FILE) + compile_deposit_contract: @cd $(SOLIDITY_DEPOSIT_CONTRACT_DIR) @git submodule update --recursive --init diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index d13213b1e8..fe8e0ee921 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -10,7 +10,8 @@ from eth2spec.utils import bls -def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 24fbe768f6..1acf45e477 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -10,7 +10,8 @@ from eth2spec.utils import bls -def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 2e44ac5ffc..23d0633b08 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -10,7 +10,8 @@ from eth2spec.utils import bls -def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -31,7 +32,7 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: if __name__ == "__main__": - phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_'+key for key in [ + phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_' + key for key in [ 'basic', 'leak', 'random', diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 95ab52473d..83166f0cf0 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -10,7 +10,8 @@ from eth2spec.utils import bls -def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -38,7 +39,7 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ 'blocks', # more phase 1 specific block tests 'shard_blocks', - ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) gen_runner.run_generator(f"sanity", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() diff --git a/tests/generators/ssz_generic/ssz_boolean.py b/tests/generators/ssz_generic/ssz_boolean.py index 9ff36ba88d..ec22c01be9 100644 --- a/tests/generators/ssz_generic/ssz_boolean.py +++ b/tests/generators/ssz_generic/ssz_boolean.py @@ -12,4 +12,3 @@ def invalid_cases(): yield "byte_rev_nibble", invalid_test_case(lambda: b'\x10') yield "byte_0x80", invalid_test_case(lambda: b'\x80') yield "byte_full", invalid_test_case(lambda: b'\xff') - diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index cf7c338398..9cd155f76b 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -85,7 +85,7 @@ def valid_cases(): def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): return b[:offset_index] + \ (change(int.from_bytes(b[offset_index:offset_index + 4], byteorder='little')) & 0xffffffff) \ - .to_bytes(length=4, byteorder='little') + \ + .to_bytes(length=4, byteorder='little') + \ b[offset_index + 4:] diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 8dfd0cc86e..a30908fe9e 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -21,7 +21,8 @@ MAX_LIST_LENGTH = 10 -def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]: +def create_test_case(rng: Random, typ, + mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]: value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield "value", "data", encode.encode(value) yield "serialized", "ssz", serialize(value) @@ -38,7 +39,8 @@ def get_spec_ssz_types(spec): ] -def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int): +def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, + mode: random_value.RandomizationMode, chaos: bool, count: int): random_mode_name = mode.to_name() # Reproducible RNG From 0c1784b8b5f126acc80d5f18753b6b3fcebd7b6d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 17 Sep 2020 16:22:21 -0600 Subject: [PATCH 17/19] skip large_validator_set tests if too large --- tests/core/pyspec/eth2spec/test/context.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 70eb05d510..2f27334d41 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -62,6 +62,8 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca p0 = phases[PHASE0] balances = balances_fn(p0) + if balances is None: + return None activation_threshold = threshold_fn(p0) state = create_genesis_state(spec=p0, validator_balances=balances, @@ -88,6 +90,9 @@ def entry(*args, spec: Spec, phases: SpecForks, **kw): global _custom_state_cache_dict if key not in _custom_state_cache_dict: state = _prepare_state(balances_fn, threshold_fn, spec, phases) + if state is None: + dump_skipping_message(f"doesn't support this configuration: {spec.CONFIG_NAME}") + return None _custom_state_cache_dict[key] = state.get_backing() # Take an entry out of the LRU. @@ -160,10 +165,14 @@ def low_single_balance(spec): def large_validator_set(spec): """ - Helper method to create a series of default balances. + Helper method to create a large series of default balances. + Return None if too large for standard test processing. Usage: `@with_custom_state(balances_fn=default_balances, ...)` """ num_validators = 2 * spec.SLOTS_PER_EPOCH * spec.MAX_COMMITTEES_PER_SLOT * spec.TARGET_COMMITTEE_SIZE + if num_validators > spec.SLOTS_PER_EPOCH * 256: + # Larger than limit of public/private keys pre-generated + return None return [spec.MAX_EFFECTIVE_BALANCE] * num_validators From 27d8c806908a93aad52bbc8de0a433c2f3b8fcac Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Sep 2020 17:05:30 +0800 Subject: [PATCH 18/19] Use `with_configs[MINIMAL]` on `large_validator_set` tests --- .../core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 2299ead7f0..721d68add1 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -21,11 +21,12 @@ from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.context import ( - PHASE0, PHASE1, + PHASE0, PHASE1, MINIMAL, spec_test, spec_state_test, dump_skipping_message, with_phases, with_all_phases, single_phase, expect_assertion_error, always_bls, disable_process_reveal_deadlines, + with_configs, with_custom_state, large_validator_set, ) @@ -92,6 +93,8 @@ def test_empty_block_transition(spec, state): @with_all_phases +@with_configs([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") @spec_test @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @single_phase @@ -318,6 +321,8 @@ def test_empty_epoch_transition(spec, state): @with_all_phases +@with_configs([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") @spec_test @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @single_phase From d257926e3a3a6e0bde2dd9d682ac0c8a136b6a90 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 18 Sep 2020 07:59:25 -0600 Subject: [PATCH 19/19] remove kludge to skip large_validator_set on mainnet config --- tests/core/pyspec/eth2spec/test/context.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 2f27334d41..cfd6724ed7 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -62,8 +62,6 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca p0 = phases[PHASE0] balances = balances_fn(p0) - if balances is None: - return None activation_threshold = threshold_fn(p0) state = create_genesis_state(spec=p0, validator_balances=balances, @@ -90,9 +88,6 @@ def entry(*args, spec: Spec, phases: SpecForks, **kw): global _custom_state_cache_dict if key not in _custom_state_cache_dict: state = _prepare_state(balances_fn, threshold_fn, spec, phases) - if state is None: - dump_skipping_message(f"doesn't support this configuration: {spec.CONFIG_NAME}") - return None _custom_state_cache_dict[key] = state.get_backing() # Take an entry out of the LRU. @@ -166,13 +161,9 @@ def low_single_balance(spec): def large_validator_set(spec): """ Helper method to create a large series of default balances. - Return None if too large for standard test processing. Usage: `@with_custom_state(balances_fn=default_balances, ...)` """ num_validators = 2 * spec.SLOTS_PER_EPOCH * spec.MAX_COMMITTEES_PER_SLOT * spec.TARGET_COMMITTEE_SIZE - if num_validators > spec.SLOTS_PER_EPOCH * 256: - # Larger than limit of public/private keys pre-generated - return None return [spec.MAX_EFFECTIVE_BALANCE] * num_validators