Merge pull request #1957 from ethereum/testgenphase1

Enable test generation for phase1

.circleci/config.yml

@@ -130,8 +130,11 @@ jobs:
           key: v3-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
       - run:
-          name: Run linter
+          name: Run linter for pyspec
           command: make lint
+      - run:
+          name: Run linter for test generators
+          command: make lint_generators
   build_deposit_contract:
     docker:
       - image: ethereum/solc:0.6.11-alpine

Makefile

@@ -1,6 +1,7 @@
 SPEC_DIR = ./specs
 SSZ_DIR = ./ssz
 TEST_LIBS_DIR = ./tests/core
+TEST_GENERATORS_DIR = ./tests/generators
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
 TEST_VECTOR_DIR = ../eth2.0-spec-tests/tests
 GENERATOR_DIR = ./tests/generators
@@ -113,6 +114,10 @@ lint: pyspec
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
 	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1
 
+lint_generators: pyspec
+	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \
+	flake8  --config $(LINTER_CONFIG_FILE)
+
 compile_deposit_contract:
 	@cd $(SOLIDITY_DEPOSIT_CONTRACT_DIR)
 	@git submodule update --recursive --init

configs/mainnet/phase0.yaml

@@ -2,6 +2,7 @@
 # Note: the intention of this file (for now) is to illustrate what a mainnet configuration could look like.
 # Some of these constants may still change before the launch of Phase 0.
 
+CONFIG_NAME: "mainnet"
 
 # Misc
 # ---------------------------------------------------------------

configs/mainnet/phase1.yaml

@@ -1,5 +1,6 @@
 # Mainnet preset - phase 1
 
+CONFIG_NAME: "mainnet"
 
 # phase1-fork
 # ---------------------------------------------------------------

configs/minimal/phase0.yaml

@@ -1,5 +1,6 @@
 # Minimal preset
 
+CONFIG_NAME: "minimal"
 
 # Misc
 # ---------------------------------------------------------------

configs/minimal/phase1.yaml

@@ -1,5 +1,6 @@
 # Minimal preset - phase 1
 
+CONFIG_NAME: "minimal"
 
 # phase1-fork
 # ---------------------------------------------------------------

setup.py

@@ -119,6 +119,8 @@ def get_spec(file_name: str) -> SpecObject:
 from eth2spec.utils.hash_function import hash
 
 SSZObject = TypeVar('SSZObject', bound=View)
+
+CONFIG_NAME = 'mainnet'
 '''
 PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0
 from eth2spec.config.config_util import apply_constants_config
@@ -151,6 +153,8 @@ def get_spec(file_name: str) -> SpecObject:
 SSZVariableName = str
 GeneralizedIndex = NewType('GeneralizedIndex', int)
 SSZObject = TypeVar('SSZObject', bound=View)
+
+CONFIG_NAME = 'mainnet'
 '''
 SUNDRY_CONSTANTS_FUNCTIONS = '''
 def ceillog2(x: int) -> uint64:

tests/core/gen_helpers/gen_base/gen_runner.py

@@ -2,13 +2,21 @@
 from pathlib import Path
 import sys
 from typing import Iterable, AnyStr, Any, Callable
+import traceback
 
 from ruamel.yaml import (
     YAML,
 )
 
 from gen_base.gen_typing import TestProvider
 
+from eth2spec.test import context
+from eth2spec.test.exceptions import SkippedTest
+
+
+# Flag that the runner does NOT run test via pytest
+context.is_pytest = False
+
 
 def validate_output_dir(path_str):
     path = Path(path_str)
@@ -134,14 +142,20 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
 
                 written_part = False
                 meta = dict()
-                for (name, out_kind, data) in test_case.case_fn():
-                    written_part = True
-                    if out_kind == "meta":
-                        meta[name] = data
-                    if out_kind == "data":
-                        output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml))
-                    if out_kind == "ssz":
-                        output_part("ssz", name, dump_ssz_fn(data, name, file_mode))
+
+                try:
+                    for (name, out_kind, data) in test_case.case_fn():
+                        written_part = True
+                        if out_kind == "meta":
+                            meta[name] = data
+                        if out_kind == "data":
+                            output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml))
+                        if out_kind == "ssz":
+                            output_part("ssz", name, dump_ssz_fn(data, name, file_mode))
+                except SkippedTest as e:
+                    print(e)
+                    continue
+
                 # Once all meta data is collected (if any), write it to a meta data file.
                 if len(meta) != 0:
                     written_part = True
@@ -152,6 +166,7 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
 
             except Exception as e:
                 print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
+                traceback.print_exc()
     print(f"completed {generator_name}")
 
 

tests/core/gen_helpers/requirements.txt

@@ -1,2 +1,3 @@
 ruamel.yaml==0.16.5
 eth-utils==1.6.0
+pytest>=4.4

tests/core/gen_helpers/setup.py

@@ -5,6 +5,7 @@
     packages=['gen_base', 'gen_from_tests'],
     install_requires=[
         "ruamel.yaml==0.16.5",
-        "eth-utils==1.6.0"
+        "eth-utils==1.6.0",
+        "pytest>=4.4",
     ]
 )

tests/core/pyspec/eth2spec/config/config_util.py

@@ -54,6 +54,8 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]:
             out[k] = [int(item) if item.isdigit() else item for item in v]
         elif isinstance(v, str) and v.startswith("0x"):
             out[k] = bytes.fromhex(v[2:])
+        elif k == "CONFIG_NAME":
+            out[k] = str(v)
         else:
             out[k] = int(v)
     return out

tests/core/pyspec/eth2spec/debug/random_value.py

@@ -68,9 +68,8 @@ def get_random_ssz_object(rng: Random,
         else:
             return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.limit()))))
     if issubclass(typ, ByteVector):
-        # Sanity, don't generate absurdly big random values
-        # If a client is aiming to performance-test, they should create a benchmark suite.
-        assert typ.type_byte_length() <= max_bytes_length
+        # Random byte vectors can be bigger than max bytes size, e.g. custody chunk data.
+        # No max-bytes-length limitation here.
         if mode == RandomizationMode.mode_zero:
             return typ(b'\x00' * typ.type_byte_length())
         elif mode == RandomizationMode.mode_max:

tests/core/pyspec/eth2spec/test/context.py

@@ -1,9 +1,11 @@
+import pytest
+
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.phase1 import spec as spec_phase1
 from eth2spec.utils import bls
 
+from .exceptions import SkippedTest
 from .helpers.genesis import create_genesis_state
-
 from .utils import vector_test, with_meta_tags
 
 from random import Random
@@ -22,11 +24,16 @@ def reload_specs():
 # Some of the Spec module functionality is exposed here to deal with phase-specific changes.
 
 SpecForkName = NewType("SpecForkName", str)
+ConfigName = NewType("ConfigName", str)
 
 PHASE0 = SpecForkName('phase0')
 PHASE1 = SpecForkName('phase1')
 ALL_PHASES = (PHASE0, PHASE1)
 
+MAINNET = ConfigName('mainnet')
+MINIMAL = ConfigName('minimal')
+
+
 # TODO: currently phases are defined as python modules.
 # It would be better if they would be more well-defined interfaces for stronger typing.
 
@@ -153,7 +160,7 @@ def low_single_balance(spec):
 
 def large_validator_set(spec):
     """
-    Helper method to create a series of default balances.
+    Helper method to create a large series of default balances.
     Usage: `@with_custom_state(balances_fn=default_balances, ...)`
     """
     num_validators = 2 * spec.SLOTS_PER_EPOCH * spec.MAX_COMMITTEES_PER_SLOT * spec.TARGET_COMMITTEE_SIZE
@@ -184,6 +191,17 @@ def entry(*args, **kw):
 DEFAULT_BLS_ACTIVE = True
 
 
+is_pytest = True
+
+
+def dump_skipping_message(reason: str) -> None:
+    message = f"[Skipped test] {reason}"
+    if is_pytest:
+        pytest.skip(message)
+    else:
+        raise SkippedTest(message)
+
+
 def spec_test(fn):
     # Bls switch must be wrapped by vector_test,
     # to fully go through the yielded bls switch data, before setting back the BLS setting.
@@ -255,6 +273,24 @@ def entry(*args, **kw):
     return entry
 
 
+def disable_process_reveal_deadlines(fn):
+    """
+    Decorator to make a function execute with `process_reveal_deadlines` OFF.
+    This is for testing long-range epochs transition without considering the reveal-deadline slashing effect.
+    """
+    def entry(*args, spec: Spec, **kw):
+        if hasattr(spec, 'process_reveal_deadlines'):
+            old_state = spec.process_reveal_deadlines
+            spec.process_reveal_deadlines = lambda state: None
+
+        yield from fn(*args, spec=spec, **kw)
+
+        if hasattr(spec, 'process_reveal_deadlines'):
+            spec.process_reveal_deadlines = old_state
+
+    return with_meta_tags({'reveal_deadlines_setting': 1})(entry)
+
+
 def with_all_phases(fn):
     """
     A decorator for running a test with every phase
@@ -284,7 +320,8 @@ def wrapper(*args, **kw):
             if 'phase' in kw:
                 phase = kw.pop('phase')
                 if phase not in phases:
-                    return
+                    dump_skipping_message(f"doesn't support this fork: {phase}")
+                    return None
                 run_phases = [phase]
 
             available_phases = set(run_phases)
@@ -309,3 +346,33 @@ def wrapper(*args, **kw):
             return ret
         return wrapper
     return decorator
+
+
+def with_configs(configs, reason=None):
+    def decorator(fn):
+        def wrapper(*args, spec: Spec, **kw):
+            available_configs = set(configs)
+            if spec.CONFIG_NAME not in available_configs:
+                message = f"doesn't support this config: {spec.CONFIG_NAME}."
+                if reason is not None:
+                    message = f"{message} Reason: {reason}"
+                dump_skipping_message(message)
+                return None
+
+            return fn(*args, spec=spec, **kw)
+        return wrapper
+    return decorator
+
+
+def only_full_crosslink(fn):
+    def is_full_crosslink(spec, state):
+        epoch = spec.compute_epoch_at_slot(state.slot)
+        return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state)
+
+    def wrapper(*args, spec: Spec, state: Any, **kw):
+        # TODO: update condition to "phase1+" if we have phase2
+        if spec.fork == PHASE1 and not is_full_crosslink(spec, state):
+            dump_skipping_message("only for full crosslink")
+            return None
+        return fn(*args, spec=spec, state=state, **kw)
+    return wrapper

tests/core/pyspec/eth2spec/test/exceptions.py

@@ -0,0 +1,2 @@
+class SkippedTest(Exception):
+    ...

tests/core/pyspec/eth2spec/test/helpers/custody.py

@@ -179,15 +179,6 @@ def get_sample_shard_transition(spec, start_slot, block_lengths):
     return shard_transition
 
 
-def get_custody_secret(spec, state, validator_index, epoch=None):
-    period = spec.get_custody_period_for_validator(validator_index, epoch if epoch is not None
-                                                   else spec.get_current_epoch(state))
-    epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, validator_index)
-    domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign)
-    signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain)
-    return bls.Sign(privkeys[validator_index], signing_root)
-
-
 def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=True):
     test_vector = get_custody_test_vector(length)
     offset = 0

tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py

@@ -35,8 +35,3 @@ def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
     shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks)
     return shard_transition
-
-
-def is_full_crosslink(spec, state):
-    epoch = spec.compute_epoch_at_slot(state.slot)
-    return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state)

tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py

@@ -193,9 +193,6 @@ def test_participants_already_slashed(spec, state):
     yield from run_attester_slashing_processing(spec, state, attester_slashing, False)
 
 
-# Some of the following tests are phase0 only: phase 1 lists participants with bitfields instead of index list.
-
-
 @with_all_phases
 @spec_state_test
 @always_bls