From 5525cc98ccfd5598c74764691c3b728435631db6 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 22 Jan 2025 22:36:46 +1100 Subject: [PATCH 1/4] Update Fulu configs --- configs/mainnet.yaml | 3 +-- configs/minimal.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index e54db49661..7f96d087c9 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -171,8 +171,7 @@ MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 CUSTODY_REQUIREMENT: 4 MAX_BLOBS_PER_BLOCK_FULU: 12 -# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU` -MAX_REQUEST_BLOB_SIDECARS_FULU: 1536 +MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096 # Whisk # `Epoch(2**8)` diff --git a/configs/minimal.yaml b/configs/minimal.yaml index a15314bb1f..559e04d70e 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -172,8 +172,7 @@ MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 CUSTODY_REQUIREMENT: 4 MAX_BLOBS_PER_BLOCK_FULU: 12 -# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU` -MAX_REQUEST_BLOB_SIDECARS_FULU: 1536 +MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096 # Whisk WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4 From b1de7ef7d0382dd1680d469f0cb603c0c3fc2182 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:35:10 -0600 Subject: [PATCH 2/4] Move dependencies to pyproject.toml (#4093) --- .circleci/config.yml | 4 +- Makefile | 3 +- pyproject.toml | 55 ++++++++++ setup.py | 102 ++++++------------ solidity_deposit_contract/Makefile | 2 +- .../requirements_preinstallation.txt | 0 6 files changed, 92 insertions(+), 74 deletions(-) create mode 100644 pyproject.toml rename requirements_preinstallation.txt => solidity_deposit_contract/web3_tester/requirements_preinstallation.txt (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 38bd6f422d..c1120b719f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,13 +36,13 @@ commands: steps: - restore_cached_venv: venv_name: v32-pyspec - reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }} + reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: venv_name: v32-pyspec - reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }} + reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }} venv_path: ./venv jobs: checkout_specs: diff --git a/Makefile b/Makefile index 09e914c3ca..a845c69685 100644 --- a/Makefile +++ b/Makefile @@ -61,10 +61,9 @@ PIP_VENV = $(VENV)/bin/pip3 CODESPELL_VENV = $(VENV)/bin/codespell # Make a virtual environment will all of the necessary dependencies. -$(VENV): requirements_preinstallation.txt +$(VENV): pyproject.toml @echo "Creating virtual environment" @python3 -m venv $(VENV) - @$(PIP_VENV) install -r requirements_preinstallation.txt ############################################################################### # Specification diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..d9485f2e8c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,55 @@ +[build-system] +requires = [ + "marko==1.0.2", + "ruamel.yaml==0.17.21", + "setuptools==75.8.0", + "wheel==0.45.1", +] + +[project] +name = "eth2spec" +dynamic = ["version"] +authors = [{ name = "ethereum" }] +description = "Ethereum consensus layer specifications package" +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.9,<4.0" +dependencies = [ + "curdleproofs==0.1.2", + "eth-typing==3.5.2", + "eth-utils==2.3.2", + "lru-dict==1.2.0", + "marko==1.0.2", + "milagro_bls_binding==1.9.0", + "py_arkworks_bls12381==0.3.8", + "py_ecc==6.0.0", + "pycryptodome==3.21.0", + "remerkleable==0.1.28", + "ruamel.yaml==0.17.21", + "setuptools==75.8.0", + "trie==3.0.1", +] + +[project.optional-dependencies] +test = [ + "pytest-cov==6.0.0", + "pytest-xdist==3.6.1", + "pytest==8.3.4", +] +lint = [ + "codespell==2.4.0", + "flake8==5.0.4", + "mypy==0.981", + "pylint==3.3.1", +] +generator = [ + "filelock==3.17.0", + "pathos==0.3.0", + "pytest==8.3.4", + "python-snappy==0.7.3", +] +docs = [ + "mdx-truly-sane-lists==1.3", + "mkdocs-awesome-pages-plugin==2.8.0", + "mkdocs-material==9.1.5", + "mkdocs==1.4.2", +] diff --git a/setup.py b/setup.py index 93c7739a52..8bfcbed222 100644 --- a/setup.py +++ b/setup.py @@ -1,74 +1,62 @@ -from setuptools import setup, find_packages, Command -from setuptools.command.build_py import build_py -from distutils import dir_util -from distutils.util import convert_path -from pathlib import Path +import ast +import copy +import json +import logging import os import string -from typing import Dict, List, Sequence, Optional, Tuple -import ast -import subprocess import sys -import copy +import warnings + from collections import OrderedDict -import json +from distutils import dir_util +from distutils.util import convert_path from functools import lru_cache +from marko.block import Heading, FencedCode, LinkRefDef, BlankLine +from marko.ext.gfm import gfm +from marko.ext.gfm.elements import Table +from marko.inline import CodeSpan +from pathlib import Path +from ruamel.yaml import YAML +from setuptools import setup, find_packages, Command +from setuptools.command.build_py import build_py +from typing import Dict, List, Sequence, Optional, Tuple + +pysetup_path = os.path.abspath(os.path.dirname(__file__)) +sys.path.insert(0, pysetup_path) from pysetup.constants import ( - # code names PHASE0, - # misc ETH2_SPEC_COMMENT_PREFIX, ) -from pysetup.spec_builders import spec_builders -from pysetup.typing import ( - BuildTarget, - ProtocolDefinition, - SpecObject, - VariableDefinition, -) from pysetup.helpers import ( combine_spec_objects, dependency_order_class_objects, objects_to_spec, parse_config_vars, ) -from pysetup.md_doc_paths import get_md_doc_paths +from pysetup.md_doc_paths import ( + get_md_doc_paths +) +from pysetup.spec_builders import ( + spec_builders +) +from pysetup.typing import ( + BuildTarget, + ProtocolDefinition, + SpecObject, + VariableDefinition, +) + # Ignore '1.5.0-alpha.*' to '1.5.0a*' messages. -import warnings warnings.filterwarnings('ignore', message='Normalizing .* to .*') # Ignore 'running' and 'creating' messages -import logging class PyspecFilter(logging.Filter): def filter(self, record): return not record.getMessage().startswith(('running ', 'creating ')) logging.getLogger().addFilter(PyspecFilter()) -# NOTE: have to programmatically include third-party dependencies in `setup.py`. -def installPackage(package: str): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', package]) - -RUAMEL_YAML_VERSION = "ruamel.yaml==0.17.21" -try: - import ruamel.yaml -except ImportError: - installPackage(RUAMEL_YAML_VERSION) - -from ruamel.yaml import YAML - -MARKO_VERSION = "marko==1.0.2" -try: - import marko -except ImportError: - installPackage(MARKO_VERSION) - -from marko.block import Heading, FencedCode, LinkRefDef, BlankLine -from marko.inline import CodeSpan -from marko.ext.gfm import gfm -from marko.ext.gfm.elements import Table - @lru_cache(maxsize=None) def _get_name_from_heading(heading: Heading) -> Optional[str]: @@ -550,12 +538,9 @@ def run(self): spec_version = f.read().strip() setup( - name='eth2spec', version=spec_version, - description="Eth2 spec, provided as Python package for tooling and testing", long_description=readme, long_description_content_type="text/markdown", - author="ethereum", url="https://github.com/ethereum/consensus-specs", include_package_data=False, package_data={ @@ -575,25 +560,4 @@ def run(self): packages=find_packages(where='tests/core/pyspec') + ['configs', 'presets', 'specs', 'presets', 'sync'], py_modules=["eth2spec"], cmdclass=commands, - python_requires=">=3.9, <4", - extras_require={ - "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], - "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==3.3.1", "codespell==2.4.0"], - "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"], - "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3", "mkdocs-awesome-pages-plugin==2.8.0"] - }, - install_requires=[ - "eth-utils>=2.0.0,<3", - "eth-typing>=3.2.0,<4.0.0", - "pycryptodome>=3.19.1", - "py_ecc==6.0.0", - "milagro_bls_binding==1.9.0", - "remerkleable==0.1.28", - "trie>=3,<4", - RUAMEL_YAML_VERSION, - "lru-dict==1.2.0", - MARKO_VERSION, - "py_arkworks_bls12381==0.3.8", - "curdleproofs==0.1.2", - ] ) diff --git a/solidity_deposit_contract/Makefile b/solidity_deposit_contract/Makefile index a353d931bb..60d45ede49 100644 --- a/solidity_deposit_contract/Makefile +++ b/solidity_deposit_contract/Makefile @@ -30,7 +30,7 @@ install_deposit_contract_web3_tester: @cd $(DEPOSIT_CONTRACT_TESTER_DIR); \ python3 -m venv venv; \ source venv/bin/activate; \ - python3 -m pip install -r ../../requirements_preinstallation.txt; \ + python3 -m pip install -r requirements_preinstallation.txt; \ python3 -m pip install -r requirements.txt test_deposit_contract_web3_tests: diff --git a/requirements_preinstallation.txt b/solidity_deposit_contract/web3_tester/requirements_preinstallation.txt similarity index 100% rename from requirements_preinstallation.txt rename to solidity_deposit_contract/web3_tester/requirements_preinstallation.txt From 1d9577aa5188c1245973218ade3d9712486867db Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:53:09 -0600 Subject: [PATCH 3/4] Bump version to 1.5.0-beta.1 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index ba25d3754e..591650acef 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.5.0-beta.0 +1.5.0-beta.1 From b87fbacdc59824fa21a8c2fcacf14cfda3ad538e Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:55:05 -0600 Subject: [PATCH 4/4] Add document with release instructions (#4065) --- docs/docs/release.md | 331 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 docs/docs/release.md diff --git a/docs/docs/release.md b/docs/docs/release.md new file mode 100644 index 0000000000..d0d163b79d --- /dev/null +++ b/docs/docs/release.md @@ -0,0 +1,331 @@ +# Release Procedure + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Open a Release Pull Request](#open-a-release-pull-request) +- [Bump the Version](#bump-the-version) +- [Pick a Release Name](#pick-a-release-name) +- [Generate Reference Tests](#generate-reference-tests) + - [Install Git Large File Storage](#install-git-large-file-storage) + - [Prepare Tests Repository](#prepare-tests-repository) + - [Commit Reference Tests](#commit-reference-tests) + - [Bundle Reference Tests](#bundle-reference-tests) +- [Merge the Release Pull Request](#merge-the-release-pull-request) +- [Create Tests Release](#create-tests-release) +- [Create Specs Release](#create-specs-release) +- [Click the Release Buttons](#click-the-release-buttons) +- [Make an Announcement](#make-an-announcement) + + + + +## Introduction + +This document describes the necessary steps to produce a consensus-specs release. + +## Open a Release Pull Request + +> [!NOTE] +> Try to do this at least a few days prior to the release. + +First, create a PR which merges `dev` into `master`. + +> [!TIP] +> Click on the following link to draft a PR: +> * https://github.com/ethereum/consensus-specs/compare/dev...master?expand=1 + +Title this PR "Release <version>" (_e.g._, "Release v1.5.0-alpha.10"). + +In the PR's description, provide a list of items that must be done. At the bottom, list unmerged PRs +which are to be included in the release; this signals to other maintainers and developers that they +should review these PRs soon. + +```markdown +- [ ] testgen +- [ ] version bump +- [ ] #1234 +- [ ] #2345 +- [ ] #3456 +``` + +## Bump the Version + +Next, update the `VERSION.txt` file which contains the eth2spec version. + +> [!TIP] +> Click on the following link to open the GitHub editor for this file: +> * https://github.com/ethereum/consensus-specs/edit/dev/tests/core/pyspec/eth2spec/VERSION.txt + +Next, change the version to the appropriate value and click the "Commit changes..." button. + +For the commit message, put "Bump version to <version>" (_e.g._, "Bump version to 1.5.0-alpha.10"). + +Next, click the "Propose changes" button and proceed to make the PR. + +## Pick a Release Name + +Generally, names are based on some theme. For example, for Electra, releases are named after +Electric-type Pokemon. + +> [!NOTE] +> Please ensure that the name you choose does not have an unwanted meaning in other languages; +use [WordSafety.com](http://wordsafety.com) to check this. + +## Generate Reference Tests + +### Install Git Large File Storage + +The consensus-spec-tests repository uses [Git LFS](https://git-lfs.com) because it contains many +large files. Attempt to install LFS support with the following command. If it does not work, please +refer to the installation directions on their website. + +```bash +git lfs install +``` + +### Prepare Tests Repository + +Next, clone the consensus-spec-tests repository. + +> [!NOTE] +> Only the single latest commit is needed to make the release. Use `--depth=1` to do this. Please +> note that even this may take some time to checkout, as the combined size of the test vectors is +> multiple gigabytes. + +```bash +git clone https://github.com/ethereum/consensus-spec-tests.git --depth=1 +cd consensus-spec-tests +``` + +Next, remove directories which will be overwritten. + +```bash +rm -rf configs presets tests +``` + +### Commit Reference Tests + +Next, change directory to outside of the tests repository. + +```bash +cd .. +``` + +Next, clone the consensus-specs repository. + +```bash +git clone https://github.com/ethereum/consensus-specs.git +cd consensus-specs +``` + +Next, copy the presets and configuration files to the tests repository. + +```bash +cp -r presets ../consensus-spec-tests +cp -r configs ../consensus-spec-tests +``` + +Next, use `make gen_all` to generate all the reference tests. The following command will run all +generators in parallel for maximum speed. The console output is saved to a file so we can check for +errors afterwards. + +> [!TIP] +> Instead of this, another option is to use the test vectors that are automatically generated each +> night. To download these files, click the following link, then click the latest action run, and +> then download all of the artifacts. Please note, if there has been a change to the dev branch +> after the test vectors were generated, you can manually trigger the action with the "Run workflow" +> button. +> +> * https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml +> +> After downloading these artifacts, move them to the `consensus-spec-tests` directory. Then unzip +> each, then untar each of the `*.tar.gz` files. Use `unzip .zip` and `tar -xvf .tar.gz` +> to do this. Note that the "Bundle Reference Tests" section can be skipped if this route is taken. + +```bash +make --jobs gen_all 2>&1 | tee ../consensustestgen.log +``` + +Next, check for errors by searching for "ERROR" in test logfile. + +```bash +grep "ERROR" ../consensustestgen.log +``` + +> [!WARNING] +> If there is an error: (1) determine what the issue is, (2) create/merge a PR to fix the issue, and +> (3) restart the release process. + +Next, change directory to the consensus-spec-tests repository: + +```bash +cd ../consensus-spec-tests +``` + +Next, check that the differences look sane; that there are no unexpected changes. Sometimes there +are several hundred (if not more) changes per release so use your best judgement when reviewing. One +might ensure that there are no changes to old forks and double check a few reference tests to ensure +they were indeed modified in this release. + +```bash +git lfs status +``` + +Next, after reviewing the changes and are reasonably confident that they are okay, stage the +changes. + +```bash +git add . +``` + +Next, commit the changes. + +> [!IMPORTANT] +> Commits to consensus-spec-tests must be signed. Please refer to [Signing +> Commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) +> for instructions on setting this up. + +```bash +git commit --gpg-sign --message "release tests" +``` + +Finally, push the changes. + +```bash +git push origin master +``` + +### Bundle Reference Tests + +Next, delete all empty directories. + +```bash +find . -type d -empty -delete +``` + +Finally, tar each group of tests into separate tarballs. + +```bash +tar -czvf general.tar.gz tests/general +tar -czvf minimal.tar.gz tests/minimal +tar -czvf mainnet.tar.gz tests/mainnet +``` + +## Merge the Release Pull Request + +After successfully generating the test vectors, we have confidence that the release is fine. We can +now merge the release pull request which brings changes from the development branch into the master +branch. Releases are made from the master branch. + +## Create Tests Release + +First, begin to draft a new consensus-spec-tests release. + +> [!TIP] +> Click on the following link to draft a new release: +> * https://github.com/ethereum/consensus-spec-tests/releases/new + +Next, click the "Choose a tag" button to create a new tag. Type in the release version (_e.g._, +"v1.5.0-alpha.10") and click the "Create new tag: <version> on publish" button. + +Next, provide a title "Spec tests for <version>" (_e.g._, "Spec tests for v1.5.0-alpha.10"). + +Next, provide a description. Use the following template: + +```markdown +Spec tests for . + +Detailed changelog can be found in [ specs release](https://github.com/ethereum/consensus-specs/releases/tag/). +``` + +Next, upload the tarballs from the [Bundle Reference Tests]() section to the release. + +> [!NOTE] +> This is expected to take a while if your upload speed is below average. The tarballs are at +> least 1 gigabyte in total. There is a progress bar shown for each artifact. + +Next, if this is an alpha/beta release, please select the "Set as a pre-release" checkbox, otherwise +select the "Set as the latest release" checkbox. + +> [!IMPORTANT] +> Do no click the release button just yet. + +## Create Specs Release + +First, begin to draft a new consensus-specs release. + +> [!TIP] +> Click on the following link to draft a new release: +> * https://github.com/ethereum/consensus-specs/releases/new + +Next, click the "Choose a tag" button to create a new tag. Type in the release version (_e.g._, +"v1.5.0-alpha.10") and click the "Create new tag: <version> on publish" button. + +Next, change the target from `dev` to `master`. + +> [!IMPORTANT] +> Do not forget this to change the target branch. + +Next, provide a title "<release-name>" (_e.g._, "Bellibolt"). + +Next, provide a description. Use the following template: + +```markdown + -- -- + +_PR showing full diff can be found here: _ + +## + +* Add new feature #1234 +* Fix problem in old feature #2345 + +## + +* Fix bug related to feature #3456 + +## Testing, repo, etc + +* Add new test for feature #4567 +* Improve documentation #5678 +``` + +If this is an alpha/beta release, please select the "Set as a pre-release" checkbox, otherwise +select the "Set as the latest release" checkbox. + +> [!IMPORTANT] +> Do no click the release button just yet. + +## Click the Release Buttons + +1. First, click the release button for consensus-specs. + +2. Then, click the release button for consensus-spec-tests. + +> [!NOTE] +> It should be done in this order because the tests release references the specs release. Also, +> we wait to push these buttons at the same time so their time/date will be approximately the same. + +## Make an Announcement + +> [!IMPORTANT] +> In order to do this, you must be granted the appropriate access. + +Finally, make an announcement to the Eth R&D server on Discord. This should be posted in the +`#announcements` channel. This will notify client developers of the new release and they will begin +to incorporate the new reference tests into their client. + +Use the following template for your announcement: + +```markdown +Consensus layer specs -- -- released! + +https://github.com/ethereum/consensus-specs/releases/tag/ + +Test vectors: https://github.com/ethereum/consensus-spec-tests/releases/tag/ +``` \ No newline at end of file