diff --git a/doc/source/user/build_system/index.rst b/doc/source/user/build_system/index.rst index 8c2a818b..989c418c 100644 --- a/doc/source/user/build_system/index.rst +++ b/doc/source/user/build_system/index.rst @@ -38,5 +38,7 @@ A full reference documentation on the CAPI2 core file format can be found in the filters.rst flags.rst generators.rst + virtual_cores.rst + mappings.rst hooks.rst vpi.rst diff --git a/doc/source/user/build_system/mappings.rst b/doc/source/user/build_system/mappings.rst new file mode 100644 index 00000000..f110debd --- /dev/null +++ b/doc/source/user/build_system/mappings.rst @@ -0,0 +1,40 @@ +.. _ug_build_system_mappings: + +Mappings: Replace cores in the dependency tree +============================================== + +Mappings allow a user of a core to substitute dependencies of the core with other cores without having to edit any of the core or it's dependencies files. +An example use case is making use of an existing core but substituting one of it's dependencies with a version that some desired changes (e.g. bug fixes). + +If you are looking to provide a core with multiple implementations, virtual cores is the recommended and more semantic solution. +See :ref:`ug_build_system_virtual_cores` for more information on virtual cores. +Note: virtual cores can also be substituted in mappings. + +Declaring mappings +------------------ + +Each core file can have one mapping. +An example mapping core file: + +.. code:: yaml + + name: "local:map:override_fpu_and_fifo" + mapping: + "vendor:lib:fpu": "local:lib:fpu" + "vendor:lib:fifo": "local:lib:fifo" + +The example above is a core file with only a mapping property, but any core file may contain a mapping in addition to other properties (e.g. filesets, targets & generators). + +Applying mappings +----------------- + +To apply a mapping, provide the VLNV of the core file that contains the desired +mapping with `fusesoc run`'s `--mapping` argument. Multiple mappings may be +provided as shown below. + +.. code:: sh + + fusesoc run \ + --mapping local:map:override_fpu_and_fifo \ + --mapping local:map:another_mapping \ + vendor:top:main diff --git a/doc/source/user/build_system/virtual_cores.rst b/doc/source/user/build_system/virtual_cores.rst new file mode 100644 index 00000000..08728ce0 --- /dev/null +++ b/doc/source/user/build_system/virtual_cores.rst @@ -0,0 +1,8 @@ +.. _ug_build_system_virtual_cores: + +Virtual Cores: Provide a common interface +========================================= + +.. todo:: + + Document virtual cores. diff --git a/fusesoc/capi2/core.py b/fusesoc/capi2/core.py index dac0188a..3154f985 100644 --- a/fusesoc/capi2/core.py +++ b/fusesoc/capi2/core.py @@ -10,6 +10,8 @@ import shutil import warnings from filecmp import cmp +from types import MappingProxyType +from typing import Mapping, Optional from fusesoc import utils from fusesoc.capi2.coredata import CoreData @@ -630,3 +632,7 @@ def get_name(self): def get_description(self): return self._coredata.get_description() + + @property + def mapping(self) -> Optional[Mapping[str, str]]: + return MappingProxyType(self._coredata.get("mapping", {})) diff --git a/fusesoc/capi2/json_schema.py b/fusesoc/capi2/json_schema.py index 71e672f2..f981da39 100644 --- a/fusesoc/capi2/json_schema.py +++ b/fusesoc/capi2/json_schema.py @@ -52,6 +52,15 @@ "items": { "type": "string" } + }, + "mapping": { + "description": "", + "type": "object", + "patternProperties": { + "^.+$": { + "type": "string" + } + } } }, "additionalProperties": false, @@ -77,18 +86,18 @@ "type": "object", "properties": { "define": { - "description": "Defines to be used for this file. These defines will be added to those specified in the target parameters section. If a define is specified both here and in the target parameter section, the value specified here will take precedence. The parameter default value can be set here with ``param=value``", - "type": "object", - "patternProperties": { - "^.+$": { - "anyOf": [ - { "type": "string" }, - { "type": "number" }, - { "type": "boolean"} - ] - } - } - }, + "description": "Defines to be used for this file. These defines will be added to those specified in the target parameters section. If a define is specified both here and in the target parameter section, the value specified here will take precedence. The parameter default value can be set here with ``param=value``", + "type": "object", + "patternProperties": { + "^.+$": { + "anyOf": [ + { "type": "string" }, + { "type": "number" }, + { "type": "boolean"} + ] + } + } + }, "is_include_file": { "description": "Treats file as an include file when true", "type": "boolean" diff --git a/fusesoc/coremanager.py b/fusesoc/coremanager.py index bdb74df0..954fb5b7 100644 --- a/fusesoc/coremanager.py +++ b/fusesoc/coremanager.py @@ -5,6 +5,9 @@ import logging import os import pathlib +from itertools import chain +from types import MappingProxyType +from typing import Iterable, Mapping from okonomiyaki.versions import EnpkgVersion from simplesat.constraints import PrettyPackageStringParser, Requirement @@ -33,6 +36,8 @@ def __str__(self): class CoreDB: + _mapping: Mapping[str, str] = MappingProxyType({}) + def __init__(self): self._cores = {} self._solver_cache = {} @@ -139,6 +144,98 @@ def _lockfile_replace(self, core: Vlnv): ) ) + def _mapping_apply(self, core: Vlnv): + """If the core matches a mapping, apply the mapping, mutating the given core.""" + remapping = self._mapping.get(core.vln_str()) + + if not remapping: + return + + previous_vlnv = str(core) + (core.vendor, core.library, core.name) = remapping.split(":") + + logger.info(f"Mapped {previous_vlnv} to {core}.") + + def mapping_set(self, mapping_vlnvs: Iterable[str]) -> None: + """Construct a mapping from the given cores' mappings. + + Takes the VLNV strings of the cores' mappings to apply. + Verifies the mappings and applies them. + """ + if self._mapping: + raise RuntimeError( + "Due to implementation details, mappings can only be applied once." + ) + + mappings = {} + for mapping_vlnv in mapping_vlnvs: + new_mapping_name = str(Vlnv(mapping_vlnv)) + new_mapping_core = self._cores.get(new_mapping_name) + if not new_mapping_core: + raise RuntimeError(f"The core '{mapping_vlnv}' wasn't found.") + + new_mapping_raw = new_mapping_core["core"].mapping + if not new_mapping_raw: + raise RuntimeError( + f"The core '{mapping_vlnv}' doesn't contain a mapping." + ) + + have_versions = list( + filter( + lambda vlnv: (vlnv.relation, vlnv.version) != (">=", "0"), + map(Vlnv, chain(new_mapping_raw.keys(), new_mapping_raw.values())), + ) + ) + if have_versions: + raise RuntimeError( + "Versions cannot be given as part of a mapping." + f"\nThe mapping of {mapping_vlnv} following has" + f" the following version constraints:\n\t{have_versions}" + ) + + new_mapping = { + Vlnv(source).vln_str(): Vlnv(destination).vln_str() + for source, destination in new_mapping_raw.items() + } + new_src_set = new_mapping.keys() + new_dest_set = frozenset(new_mapping.values()) + curr_src_set = mappings.keys() + curr_dest_set = frozenset(mappings.values()) + + new_src_dest_overlap = new_mapping.keys() & new_dest_set + if new_src_dest_overlap: + raise RuntimeError( + "Recursive mappings are not supported." + f"\nThe mapping {mapping_vlnv} has the following VLNV's" + f" in both it's sources and destinations:\n\t{new_src_dest_overlap}." + ) + + source_overlap = curr_src_set & new_src_set + if source_overlap: + raise RuntimeError( + f"The following sources are in multiple mappings:\n\t{source_overlap}." + ) + + dest_overlap = new_dest_set & curr_dest_set + if dest_overlap: + raise RuntimeError( + f"The following destinations are in multiple mappings:\n\t{dest_overlap}." + ) + + src_dest_overlap = (new_src_set | curr_src_set) & ( + new_dest_set | curr_dest_set + ) + if src_dest_overlap: + raise RuntimeError( + "Recursive mappings are not supported." + f"\nThe following VLNV's are in both the sources and" + f" destinations:\n\t{src_dest_overlap}." + ) + + mappings.update(new_mapping) + + self._mapping = MappingProxyType(mappings) + def solve(self, top_core, flags): return self._solve(top_core, flags) @@ -225,6 +322,7 @@ def eq_vln(this, that): _depends = core.get_depends(_flags) if _depends: for depend in _depends: + self._mapping_apply(depend) self._lockfile_replace(depend) _s = "; depends ( {} )" package_str += _s.format(self._parse_depend(_depends)) diff --git a/fusesoc/main.py b/fusesoc/main.py index 58f2c294..f4af03e7 100644 --- a/fusesoc/main.py +++ b/fusesoc/main.py @@ -307,6 +307,8 @@ def run(fs, args): else: flags[flag] = True + fs.cm.db.mapping_set(args.mapping) + if args.lockfile is not None: try: fs.cm.db.load_lockfile(args.lockfile) @@ -679,6 +681,12 @@ def get_parser(): help="Lockfile file path", type=pathlib.Path, ) + parser_run.add_argument( + "--mapping", + help="The VLNV of a core's mapping to apply.", + default=[], + action="append", + ) parser_run.set_defaults(func=run) # config subparser diff --git a/fusesoc/vlnv.py b/fusesoc/vlnv.py index 255424cd..cfe00a4e 100644 --- a/fusesoc/vlnv.py +++ b/fusesoc/vlnv.py @@ -108,6 +108,8 @@ def __str__(self): self.vendor, self.library, self.name, self.version, revision ) + __repr__ = __str__ + def __hash__(self): return hash(str(self)) diff --git a/tests/capi2_cores/mapping/a.core b/tests/capi2_cores/mapping/a.core new file mode 100644 index 00000000..1d1ab187 --- /dev/null +++ b/tests/capi2_cores/mapping/a.core @@ -0,0 +1,8 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: test_mapping:l:a:0 +virtual: + - test_mapping:v:x:0 diff --git a/tests/capi2_cores/mapping/b.core b/tests/capi2_cores/mapping/b.core new file mode 100644 index 00000000..72f4b286 --- /dev/null +++ b/tests/capi2_cores/mapping/b.core @@ -0,0 +1,6 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: test_mapping:l:b:0 diff --git a/tests/capi2_cores/mapping/c.core b/tests/capi2_cores/mapping/c.core new file mode 100644 index 00000000..e4da989e --- /dev/null +++ b/tests/capi2_cores/mapping/c.core @@ -0,0 +1,9 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: test_mapping:l:c:0 + +mapping: + "test_mapping:l:c": "test_mapping:v:x" diff --git a/tests/capi2_cores/mapping/d.core b/tests/capi2_cores/mapping/d.core new file mode 100644 index 00000000..46d67ac9 --- /dev/null +++ b/tests/capi2_cores/mapping/d.core @@ -0,0 +1,10 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: test_mapping:l:d:0 + +mapping: + "test_mapping:l:b": "test_mapping:l:d" + "test_mapping:l:c": "test_mapping:l:e" diff --git a/tests/capi2_cores/mapping/e.core b/tests/capi2_cores/mapping/e.core new file mode 100644 index 00000000..82b3542d --- /dev/null +++ b/tests/capi2_cores/mapping/e.core @@ -0,0 +1,6 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: test_mapping:l:e:0 diff --git a/tests/capi2_cores/mapping/f.core b/tests/capi2_cores/mapping/f.core new file mode 100644 index 00000000..6038c1a7 --- /dev/null +++ b/tests/capi2_cores/mapping/f.core @@ -0,0 +1,6 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: test_mapping:l:f:0 diff --git a/tests/capi2_cores/mapping/map_rec.core b/tests/capi2_cores/mapping/map_rec.core new file mode 100644 index 00000000..86061e1f --- /dev/null +++ b/tests/capi2_cores/mapping/map_rec.core @@ -0,0 +1,9 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: "test_mapping:m:map_rec:0" +mapping: + "test_mapping:l:b": "test_mapping:l:c" + "test_mapping:l:c": "test_mapping:l:b" diff --git a/tests/capi2_cores/mapping/map_vers.core b/tests/capi2_cores/mapping/map_vers.core new file mode 100644 index 00000000..17b32f11 --- /dev/null +++ b/tests/capi2_cores/mapping/map_vers.core @@ -0,0 +1,8 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: test_mapping:m:map_vers:0 +mapping: + "test_mapping:l:b:0": "test_mapping:l:c" diff --git a/tests/capi2_cores/mapping/top.core b/tests/capi2_cores/mapping/top.core new file mode 100644 index 00000000..25a84e83 --- /dev/null +++ b/tests/capi2_cores/mapping/top.core @@ -0,0 +1,21 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: "test_mapping:t:top:0" +filesets: + rtl: + depend: + - "test_mapping:v:x:0" + - "test_mapping:l:b:0" + - "test_mapping:l:c:0" + +mapping: + "test_mapping:v:x": "test_mapping:l:f" + +targets: + default: + filesets: + - rtl + toplevel: top diff --git a/tests/test_coremanager.py b/tests/test_coremanager.py index 4019708a..ad68189f 100644 --- a/tests/test_coremanager.py +++ b/tests/test_coremanager.py @@ -367,6 +367,98 @@ def test_virtual_non_deterministic_virtual(caplog): ] +def test_mapping_success_cases(): + from pathlib import Path + + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv + + core_dir = Path(__file__).parent / "capi2_cores" / "mapping" + + top = "test_mapping:t:top" + top_vlnv = Vlnv(top) + + for mappings, expected_deps in ( + ( + [], + { + "test_mapping:t:top:0", + "test_mapping:l:a:0", + "test_mapping:l:b:0", + "test_mapping:l:c:0", + }, + ), + ( + [top], + { + "test_mapping:t:top:0", + "test_mapping:l:f:0", + "test_mapping:l:b:0", + "test_mapping:l:c:0", + }, + ), + ( + ["test_mapping:l:d"], + { + "test_mapping:t:top:0", + "test_mapping:l:a:0", + "test_mapping:l:d:0", + "test_mapping:l:e:0", + }, + ), + ( + [top, "test_mapping:l:d"], + { + "test_mapping:t:top:0", + "test_mapping:l:f:0", + "test_mapping:l:d:0", + "test_mapping:l:e:0", + }, + ), + ( + ["test_mapping:l:c"], + { + "test_mapping:t:top:0", + "test_mapping:l:a:0", + "test_mapping:l:b:0", + }, + ), + ): + cm = CoreManager(Config()) + cm.add_library(Library("mapping_test", core_dir), []) + + cm.db.mapping_set(mappings) + + actual_deps = {str(c) for c in cm.get_depends(top_vlnv, {})} + + assert expected_deps == actual_deps + + +def test_mapping_failure_cases(): + from pathlib import Path + + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager + from fusesoc.librarymanager import Library + + core_dir = Path(__file__).parent / "capi2_cores" / "mapping" + + for mappings in ( + ["test_mapping:m:non_existent"], + ["test_mapping:m:map_vers"], + ["test_mapping:m:map_rec"], + ["test_mapping:t:top", "test_mapping:l:c"], + ["test_mapping:t:top", "test_mapping:t:top"], + ): + cm = CoreManager(Config()) + cm.add_library(Library("mapping_test", core_dir), []) + + with pytest.raises(RuntimeError): + cm.db.mapping_set(mappings) + + def test_lockfile(caplog): """ Test core selection with a core pinned by a lock file