Skip to content

Remove prune_dependency_tree and reuse getfixtureclosure logic #11243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Remove prune_dependency_tree and reuse getfixtureclosure logic #11243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 38 additions & 43 deletions src/_pytest/fixtures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,33 +337,6 @@ class FuncFixtureInfo:
# sequence is ordered from furthest to closes to the function.
name2fixturedefs: dict[str, Sequence[FixtureDef[Any]]]

def prune_dependency_tree(self) -> None:
"""Recompute names_closure from initialnames and name2fixturedefs.

Can only reduce names_closure, which means that the new closure will
always be a subset of the old one. The order is preserved.

This method is needed because direct parametrization may shadow some
of the fixtures that were included in the originally built dependency
tree. In this way the dependency tree can get pruned, and the closure
of argnames may get reduced.
"""
closure: set[str] = set()
working_set = set(self.initialnames)
while working_set:
argname = working_set.pop()
# Argname may be something not included in the original names_closure,
# in which case we ignore it. This currently happens with pseudo
# FixtureDefs which wrap 'get_direct_param_fixture_func(request)'.
# So they introduce the new dependency 'request' which might have
# been missing in the original tree (closure).
if argname not in closure and argname in self.names_closure:
closure.add(argname)
if argname in self.name2fixturedefs:
working_set.update(self.name2fixturedefs[argname][-1].argnames)

self.names_closure[:] = sorted(closure, key=self.names_closure.index)


class FixtureRequest(abc.ABC):
"""The type of the ``request`` fixture.
Expand Down Expand Up @@ -968,7 +941,6 @@ def _eval_scope_callable(
return result


@final
class FixtureDef(Generic[FixtureValue]):
"""A container for a fixture definition.

Expand Down Expand Up @@ -1136,6 +1108,26 @@ def __repr__(self) -> str:
return f"<FixtureDef argname={self.argname!r} scope={self.scope!r} baseid={self.baseid!r}>"


class IdentityFixtureDef(FixtureDef[FixtureValue]):
def __init__(
self,
config: Config,
argname: str,
scope: Scope | _ScopeName | Callable[[str, Config], _ScopeName] | None,
*,
_ispytest: bool = False,
):
super().__init__(
config,
"",
argname,
lambda request: request.param,
scope,
None,
_ispytest=_ispytest,
)


def resolve_fixture_function(
fixturedef: FixtureDef[FixtureValue], request: FixtureRequest
) -> _FixtureFunc[FixtureValue]:
Expand Down Expand Up @@ -1567,10 +1559,11 @@ def getfixtureinfo(
initialnames = deduplicate_names(autousenames, usefixturesnames, argnames)

direct_parametrize_args = _get_direct_parametrize_args(node)

names_closure, arg2fixturedefs = self.getfixtureclosure(
arg2fixturedefs: dict[str, Sequence[FixtureDef[Any]]] = {}
names_closure = self.getfixtureclosure(
parentnode=node,
initialnames=initialnames,
arg2fixturedefs=arg2fixturedefs,
ignore_args=direct_parametrize_args,
)

Expand Down Expand Up @@ -1622,30 +1615,32 @@ def getfixtureclosure(
self,
parentnode: nodes.Node,
initialnames: tuple[str, ...],
arg2fixturedefs: dict[str, Sequence[FixtureDef[Any]]],
ignore_args: AbstractSet[str],
) -> tuple[list[str], dict[str, Sequence[FixtureDef[Any]]]]:
) -> list[str]:
# Collect the closure of all fixtures, starting with the given
# fixturenames as the initial set. As we have to visit all
# factory definitions anyway, we also return an arg2fixturedefs
# mapping so that the caller can reuse it and does not have
# to re-discover fixturedefs again for each fixturename
# initialnames containing function arguments, `usefixture` markers
# and `autouse` fixtures as the initial set. As we have to visit all
# factory definitions anyway, we also populate arg2fixturedefs mapping
# for the args missing therein so that the caller can reuse it and does
# not have to re-discover fixturedefs again for each fixturename
# (discovering matching fixtures for a given name/node is expensive).

fixturenames_closure = list(initialnames)

arg2fixturedefs: dict[str, Sequence[FixtureDef[Any]]] = {}
lastlen = -1
while lastlen != len(fixturenames_closure):
lastlen = len(fixturenames_closure)
for argname in fixturenames_closure:
if argname in ignore_args:
continue
if argname in arg2fixturedefs:
continue
fixturedefs = self.getfixturedefs(argname, parentnode)
if fixturedefs:
arg2fixturedefs[argname] = fixturedefs

if argname not in arg2fixturedefs:
fixturedefs = self.getfixturedefs(argname, parentnode)
if fixturedefs:
arg2fixturedefs[argname] = fixturedefs
else:
fixturedefs = arg2fixturedefs[argname]
if fixturedefs and not isinstance(fixturedefs[-1], IdentityFixtureDef):
# Add dependencies from this fixture.
# If it overrides a fixture with the same name and requests
# it, also add dependencies from the overridden fixtures in
Expand All @@ -1667,7 +1662,7 @@ def sort_by_scope(arg_name: str) -> Scope:
return fixturedefs[-1]._scope

fixturenames_closure.sort(key=sort_by_scope, reverse=True)
return fixturenames_closure, arg2fixturedefs
return fixturenames_closure

def pytest_generate_tests(self, metafunc: Metafunc) -> None:
"""Generate new tests based on parametrized fixtures used by the given metafunc"""
Expand Down
70 changes: 40 additions & 30 deletions src/_pytest/python.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@
from _pytest.config import hookimpl
from _pytest.config.argparsing import Parser
from _pytest.deprecated import check_ispytest
from _pytest.fixtures import FixtureDef
from _pytest.fixtures import FixtureRequest
from _pytest.fixtures import _get_direct_parametrize_args
from _pytest.fixtures import FuncFixtureInfo
from _pytest.fixtures import get_scope_node
from _pytest.fixtures import IdentityFixtureDef
from _pytest.main import Session
from _pytest.mark import ParameterSet
from _pytest.mark.structures import _HiddenParam
Expand Down Expand Up @@ -448,8 +448,6 @@ def _genfunctions(self, name: str, funcobj) -> Iterator[Function]:
definition = FunctionDefinition.from_parent(self, name=name, callobj=funcobj)
fixtureinfo = definition._fixtureinfo

# pytest_generate_tests impls call metafunc.parametrize() which fills
# metafunc._calls, the outcome of the hook.
metafunc = Metafunc(
definition=definition,
fixtureinfo=fixtureinfo,
Expand All @@ -458,24 +456,34 @@ def _genfunctions(self, name: str, funcobj) -> Iterator[Function]:
module=module,
_ispytest=True,
)
methods = []

def prune_dependency_tree_if_test_is_directly_parametrized(
metafunc: Metafunc,
) -> None:
# Direct (those with `indirect=False`) parametrizations taking place in
# module/class-specific `pytest_generate_tests` hooks, a.k.a dynamic direct
# parametrizations using `metafunc.parametrize`, may have shadowed some
# fixtures, making some fixtures no longer reachable. Update the dependency
# tree to reflect what the item really needs.
# Note that direct parametrizations using `@pytest.mark.parametrize` have
# already been considered into making the closure using the `ignore_args`
# arg to `getfixtureclosure`.
if metafunc._has_direct_parametrization:
metafunc._update_dependency_tree()

methods = [prune_dependency_tree_if_test_is_directly_parametrized]
if hasattr(module, "pytest_generate_tests"):
methods.append(module.pytest_generate_tests)
if cls is not None and hasattr(cls, "pytest_generate_tests"):
methods.append(cls().pytest_generate_tests)

# pytest_generate_tests impls call metafunc.parametrize() which fills
# metafunc._calls, the outcome of the hook.
self.ihook.pytest_generate_tests.call_extra(methods, dict(metafunc=metafunc))

if not metafunc._calls:
yield Function.from_parent(self, name=name, fixtureinfo=fixtureinfo)
else:
metafunc._recompute_direct_params_indices()
# Direct parametrizations taking place in module/class-specific
# `metafunc.parametrize` calls may have shadowed some fixtures, so make sure
# we update what the function really needs a.k.a its fixture closure. Note that
# direct parametrizations using `@pytest.mark.parametrize` have already been considered
# into making the closure using `ignore_args` arg to `getfixtureclosure`.
fixtureinfo.prune_dependency_tree()

for callspec in metafunc._calls:
subname = f"{name}[{callspec.id}]" if callspec._idlist else name
yield Function.from_parent(
Expand Down Expand Up @@ -1108,12 +1116,8 @@ def id(self) -> str:
return "-".join(self._idlist)


def get_direct_param_fixture_func(request: FixtureRequest) -> Any:
return request.param


# Used for storing pseudo fixturedefs for direct parametrization.
name2pseudofixturedef_key = StashKey[dict[str, FixtureDef[Any]]]()
name2pseudofixturedef_key = StashKey[dict[str, IdentityFixtureDef[Any]]]()


@final
Expand Down Expand Up @@ -1162,6 +1166,9 @@ def __init__(

self._params_directness: dict[str, Literal["indirect", "direct"]] = {}

# Whether it's ever been directly parametrized, i.e. with `indirect=False`.
self._has_direct_parametrization = False

def parametrize(
self,
argnames: str | Sequence[str],
Expand Down Expand Up @@ -1308,24 +1315,21 @@ def parametrize(
node = collector.session
else:
assert False, f"Unhandled missing scope: {scope}"
default: dict[str, FixtureDef[Any]] = {}
default: dict[str, IdentityFixtureDef[Any]] = {}
name2pseudofixturedef = node.stash.setdefault(
name2pseudofixturedef_key, default
)
for argname in argnames:
if arg_directness[argname] == "indirect":
continue
self._has_direct_parametrization = True
if name2pseudofixturedef is not None and argname in name2pseudofixturedef:
fixturedef = name2pseudofixturedef[argname]
else:
fixturedef = FixtureDef(
config=self.config,
baseid="",
fixturedef = IdentityFixtureDef(
config=self.definition.config,
argname=argname,
func=get_direct_param_fixture_func,
scope=scope_,
params=None,
ids=None,
_ispytest=True,
)
if name2pseudofixturedef is not None:
Expand Down Expand Up @@ -1485,11 +1489,17 @@ def _validate_if_using_arg_names(
pytrace=False,
)

def _recompute_direct_params_indices(self) -> None:
for argname, param_type in self._params_directness.items():
if param_type == "direct":
for i, callspec in enumerate(self._calls):
callspec.indices[argname] = i
def _update_dependency_tree(self) -> None:
definition = self.definition
assert definition.parent is not None
fm = definition.parent.session._fixturemanager
fixture_closure = fm.getfixtureclosure(
parentnode=definition,
initialnames=definition._fixtureinfo.initialnames,
arg2fixturedefs=definition._fixtureinfo.name2fixturedefs,
ignore_args=_get_direct_parametrize_args(definition),
)
definition._fixtureinfo.names_closure[:] = fixture_closure


def _find_parametrized_scope(
Expand Down
Loading
Loading