0.10.3: test_looponfail fails

[mtelka]

The test_looponfail test fails:

=========================== short test summary info ============================
FAILED tests/test_pytest_mypy.py::test_looponfail[test_demo] - pexpect.except...
FAILED tests/test_pytest_mypy.py::test_looponfail[__init__] - pexpect.excepti...
============= 2 failed, 48 passed, 6 xfailed in 534.07s (0:08:54) ==============

With this error:

__________________________ test_looponfail[__init__] ___________________________
[gw1] sunos5 -- Python 3.9.16 $(BUILD_DIR)/.tox/py39/bin/python

testdir = <Testdir local('/tmp/pytest-of-marcel/pytest-16/popen-gw1/test_looponfail0')>
module_name = '__init__'

    @pytest.mark.parametrize("module_name", ["__init__", "test_demo"])
    def test_looponfail(testdir, module_name):
        """Ensure that the plugin works with --looponfail."""

        pass_source = textwrap.dedent(
            """\
            def pyfunc(x: int) -> int:
                return x * 2
            """,
        )
        fail_source = textwrap.dedent(
            """\
            def pyfunc(x: int) -> str:
                return x * 2
            """,
        )
        pyfile = testdir.makepyfile(**{module_name: fail_source})
        looponfailroot = testdir.mkdir("looponfailroot")
        looponfailroot_pyfile = looponfailroot.join(pyfile.basename)
        pyfile.move(looponfailroot_pyfile)
        pyfile = looponfailroot_pyfile
        testdir.makeini(
            textwrap.dedent(
                """\
                [pytest]
                looponfailroots = {looponfailroots}
                """.format(
                    looponfailroots=looponfailroot,
                ),
            ),
        )

        child = testdir.spawn_pytest(
            "--mypy --looponfail " + str(pyfile),
            expect_timeout=60.0,
        )

        num_tests = 2
        if module_name == "__init__" and Version("3.10") <= PYTEST_VERSION < Version("6.2"):
            # https://github.com/pytest-dev/pytest/issues/8016
            # Pytest had a bug where it assumed only a Package would have a basename of
            # __init__.py. In this test, Pytest mistakes MypyFile for a Package and
            # returns after collecting only one object (the MypyFileItem).
            num_tests = 1

        def _expect_session():
            child.expect("==== test session starts ====")

        def _expect_failure():
            _expect_session()
            child.expect("==== FAILURES ====")
            child.expect(pyfile.basename + " ____")
            child.expect("2: error: Incompatible return value")
            # if num_tests == 2:
            #     # These only show with mypy>=0.730:
            #     child.expect("==== mypy ====")
            #     child.expect("Found 1 error in 1 file (checked 1 source file)")
            child.expect(str(num_tests) + " failed")
            child.expect("#### LOOPONFAILING ####")
            _expect_waiting()

        def _expect_waiting():
            child.expect("#### waiting for changes ####")
            child.expect("Watching")

        def _fix():
            pyfile.write(pass_source)
            _expect_changed()
            _expect_success()

        def _expect_changed():
            child.expect("MODIFIED " + str(pyfile))

        def _expect_success():
            for _ in range(2):
                _expect_session()
                # if num_tests == 2:
                #     # These only show with mypy>=0.730:
                #     child.expect("==== mypy ====")
                #     child.expect("Success: no issues found in 1 source file")
                try:
                    child.expect(str(num_tests) + " passed")
                except pexpect.exceptions.TIMEOUT:
                    if module_name == "__init__" and (
                        Version("6.0") <= PYTEST_VERSION < Version("6.2")
                    ):
                        # MypyItems hit the __init__.py bug too when --looponfail
                        # re-collects them after the failing file is modified.
                        # Unlike MypyFile, MypyItem is not a Collector, so this used
                        # to cause an AttributeError until a workaround was added
                        # (MypyItem.collect was defined to yield itself).
                        # Mypy probably noticed the __init__.py problem during the
                        # development of Pytest 6.0, but the error was addressed
                        # with an isinstance assertion, which broke the workaround.
                        # Here, we hit that assertion:
                        child.expect("AssertionError")
                        child.expect("1 error")
                        pytest.xfail("https://github.com/pytest-dev/pytest/issues/8016")
                    raise
            _expect_waiting()

        def _break():
            pyfile.write(fail_source)
            _expect_changed()
            _expect_failure()

>       _expect_failure()

$(BUILD_DIR)/tests/test_pytest_mypy.py:504:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
$(BUILD_DIR)/tests/test_pytest_mypy.py:447: in _expect_failure
    _expect_session()
$(BUILD_DIR)/tests/test_pytest_mypy.py:444: in _expect_session
    child.expect("==== test session starts ====")
/usr/lib/python3.9/vendor-packages/pexpect/spawnbase.py:343: in expect
    return self.expect_list(compiled_pattern_list,
/usr/lib/python3.9/vendor-packages/pexpect/spawnbase.py:372: in expect_list
    return exp.expect_loop(timeout)
/usr/lib/python3.9/vendor-packages/pexpect/expect.py:179: in expect_loop
    return self.eof(e)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <pexpect.expect.Expecter object at 0x7fffa7f98130>
err = EOF('End Of File (EOF). Empty string style platform.')

    def eof(self, err=None):
        spawn = self.spawn

        spawn.before = spawn._before.getvalue()
        spawn._buffer = spawn.buffer_type()
        spawn._before = spawn.buffer_type()
        spawn.after = EOF
        index = self.searcher.eof_index
        if index >= 0:
            spawn.match = EOF
            spawn.match_index = index
            return index
        else:
            spawn.match = None
            spawn.match_index = None
            msg = str(spawn)
            msg += '\nsearcher: %s' % self.searcher
            if err is not None:
                msg = str(err) + '\n' + msg

            exc = EOF(msg)
            exc.__cause__ = None # in Python 3.x we can use "raise exc from None"
>           raise exc
E           pexpect.exceptions.EOF: End Of File (EOF). Empty string style platform.
E           <pexpect.pty_spawn.spawn object at 0x7fffa7f98a90>
E           command: $(BUILD_DIR)/.tox/py39/bin/python
E           args: ['$(BUILD_DIR)/.tox/py39/bin/python', '-mpytest', '--basetemp=/tmp/pytest-of-marcel/pytest-16/popen-gw1/test_looponfail0/temp-pexpect', '--mypy', '--looponfail', '/tmp/pytest-of-marcel/pytest-16/popen-gw1/test_looponfail0/looponfailroot/__init__.py']
E           buffer (last 100 chars): b''
E           before (last 100 chars): b"ing = sys.stdout.encoding\r\nAttributeError: 'ChannelFileWrite' object has no attribute 'encoding'\r\n\r\n"
E           after: <class 'pexpect.exceptions.EOF'>
E           match: None
E           match_index: None
E           exitstatus: None
E           flag_eof: True
E           pid: 2019
E           child_fd: 16
E           closed: False
E           timeout: 60.0
E           delimiter: <class 'pexpect.exceptions.EOF'>
E           logfile: <_io.BufferedWriter name='/tmp/pytest-of-marcel/pytest-16/popen-gw1/test_looponfail0/spawn.out'>
E           logfile_read: None
E           logfile_send: None
E           maxread: 2000
E           ignorecase: False
E           searchwindowsize: None
E           delaybeforesend: 0.05
E           delayafterclose: 0.1
E           delayafterterminate: 0.1
E           searcher: searcher_re:
E               0: re.compile(b'==== test session starts ====')

/usr/lib/python3.9/vendor-packages/pexpect/expect.py:122: EOF

[dmtucker]

I haven't been able to reproduce this. Could you send a pip freeze? (tox should dump one at the top of the run.)

[mtelka]

pip freeze:

aiosignal==1.3.1
alabaster==0.7.12
ansible==2.9.27
appdirs==1.4.4
argcomplete==2.0.0
argh==0.26.2
asgiref==3.6.0
asn1crypto==1.5.1
astor==0.8.1
astroid==2.13.2
async-generator==1.10
async-timeout==4.0.2
atomicwrites==1.4.1
attrs==22.2.0
autocommand==2.2.1
Automat==22.10.0
Babel==2.11.0
backcall==0.2.0
backports-abc==0.5
backports.cached-property==1.0.2
backports.entry-points-selectable==1.2.0
backports.functools-lru-cache==1.6.4
backports.ssl-match-hostname==3.7.0.1
barman==3.3.0
bcrypt==3.2.0
black==22.12.0
bleach==5.0.1
blinker==1.5
borgbackup==1.2.3
build==0.10.0
CacheControl==0.12.11
cached-property==1.5.2
cachetools==5.2.1
calver==2022.6.26
certifi==2020.6.20
cffi==1.14.0
cfgv==3.3.1
characteristic==14.3.0
chardet==5.0.0
charset-normalizer==3.0.1
cheroot==8.5.2
CherryPy==17.3.0
chevron==0.14.0
click==8.1.3
codecov==2.1.12
colorama==0.4.6
colorlog==6.7.0
commonmark==0.9.1
configobj==5.0.6
constantly==15.1.0
contextlib2==0.5.5
covdefaults==2.2.0
coverage==7.0.5
coveralls==3.3.1
crashtest==0.4.1
crmsh==4.0.0
cryptography==2.6.1
curio==1.6
Cython==0.29.33
cython-test-exception-raiser==1.0.2
dbus-python==1.2.18
ddt==1.6.0
decorator==5.1.1
defusedxml==0.7.1
diff-cover==7.3.0
distlib==0.3.6
distro==1.5.0
dnspython==2.2.1
docopt==0.6.2
docutils==0.18.1
editables==0.3
elastic-transport==8.4.0
elasticsearch==8.5.3
elementpath==2.5.0
ephemeral-port-reserve==1.1.4
exceptiongroup==1.1.0
execnet==1.9.0
extras==1.0.0
fastjsonschema==2.16.2
fields==5.0.0
filelock==3.9.0
findpython==0.2.2
fixtures==4.0.1
flake8==5.0.4
flake8-2020==1.7.0
flake8-noqa==1.2.9
flake8-typing-imports==1.14.0
flaky==3.7.0
flamegraph==0.1
flit_core==3.8.0
flit_scm==1.7.0
flufl.flake8==0.10
freezegun==1.1.0
frozendict==2.3.4
frozenlist==1.3.3
func-timeout==4.3.5
future==0.18.2
ghp-import==2.1.0
gi-docgen==2023.1
gprof2dot==2022.7.29
graphviz==0.20.1
hatch-fancy-pypi-readme==22.8.0
hatch-nodejs-version==0.3.1
hatch-vcs==0.3.0
hatchling==1.12.2
http-parser==0.9.0
hupper==1.11
hyperlink==21.0.0
hypothesis==6.60.0
identify==2.5.9
idna==2.10
imagesize==1.4.1
importlib-metadata==5.2.0
importlib-resources==5.10.2
incremental==22.10.0
iniconfig==2.0.0
installer==0.6.0
iocapture==0.1.2
isc==2.0
iso8601==1.1.0
isort==5.9.3
itsdangerous==2.1.2
jaraco.context==4.2.0
jaraco.envs==2.4.0
jaraco.functools==3.5.2
Jinja2==3.0.3
jsonrpclib-pelix==0.4.3
jsonschema==4.17.3
jupyter_core==5.1.1
kgb==7.1.1
lazy==1.5
lazy-object-proxy==1.9.0
lxml==4.6.1
Mako==1.1.6
Markdown==3.4.1
MarkupSafe==2.0.1
matplotlib-inline==0.1.6
mccabe==0.7.0
mercurial==5.9.3
mergedeep==1.3.4
meson==1.0.0
mkdocs==1.4.2
mkdocs-bootstrap==1.1
mkdocs-bootswatch==1.1
mock==5.0.0
more-itertools==8.6.0
msgpack==1.0.2
multidict==6.0.4
mypy==0.991
mypy-extensions==0.4.3
nbformat==5.7.2
netsnmp-python==1.0a1
nodeenv==1.7.0
nose==1.3.7
nox==2022.11.21
objgraph==3.5.0
outcome==1.2.0
packaging==22.0
paramiko==2.12.0
parso==0.8.3
Paste==3.5.2
PasteDeploy==3.0.1
path==16.6.0
path.py==12.5.0
pathlib2==2.3.7.post1
pathspec==0.10.3
pbr==5.11.0
pdm-pep517==1.0.6
pep440==0.1.2
pep517==0.13.0
pexpect==4.8.0
pickleshare==0.7.5
Pillow==9.1.1
pip-run==8.8.0
pipdeptree==2.2.1
pkg==0.1
pkgconfig==1.5.5
pkginfo==1.8.3
pkgutil_resolve_name==1.3.10
plaster==1.1.2
plaster-pastedeploy==1.0.1
platformdirs==2.6.2
pluggy==1.0.0
ply==3.11
poetry-core==1.3.2
portend==3.1.0
powerline-status==2.8.2
pre-commit==2.21.0
pretend==1.0.9
prettytable==0.7.2
process-tests==2.1.2
prompt-toolkit==3.0.36
psutil==5.9.0
psycopg2==2.9.4
ptyprocess==0.7.0
py==1.11.0
py-cpuinfo==9.0.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pybind11==2.9.2
pybonjour==1.1.1
pycairo==1.20.1
pycodestyle==2.10.0
PyContracts==2.0.1
pycparser==2.21
pycurl==7.44.1
pydantic==1.10.2
pydiffx==1.1
pydocstyle==6.1.1
pyfakefs==5.0.0
pyflakes==2.5.0
Pygments==2.13.0
PyGObject==3.42.0
PyHamcrest==2.0.4
pylint==2.12.2
PyNaCl==1.4.0
pyOpenSSL==19.0.0
pyparsing==3.0.9
pyproject-installer==0.4.0
pyproject_api==1.4.0
pyproject_fmt==0.4.1
pyproject_hooks==1.0.0
PyQt-builder==1.12.2
PyQt5==5.15.6
PyQt5-sip==12.10.1
Pyro4==4.80
pyrsistent==0.18.0
PySocks==1.7.1
pytest==7.2.1
pytest-asyncio==0.20.3
pytest-benchmark==4.0.0
pytest-black==0.3.12
pytest-checkdocs==2.9.0
pytest-console-scripts==1.3.1
pytest-cov==4.0.0
pytest-datadir==1.4.1
pytest-enabler==2.0.0
pytest-env==0.8.1
pytest-expect==1.1.0
pytest-fixture-config==1.7.0
pytest-flake8==1.1.1
pytest-forked==1.4.0
pytest-freezegun==0.4.2
pytest-mock==3.7.0
pytest-perf==0.12.0
pytest-randomly==3.11.0
pytest-reporter==0.5.2
pytest-subtests==0.9.0
pytest-timeout==2.0.2
pytest-travis-fold==1.3.0
pytest-xdist==3.1.0
python-dateutil==2.8.1
python-magic==0.4.27
python-rapidjson==1.4
pytz==2021.3
pyxdg==0.28
PyYAML==6.0
pyyaml_env_tag==0.1
pyzmq==19.0.0
qualname==0.1.0
re-assert==1.1.0
redis==3.5.3
regex==2022.10.31
repoze.lru==0.7
requests==2.28.2
resolvelib==0.9.0
rich==12.6.0
rrdtool==0.1.10
ruamel.yaml==0.17.21
ruamel.yaml.clib==0.2.7
schema==0.7.5
SCons==4.0.1
semantic-version==2.10.0
serpent==1.30.2
service-identity==21.1.0
setuptools-scm==7.1.0
setuptools-scm-git-archive==1.4
sh==1.14.3
shellingham==1.5.0
simplejson==3.8.2
singledispatch==3.4.0.3
sip==6.6.1
six==1.16.0
smartypants==2.0.1
smmap==5.0.0
sniffio==1.3.0
snowballstemmer==2.2.0
sortedcontainers==2.4.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
sqlparse==0.4.3
stevedore==4.1.1
strictyaml==1.6.2
tabulate==0.9.0
teamcity-messages==1.32
tempora==5.2.0
termcolor==2.1.1
terminator==2.1.1
termstyle==0.1.11
testpath==0.6.0
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
texttable==1.6.4
time-machine==2.9.0
tokenize-rt==5.0.0
toml==0.10.2
tomli==2.0.1
tomli_w==1.0.0
tomlkit==0.11.6
tornado==6.2
tox==4.3.5
tox-current-env==0.0.11
tqdm==4.64.0
traitlets==5.5.0
translationstring==1.4
trove-classifiers==2022.12.1
trustme==0.9.0
Twisted==22.10.0
typed-ast==1.5.4
types-docutils==0.19.1.1
types-freezegun==1.1.10
types-mock==4.0.15.2
types-psutil==5.9.5.5
types-pytz==2022.6.0.1
types-setuptools==65.7.0.3
types-toml==0.10.8.1
types-typed-ast==1.5.8.3
typing_extensions==4.4.0
typogrify==2.0.7
tzdata==2022.7
u-msgpack-python==2.7.2
urllib3==1.25.1
validate-pyproject==0.10.1
validators==0.20.0
venusian==3.0.0
virtualenv==20.17.1
voluptuous==0.13.1
waitress==2.1.2
watchdog==2.2.1
wcag-contrast-ratio==0.9
wcwidth==0.2.6
webencodings==0.5.1
wrapt==1.13.2
xmlschema==1.9.2
zc.lockfile==2.0
zipp==3.11.0
zope.event==4.5.0
zope.exceptions==4.6
zope.interface==5.5.2
zope.testrunner==5.5.1
zstd==1.5.2.6

tox was started using this:

/usr/bin/tox-3.9 --current-env --no-provision --recreate -e py39

FYI, I'm trying to package pytest-mypy for OpenIndiana.

[dmtucker]

I still cannot reproduce this, but I'm guessing it's related to pytest-dev/execnet#161

[dmtucker]

Here's a repro with the following requirements.txt:

attrs==22.2.0
coverage==7.0.5
exceptiongroup==1.1.0
execnet==1.9.0
filelock==3.9.0
importlib-metadata==5.2.0
iniconfig==2.0.0
mypy==0.991
mypy-extensions==0.4.3
packaging==22.0
pexpect==4.8.0
pluggy==1.0.0
ptyprocess==0.7.0
py==1.11.0
pyparsing==3.0.9
pytest==7.2.1
pytest-cov==4.0.0
pytest-forked==1.4.0
pytest-randomly==3.11.0
pytest-xdist==3.1.0
six==1.16.0
teamcity-messages==1.32
toml==0.10.2
tomli==2.0.1
typing_extensions==4.4.0
zipp==3.11.0

tox devenv -e py39 venv-issue151
venv-issue151/bin/python -m pip install -r requirements.txt
venv-issue151/bin/pytest -p no:mypy --cov pytest_mypy --cov-fail-under 100 --cov-report term-missing -n auto

This is definitely pytest-dev/execnet#161 (no repro without teamcity-messages).