$ cat conftest.py # content of conftest.py import pytest def pytest_collect_file(parent, path): if path.ext == ".yaml" and path.basename.startswith("test"): return YamlFile(path, parent) class YamlFile(pytest.File): def collect(self): import yaml # we need a yaml parser, e.g. PyYAML raw = yaml.safe_load(self.fspath.open()) for name, spec in sorted(raw.items()): yield YamlItem(name, self, spec) class YamlItem(pytest.Item): def __init__(self, name, parent, spec): super().__init__(name, parent) self.spec = spec def runtest(self): for name, value in sorted(self.spec.items()): # some custom test execution (dumb example follows) if name != value: raise YamlException(self, name, value) def repr_failure(self, excinfo): """ called when self.runtest() raises an exception. """ if isinstance(excinfo.value, YamlException): return "\n".join( [ "usecase execution failed", " spec failed: {1!r}: {2!r}".format(*excinfo.value.args), " no further details known at this point.", ] ) def reportinfo(self): return self.fspath, 0, "usecase: {}".format(self.name) class YamlException(Exception): """ custom exception for error reporting. """ $ cat test_simple.yaml # test_simple.yaml ok: sub1: sub1 hello: world: world some: other 

$ python --version Python 3.7.4 $ pip list Package Version ------------------ ------- atomicwrites 1.3.0 attrs 19.1.0 importlib-metadata 0.23 more-itertools 7.2.0 packaging 19.2 pip 19.2.3 pluggy 0.13.0 py 1.8.0 pyparsing 2.4.2 pytest 5.1.3 PyYAML 5.1.2 setuptools 41.2.0 six 1.12.0 wcwidth 0.1.7 wheel 0.33.6 zipp 0.6.0 

$ pytest ==================================== test session starts ===================================== platform darwin -- Python 3.7.4, pytest-5.1.3, py-1.8.0, pluggy-0.13.0 rootdir: /private/tmp/pytest_bug collected 2 items test_simple.yaml F. [100%] ==== FAILURES ==== _______________________________________ usecase: hello _______________________________________ usecase execution failed spec failed: 'some': 'other' no further details known at this point. ================================ 1 failed, 1 passed in 0.04s ================================= 

$ pytest --setup-plan ==================================== test session starts ===================================== platform darwin -- Python 3.7.4, pytest-5.1.3, py-1.8.0, pluggy-0.13.0 rootdir: /private/tmp/pytest_bug collected 2 items test_simple.yaml test_simple.yaml::hello INTERNALERROR> Traceback (most recent call last): INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/_pytest/main.py", line 191, in wrap_session INTERNALERROR> session.exitstatus = doit(config, session) or 0 INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/_pytest/main.py", line 235, in _main INTERNALERROR> config.hook.pytest_runtestloop(session=session) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/hooks.py", line 286, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/manager.py", line 92, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/manager.py", line 86, in <lambda> INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/_pytest/main.py", line 256, in pytest_runtestloop INTERNALERROR> item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/hooks.py", line 286, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/manager.py", line 92, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/manager.py", line 86, in <lambda> INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/_pytest/runner.py", line 72, in pytest_runtest_protocol INTERNALERROR> runtestprotocol(item, nextitem=nextitem) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/_pytest/runner.py", line 85, in runtestprotocol INTERNALERROR> show_test_item(item) INTERNALERROR> File "/usr/local/virtualenvs/tmp-68a55808020a45b9/lib/python3.7/site-packages/_pytest/runner.py", line 103, in show_test_item INTERNALERROR> used_fixtures = sorted(item._fixtureinfo.name2fixturedefs.keys()) INTERNALERROR> AttributeError: 'YamlItem' object has no attribute '_fixtureinfo' =================================== no tests ran in 0.05s ====================================