Use execnet main_thread_only execmodel (pytest-dev#1027)

Use the execnet main_thread_only execmodel so that code which expects to run in the main thread will just work. This execmodel has been merged to the execnet master branch via pytest-dev/execnet#243, so this patch should not be merged until there is a released version of execnet supporting the main_thread_only execmodel. Closes pytest-dev#620

Author: zmedico

changelog/1027.improvement.rst

@@ -0,0 +1,3 @@
+``pytest-xdist`` workers now always execute the tests in the main thread.
+
+Previously some tests might end up executing in a separate thread other than ``main`` in the workers, due to some internal `èxecnet`` details. This can cause problems specially with async frameworks where the event loop is running in the ``main`` thread (for example `#620 <https://github.com/pytest-dev/pytest-xdist/issues/620>`__).

changelog/620.bugfix

@@ -0,0 +1 @@
+Use the ``execnet`` new ``main_thread_only`` "execmodel" so that code which expects to only run in the main thread will now work as expected.

src/xdist/looponfail.py

@@ -82,7 +82,7 @@ def trace(self, *args: object) -> None:
  print("RemoteControl:", msg)
 
  def initgateway(self) -> execnet.Gateway:
- return execnet.makegateway("popen")
+ return execnet.makegateway("execmodel=main_thread_only//popen")
 
  def setup(self) -> None:
  if hasattr(self, "gateway"):

src/xdist/workermanage.py

@@ -56,13 +56,15 @@ def __init__(
  self.testrunuid = self.config.getoption("testrunuid")
  if self.testrunuid is None:
  self.testrunuid = uuid.uuid4().hex
- self.group = execnet.Group()
+ self.group = execnet.Group(execmodel="main_thread_only")
  if specs is None:
  specs = self._getxspecs()
  self.specs: list[execnet.XSpec] = []
  for spec in specs:
  if not isinstance(spec, execnet.XSpec):
  spec = execnet.XSpec(spec)
+ if getattr(spec, "execmodel", None) != "main_thread_only":
+ spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
  if not spec.chdir and not spec.popen:
  spec.chdir = defaultchdir
  self.group.allocate_id(spec)
@@ -90,6 +92,8 @@ def setup_node(
  spec: execnet.XSpec,
  putevent: Callable[[tuple[str, dict[str, Any]]], None],
  ) -> WorkerController:
+ if getattr(spec, "execmodel", None) != "main_thread_only":
+ spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
  gw = self.group.makegateway(spec)
  self.config.hook.pytest_xdist_newgateway(gateway=gw)
  self.rsync_roots(gw)

testing/test_remote.py

@@ -49,7 +49,7 @@ def __init__(
  def setup(self) -> None:
  self.pytester.chdir()
  # import os ; os.environ['EXECNET_DEBUG'] = "2"
- self.gateway = execnet.makegateway()
+ self.gateway = execnet.makegateway("execmodel=main_thread_only//popen")
  self.config = config = self.pytester.parseconfigure()
  putevent = self.events.put if self.use_callback else None
 

testing/test_workermanage.py

@@ -83,7 +83,7 @@ def test_popen_makegateway_events(
  assert len(call.specs) == 2
 
  call = hookrecorder.popcall("pytest_xdist_newgateway")
- assert call.gateway.spec == execnet.XSpec("popen")
+ assert call.gateway.spec == execnet.XSpec("execmodel=main_thread_only//popen")
  assert call.gateway.id == "gw0"
  call = hookrecorder.popcall("pytest_xdist_newgateway")
  assert call.gateway.id == "gw1"
@@ -177,7 +177,7 @@ def test_hrsync_filter(self, source: Path, dest: Path) -> None:
  assert names == {"dir", "file.txt", "somedir"}
 
  def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
- gw = execnet.makegateway("popen//chdir=%s" % dest)
+ gw = execnet.makegateway("execmodel=main_thread_only//popen//chdir=%s" % dest)
  finished = []
  rsync = HostRSync(source)
  rsync.add_target_host(gw, finished=lambda: finished.append(1))