bpo-32604: [_xxsubinterpreters] Propagate exceptions. (GH-19768)

(Note: PEP 554 is not accepted and the implementation in the code base is a private one for use in the test suite.) If code running in a subinterpreter raises an uncaught exception then the "run" call in the calling interpreter fails. A RunFailedError is raised there that summarizes the original exception as a string. The actual exception type, __cause__, __context__, state, etc. are all discarded. This turned out to be functionally insufficient in practice. There is a more helpful solution (and PEP 554 has been updated appropriately). This change adds the exception propagation behavior described in PEP 554 to the _xxsubinterpreters module. With this change a copy of the original exception is set to __cause__ on the RunFailedError. For now we are using "pickle", which preserves the exception's state. We also preserve the original __cause__, __context__, and __traceback__ (since "pickle" does not preserve those). https://bugs.python.org/issue32604

Author: ericsnowcurrently

Lib/test/test__xxsubinterpreters.py

@@ -1,3 +1,4 @@
+import builtins
 from collections import namedtuple
 import contextlib
 import itertools
@@ -866,10 +867,11 @@ def assert_run_failed(self, exctype, msg=None):
  yield
  if msg is None:
  self.assertEqual(str(caught.exception).split(':')[0],
- str(exctype))
+ exctype.__name__)
  else:
  self.assertEqual(str(caught.exception),
- "{}: {}".format(exctype, msg))
+ "{}: {}".format(exctype.__name__, msg))
+ self.assertIsInstance(caught.exception.__cause__, exctype)
 
  def test_invalid_syntax(self):
  with self.assert_run_failed(SyntaxError):
@@ -1060,6 +1062,301 @@ def f():
  self.assertEqual(retcode, 0)
 
 
+def build_exception(exctype, /, *args, **kwargs):
+ # XXX Use __qualname__?
+ name = exctype.__name__
+ argreprs = [repr(a) for a in args]
+ if kwargs:
+ kwargreprs = [f'{k}={v!r}' for k, v in kwargs.items()]
+ script = f'{name}({", ".join(argreprs)}, {", ".join(kwargreprs)})'
+ else:
+ script = f'{name}({", ".join(argreprs)})'
+ expected = exctype(*args, **kwargs)
+ return script, expected
+
+
+def build_exceptions(self, *exctypes, default=None, custom=None, bases=True):
+ if not exctypes:
+ raise NotImplementedError
+ if not default:
+ default = ((), {})
+ elif isinstance(default, str):
+ default = ((default,), {})
+ elif type(default) is not tuple:
+ raise NotImplementedError
+ elif len(default) != 2:
+ default = (default, {})
+ elif type(default[0]) is not tuple:
+ default = (default, {})
+ elif type(default[1]) is not dict:
+ default = (default, {})
+ # else leave it alone
+
+ for exctype in exctypes:
+ customtype = None
+ values = default
+ if custom:
+ if exctype in custom:
+ customtype = exctype
+ elif bases:
+ for customtype in custom:
+ if issubclass(exctype, customtype):
+ break
+ else:
+ customtype = None
+ if customtype is not None:
+ values = custom[customtype]
+ if values is None:
+ continue
+ args, kwargs = values
+ script, expected = build_exception(exctype, *args, **kwargs)
+ yield exctype, customtype, script, expected
+
+
+try:
+ raise Exception
+except Exception as exc:
+ assert exc.__traceback__ is not None
+ Traceback = type(exc.__traceback__)
+
+
+class RunFailedTests(TestBase):
+
+ BUILTINS = [v
+ for v in vars(builtins).values()
+ if (type(v) is type
+ and issubclass(v, Exception)
+ #and issubclass(v, BaseException)
+ )
+ ]
+ BUILTINS_SPECIAL = [
+ # These all have extra attributes (i.e. args/kwargs)
+ SyntaxError,
+ ImportError,
+ UnicodeError,
+ OSError,
+ SystemExit,
+ StopIteration,
+ ]
+
+ @classmethod
+ def build_exceptions(cls, exctypes=None, default=(), custom=None):
+ if exctypes is None:
+ exctypes = cls.BUILTINS
+ if custom is None:
+ # Skip the "special" ones.
+ custom = {et: None for et in cls.BUILTINS_SPECIAL}
+ yield from build_exceptions(*exctypes, default=default, custom=custom)
+
+ def assertExceptionsEqual(self, exc, expected, *, chained=True):
+ if type(expected) is type:
+ self.assertIs(type(exc), expected)
+ return
+ elif not isinstance(exc, Exception):
+ self.assertEqual(exc, expected)
+ elif not isinstance(expected, Exception):
+ self.assertEqual(exc, expected)
+ else:
+ # Plain equality doesn't work, so we have to compare manually.
+ self.assertIs(type(exc), type(expected))
+ self.assertEqual(exc.args, expected.args)
+ self.assertEqual(exc.__reduce__(), expected.__reduce__())
+ if chained:
+ self.assertExceptionsEqual(exc.__context__,
+ expected.__context__)
+ self.assertExceptionsEqual(exc.__cause__,
+ expected.__cause__)
+ self.assertEqual(exc.__suppress_context__,
+ expected.__suppress_context__)
+
+ def assertTracebacksEqual(self, tb, expected):
+ if not isinstance(tb, Traceback):
+ self.assertEqual(tb, expected)
+ elif not isinstance(expected, Traceback):
+ self.assertEqual(tb, expected)
+ else:
+ self.assertEqual(tb.tb_frame.f_code.co_name,
+ expected.tb_frame.f_code.co_name)
+ self.assertEqual(tb.tb_frame.f_code.co_filename,
+ expected.tb_frame.f_code.co_filename)
+ self.assertEqual(tb.tb_lineno, expected.tb_lineno)
+ self.assertTracebacksEqual(tb.tb_next, expected.tb_next)
+
+ # XXX Move this to TestBase?
+ @contextlib.contextmanager
+ def expected_run_failure(self, expected):
+ exctype = expected if type(expected) is type else type(expected)
+
+ with self.assertRaises(interpreters.RunFailedError) as caught:
+ yield caught
+ exc = caught.exception
+
+ modname = exctype.__module__
+ if modname == 'builtins' or modname == '__main__':
+ exctypename = exctype.__name__
+ else:
+ exctypename = f'{modname}.{exctype.__name__}'
+ if exctype is expected:
+ self.assertEqual(str(exc).split(':')[0], exctypename)
+ else:
+ self.assertEqual(str(exc), f'{exctypename}: {expected}')
+ self.assertExceptionsEqual(exc.__cause__, expected)
+ if exc.__cause__ is not None:
+ self.assertIsNotNone(exc.__cause__.__traceback__)
+
+ def test_builtin_exceptions(self):
+ interpid = interpreters.create()
+ msg = '<a message>'
+ for i, info in enumerate(self.build_exceptions(
+ default=msg,
+ custom={
+ SyntaxError: ((msg, '<stdin>', 1, 3, 'a +?'), {}),
+ ImportError: ((msg,), {'name': 'spam', 'path': '/x/spam.py'}),
+ UnicodeError: None,
+ #UnicodeError: ((), {}),
+ #OSError: ((), {}),
+ SystemExit: ((1,), {}),
+ StopIteration: (('<a value>',), {}),
+ },
+ )):
+ exctype, _, script, expected = info
+ testname = f'{i+1} - {script}'
+ script = f'raise {script}'
+
+ with self.subTest(testname):
+ with self.expected_run_failure(expected):
+ interpreters.run_string(interpid, script)
+
+ def test_custom_exception_from___main__(self):
+ script = dedent("""
+ class SpamError(Exception):
+ def __init__(self, q):
+ super().__init__(f'got {q}')
+ self.q = q
+ raise SpamError('eggs')
+ """)
+ expected = Exception(f'SpamError: got {"eggs"}')
+
+ interpid = interpreters.create()
+ with self.assertRaises(interpreters.RunFailedError) as caught:
+ interpreters.run_string(interpid, script)
+ cause = caught.exception.__cause__
+
+ self.assertExceptionsEqual(cause, expected)
+
+ class SpamError(Exception):
+ # The normal Exception.__reduce__() produces a funny result
+ # here. So we have to use a custom __new__().
+ def __new__(cls, q):
+ if type(q) is SpamError:
+ return q
+ return super().__new__(cls, q)
+ def __init__(self, q):
+ super().__init__(f'got {q}')
+ self.q = q
+
+ def test_custom_exception(self):
+ script = dedent("""
+ import test.test__xxsubinterpreters
+ SpamError = test.test__xxsubinterpreters.RunFailedTests.SpamError
+ raise SpamError('eggs')
+ """)
+ try:
+ ns = {}
+ exec(script, ns, ns)
+ except Exception as exc:
+ expected = exc
+
+ interpid = interpreters.create()
+ with self.expected_run_failure(expected):
+ interpreters.run_string(interpid, script)
+
+ class SpamReducedError(Exception):
+ def __init__(self, q):
+ super().__init__(f'got {q}')
+ self.q = q
+ def __reduce__(self):
+ return (type(self), (self.q,), {})
+
+ def test_custom___reduce__(self):
+ script = dedent("""
+ import test.test__xxsubinterpreters
+ SpamError = test.test__xxsubinterpreters.RunFailedTests.SpamReducedError
+ raise SpamError('eggs')
+ """)
+ try:
+ exec(script, (ns := {'__name__': '__main__'}), ns)
+ except Exception as exc:
+ expected = exc
+
+ interpid = interpreters.create()
+ with self.expected_run_failure(expected):
+ interpreters.run_string(interpid, script)
+
+ def test_traceback_propagated(self):
+ script = dedent("""
+ def do_spam():
+ raise Exception('uh-oh')
+ def do_eggs():
+ return do_spam()
+ class Spam:
+ def do(self):
+ return do_eggs()
+ def get_handler():
+ def handler():
+ return Spam().do()
+ return handler
+ go = (lambda: get_handler()())
+ def iter_all():
+ yield from (go() for _ in [True])
+ yield None
+ def main():
+ for v in iter_all():
+ pass
+ main()
+ """)
+ try:
+ ns = {}
+ exec(script, ns, ns)
+ except Exception as exc:
+ expected = exc
+ expectedtb = exc.__traceback__.tb_next
+
+ interpid = interpreters.create()
+ with self.expected_run_failure(expected) as caught:
+ interpreters.run_string(interpid, script)
+ exc = caught.exception
+
+ self.assertTracebacksEqual(exc.__cause__.__traceback__,
+ expectedtb)
+
+ def test_chained_exceptions(self):
+ script = dedent("""
+ try:
+ raise ValueError('msg 1')
+ except Exception as exc1:
+ try:
+ raise TypeError('msg 2')
+ except Exception as exc2:
+ try:
+ raise IndexError('msg 3') from exc2
+ except Exception:
+ raise AttributeError('msg 4')
+ """)
+ try:
+ exec(script, {}, {})
+ except Exception as exc:
+ expected = exc
+
+ interpid = interpreters.create()
+ with self.expected_run_failure(expected) as caught:
+ interpreters.run_string(interpid, script)
+ exc = caught.exception
+
+ # ...just to be sure.
+ self.assertIs(type(exc.__cause__), AttributeError)
+
+
 ##################################
 # channel tests