GH-107263: Increase C stack limit for most functions, except _PyEval_EvalFrameDefault()

Doc/whatsnew/3.12.rst

@@ -801,6 +801,11 @@ sys
  exception instance, rather than to a ``(typ, exc, tb)`` tuple.
  (Contributed by Irit Katriel in :gh:`103176`.)
 
+* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`.
+ The recursion limit now applies only to Python code. Builtin functions do
+ not use the recursion limit, but are protected by a different mechanism
+ that prevents recursion from causing a virtual machine crash.
+
 tempfile
 --------
 

Include/cpython/pystate.h

@@ -222,7 +222,7 @@ struct _ts {
 # ifdef __wasi__
 # define C_RECURSION_LIMIT 500
 # else
-# define C_RECURSION_LIMIT 800
+# define C_RECURSION_LIMIT 2400
 # endif
 #endif
 

Lib/test/list_tests.py

@@ -61,7 +61,7 @@ def test_repr(self):
 
  def test_repr_deep(self):
  a = self.type2test([])
- for i in range(sys.getrecursionlimit() + 100):
+ for i in range(3_000):
  a = self.type2test([a])
  self.assertRaises(RecursionError, repr, a)
 

Lib/test/mapping_tests.py

@@ -624,7 +624,7 @@ def __repr__(self):
 
  def test_repr_deep(self):
  d = self._empty_mapping()
- for i in range(sys.getrecursionlimit() + 100):
+ for i in range(3_000):
  d0 = d
  d = self._empty_mapping()
  d[1] = d0

Lib/test/support/__init__.py

@@ -64,7 +64,7 @@
  "run_with_tz", "PGO", "missing_compiler_executable",
  "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST",
  "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT",
- "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT",
+ "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT",
  ]
 
 
@@ -2460,3 +2460,5 @@ def adjust_int_max_str_digits(max_digits):
 
 #For recursion tests, easily exceeds default recursion limit
 EXCEEDS_RECURSION_LIMIT = 5000
+
+C_RECURSION_LIMIT = 2400

Lib/test/test_compile.py

@@ -11,10 +11,9 @@
 import warnings
 from test import support
 from test.support import (script_helper, requires_debug_ranges,
- requires_specialization)
+ requires_specialization, C_RECURSION_LIMIT)
 from test.support.os_helper import FakePath
 
-
 class TestSpecifics(unittest.TestCase):
 
  def compile_single(self, source):
@@ -112,7 +111,7 @@ def __getitem__(self, key):
 
  @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
  def test_extended_arg(self):
- repeat = 2000
+ repeat = int(C_RECURSION_LIMIT * 0.9)
  longexpr = 'x = x or ' + '-x' * repeat
  g = {}
  code = textwrap.dedent('''
@@ -558,16 +557,12 @@ def test_yet_more_evil_still_undecodable(self):
  @support.cpython_only
  @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
  def test_compiler_recursion_limit(self):
- # Expected limit is sys.getrecursionlimit() * the scaling factor
- # in symtable.c (currently 3)
- # We expect to fail *at* that limit, because we use up some of
- # the stack depth limit in the test suite code
- # So we check the expected limit and 75% of that
- # XXX (ncoghlan): duplicating the scaling factor here is a little
- # ugly. Perhaps it should be exposed somewhere...
- fail_depth = sys.getrecursionlimit() * 3
- crash_depth = sys.getrecursionlimit() * 300
- success_depth = int(fail_depth * 0.75)
+ # Expected limit is C_RECURSION_LIMIT
+ # Duplicating the limit here is a little ugly.
+ # Perhaps it should be exposed somewhere...
+ fail_depth = C_RECURSION_LIMIT + 1
+ crash_depth = C_RECURSION_LIMIT * 100
+ success_depth = int(C_RECURSION_LIMIT * 0.9)
 
  def check_limit(prefix, repeated, mode="single"):
  expect_ok = prefix + repeated * success_depth

Lib/test/test_dict.py

@@ -596,7 +596,7 @@ def __repr__(self):
 
  def test_repr_deep(self):
  d = {}
- for i in range(sys.getrecursionlimit() + 100):
+ for i in range(3_000):
  d = {1: d}
  self.assertRaises(RecursionError, repr, d)
 

Lib/test/test_dictviews.py

@@ -3,6 +3,7 @@
 import pickle
 import sys
 import unittest
+from test.support import C_RECURSION_LIMIT
 
 class DictSetTest(unittest.TestCase):
 
@@ -279,7 +280,7 @@ def test_recursive_repr(self):
 
  def test_deeply_nested_repr(self):
  d = {}
- for i in range(sys.getrecursionlimit() + 100):
+ for i in range(C_RECURSION_LIMIT//2 + 100):
  d = {42: d.values()}
  self.assertRaises(RecursionError, repr, d)
 

Lib/test/test_exception_group.py

@@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self):
 class DeepRecursionInSplitAndSubgroup(unittest.TestCase):
  def make_deep_eg(self):
  e = TypeError(1)
- for i in range(2000):
+ for i in range(3_000):
  e = ExceptionGroup('eg', [e])
  return e
 

Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst

@@ -0,0 +1,3 @@
+Increase C recursion limit for functions other than the main interpreter
+from 800 to 2500. This should allow functions like ``list.__repr__`` and
+``json.dumps`` to handle all the inputs that they could prior to 3.12

Parser/asdl_c.py

@@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor):
 
  int starting_recursion_depth;
  /* Be careful here to prevent overflow. */
- int COMPILER_STACK_FRAME_SCALE = 3;
+ int COMPILER_STACK_FRAME_SCALE = 1;
  PyThreadState *tstate = _PyThreadState_GET();
  if (!tstate) {
  return 0;

Python/ast.c

@@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps)
 
 
 /* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 3
+#define COMPILER_STACK_FRAME_SCALE 1
 
 int
 _PyAST_Validate(mod_ty mod)

Python/ast_opt.c

@@ -1112,7 +1112,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat
 #undef CALL_SEQ
 
 /* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 3
+#define COMPILER_STACK_FRAME_SCALE 1
 
 int
 _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features)

Python/bytecodes.c

@@ -741,7 +741,7 @@ dummy_func(
  tstate->cframe = cframe.previous;
  assert(tstate->cframe->current_frame == frame->previous);
  assert(!_PyErr_Occurred(tstate));
- _Py_LeaveRecursiveCallTstate(tstate);
+ tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
  return retval;
  }
 

Python/ceval.c

@@ -624,6 +624,11 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup;
 # pragma warning(disable:4102)
 #endif
 
+
+/* _PyEval_EvalFrameDefault() is a *big* function,
+ * so consume 3 units of C stack */
+#define PY_EVAL_C_STACK_UNITS 3
+
 PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
 {
@@ -676,6 +681,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
  frame->previous = &entry_frame;
  cframe.current_frame = frame;
 
+ tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1);
  if (_Py_EnterRecursiveCallTstate(tstate, "")) {
  tstate->c_recursion_remaining--;
  tstate->py_recursion_remaining--;
@@ -907,7 +913,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
  /* Restore previous cframe and exit */
  tstate->cframe = cframe.previous;
  assert(tstate->cframe->current_frame == frame->previous);
- _Py_LeaveRecursiveCallTstate(tstate);
+ tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
  return NULL;
  }
 

Python/symtable.c

@@ -282,17 +282,10 @@ symtable_new(void)
  return NULL;
 }
 
-/* When compiling the use of C stack is probably going to be a lot
- lighter than when executing Python code but still can overflow
- and causing a Python crash if not checked (e.g. eval("()"*300000)).
- Using the current recursion limit for the compiler seems too
- restrictive (it caused at least one test to fail) so a factor is
- used to allow deeper recursion when compiling an expression.
-
- Using a scaling factor means this should automatically adjust when
+/* Using a scaling factor means this should automatically adjust when
  the recursion limit is adjusted for small or large C stack allocations.
 */
-#define COMPILER_STACK_FRAME_SCALE 3
+#define COMPILER_STACK_FRAME_SCALE 1
 
 struct symtable *
 _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)