bpo-40513: Per-interpreter GIL (GH-19943)

In the experimental isolated subinterpreters build mode, the GIL is now per-interpreter. Move gil from _PyRuntimeState.ceval to PyInterpreterState.ceval. new_interpreter() always get the config from the main interpreter.

Author: vstinner

Include/internal/pycore_ceval.h

@@ -50,7 +50,11 @@ extern PyObject *_PyEval_EvalCode(
  PyObject *kwdefs, PyObject *closure,
  PyObject *name, PyObject *qualname);
 
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+extern int _PyEval_ThreadsInitialized(PyInterpreterState *interp);
+#else
 extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime);
+#endif
 extern PyStatus _PyEval_InitGIL(PyThreadState *tstate);
 extern void _PyEval_FiniGIL(PyThreadState *tstate);
 

Include/internal/pycore_interp.h

@@ -46,6 +46,9 @@ struct _ceval_state {
  /* Request for dropping the GIL */
  _Py_atomic_int gil_drop_request;
  struct _pending_calls pending;
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ struct _gil_runtime_state gil;
+#endif
 };
 
 

Include/internal/pycore_runtime.h

@@ -19,7 +19,9 @@ struct _ceval_runtime_state {
  the main thread of the main interpreter can handle signals: see
  _Py_ThreadCanHandleSignals(). */
  _Py_atomic_int signals_pending;
+#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
  struct _gil_runtime_state gil;
+#endif
 };
 
 /* GIL state */

Python/ceval.c

@@ -250,6 +250,21 @@ ensure_tstate_not_null(const char *func, PyThreadState *tstate)
 }
 
 
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+int
+_PyEval_ThreadsInitialized(PyInterpreterState *interp)
+{
+ return gil_created(&interp->ceval.gil);
+}
+
+int
+PyEval_ThreadsInitialized(void)
+{
+ // Fatal error if there is no current interpreter
+ PyInterpreterState *interp = PyInterpreterState_Get();
+ return _PyEval_ThreadsInitialized(interp);
+}
+#else
 int
 _PyEval_ThreadsInitialized(_PyRuntimeState *runtime)
 {
@@ -262,18 +277,25 @@ PyEval_ThreadsInitialized(void)
  _PyRuntimeState *runtime = &_PyRuntime;
  return _PyEval_ThreadsInitialized(runtime);
 }
+#endif
 
 PyStatus
 _PyEval_InitGIL(PyThreadState *tstate)
 {
+#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
  if (!_Py_IsMainInterpreter(tstate)) {
  /* Currently, the GIL is shared by all interpreters,
  and only the main interpreter is responsible to create
  and destroy it. */
  return _PyStatus_OK();
  }
+#endif
 
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ struct _gil_runtime_state *gil = &tstate->interp->ceval.gil;
+#else
  struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
+#endif
  assert(!gil_created(gil));
 
  PyThread_init_thread();
@@ -288,14 +310,20 @@ _PyEval_InitGIL(PyThreadState *tstate)
 void
 _PyEval_FiniGIL(PyThreadState *tstate)
 {
+#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
  if (!_Py_IsMainInterpreter(tstate)) {
  /* Currently, the GIL is shared by all interpreters,
  and only the main interpreter is responsible to create
  and destroy it. */
  return;
  }
+#endif
 
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ struct _gil_runtime_state *gil = &tstate->interp->ceval.gil;
+#else
  struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
+#endif
  if (!gil_created(gil)) {
  /* First Py_InitializeFromConfig() call: the GIL doesn't exist
  yet: do nothing. */
@@ -413,13 +441,18 @@ PyEval_ReleaseThread(PyThreadState *tstate)
 void
 _PyEval_ReInitThreads(_PyRuntimeState *runtime)
 {
+ PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+ ensure_tstate_not_null(__func__, tstate);
+
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ struct _gil_runtime_state *gil = &tstate->interp->ceval.gil;
+#else
  struct _gil_runtime_state *gil = &runtime->ceval.gil;
+#endif
  if (!gil_created(gil)) {
  return;
  }
  recreate_gil(gil);
- PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
- ensure_tstate_not_null(__func__, tstate);
 
  take_gil(tstate);
 
@@ -457,7 +490,11 @@ PyEval_SaveThread(void)
 
  struct _ceval_runtime_state *ceval = &runtime->ceval;
  struct _ceval_state *ceval2 = &tstate->interp->ceval;
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ assert(gil_created(&ceval2->gil));
+#else
  assert(gil_created(&ceval->gil));
+#endif
  drop_gil(ceval, ceval2, tstate);
  return tstate;
 }
@@ -716,7 +753,9 @@ void
 _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval)
 {
  _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;
+#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
  _gil_initialize(&ceval->gil);
+#endif
 }
 
 int
@@ -731,6 +770,11 @@ _PyEval_InitState(struct _ceval_state *ceval)
  if (pending->lock == NULL) {
  return -1;
  }
+
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ _gil_initialize(&ceval->gil);
+#endif
+
  return 0;
 }
 

Python/ceval_gil.h

@@ -144,7 +144,11 @@ static void
 drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2,
  PyThreadState *tstate)
 {
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ struct _gil_runtime_state *gil = &ceval2->gil;
+#else
  struct _gil_runtime_state *gil = &ceval->gil;
+#endif
  if (!_Py_atomic_load_relaxed(&gil->locked)) {
  Py_FatalError("drop_gil: GIL is not locked");
  }
@@ -228,7 +232,11 @@ take_gil(PyThreadState *tstate)
  PyInterpreterState *interp = tstate->interp;
  struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
  struct _ceval_state *ceval2 = &interp->ceval;
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ struct _gil_runtime_state *gil = &ceval2->gil;
+#else
  struct _gil_runtime_state *gil = &ceval->gil;
+#endif
 
  /* Check that _PyEval_InitThreads() was called to create the lock */
  assert(gil_created(gil));
@@ -320,10 +328,22 @@ take_gil(PyThreadState *tstate)
 
 void _PyEval_SetSwitchInterval(unsigned long microseconds)
 {
- _PyRuntime.ceval.gil.interval = microseconds;
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ PyInterpreterState *interp = PyInterpreterState_Get();
+ struct _gil_runtime_state *gil = &interp->ceval.gil;
+#else
+ struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
+#endif
+ gil->interval = microseconds;
 }
 
 unsigned long _PyEval_GetSwitchInterval()
 {
- return _PyRuntime.ceval.gil.interval;
+#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
+ PyInterpreterState *interp = PyInterpreterState_Get();
+ struct _gil_runtime_state *gil = &interp->ceval.gil;
+#else
+ struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
+#endif
+ return gil->interval;
 }

Python/pylifecycle.c

@@ -1561,9 +1561,13 @@ new_interpreter(PyThreadState **tstate_p, int isolated_subinterpreter)
 
  /* Copy the current interpreter config into the new interpreter */
  const PyConfig *config;
+#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
  if (save_tstate != NULL) {
  config = _PyInterpreterState_GetConfig(save_tstate->interp);
- } else {
+ }
+ else
+#endif
+ {
  /* No current thread state, copy from the main interpreter */
  PyInterpreterState *main_interp = PyInterpreterState_Main();
  config = _PyInterpreterState_GetConfig(main_interp);