bpo-38787: C API for module state access from extension methods (PEP 573) (GH-19936)

Module C state is now accessible from C-defined heap type methods (PEP 573). Patch by Marcel Plch and Petr Viktorin. Co-authored-by: Marcel Plch <mplch@redhat.com> Co-authored-by: Victor Stinner <vstinner@python.org>

Author: 3 people

Doc/c-api/structures.rst

@@ -147,23 +147,56 @@ Implementing functions and methods
  value of the function as exposed in Python. The function must return a new
  reference.
 
+ The function signature is::
+
+ PyObject *PyCFunction(PyObject *self,
+ PyObject *const *args);
 
 .. c:type:: PyCFunctionWithKeywords
 
  Type of the functions used to implement Python callables in C
  with signature :const:`METH_VARARGS | METH_KEYWORDS`.
+ The function signature is::
+
+ PyObject *PyCFunctionWithKeywords(PyObject *self,
+ PyObject *const *args,
+ PyObject *kwargs);
 
 
 .. c:type:: _PyCFunctionFast
 
  Type of the functions used to implement Python callables in C
  with signature :const:`METH_FASTCALL`.
+ The function signature is::
 
+ PyObject *_PyCFunctionFast(PyObject *self,
+ PyObject *const *args,
+ Py_ssize_t nargs);
 
 .. c:type:: _PyCFunctionFastWithKeywords
 
  Type of the functions used to implement Python callables in C
  with signature :const:`METH_FASTCALL | METH_KEYWORDS`.
+ The function signature is::
+
+ PyObject *_PyCFunctionFastWithKeywords(PyObject *self,
+ PyObject *const *args,
+ Py_ssize_t nargs,
+ PyObject *kwnames);
+
+.. c:type:: PyCMethod
+
+ Type of the functions used to implement Python callables in C
+ with signature :const:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS`.
+ The function signature is::
+
+ PyObject *PyCMethod(PyObject *self,
+ PyTypeObject *defining_class,
+ PyObject *const *args,
+ Py_ssize_t nargs,
+ PyObject *kwnames)
+
+ .. versionadded:: 3.9
 
 
 .. c:type:: PyMethodDef
@@ -197,9 +230,7 @@ The :attr:`ml_flags` field is a bitfield which can include the following flags.
 The individual flags indicate either a calling convention or a binding
 convention.
 
-There are four basic calling conventions for positional arguments
-and two of them can be combined with :const:`METH_KEYWORDS` to support
-also keyword arguments. So there are a total of 6 calling conventions:
+There are these calling conventions:
 
 .. data:: METH_VARARGS
 
@@ -250,6 +281,19 @@ also keyword arguments. So there are a total of 6 calling conventions:
  .. versionadded:: 3.7
 
 
+.. data:: METH_METHOD | METH_FASTCALL | METH_KEYWORDS
+
+ Extension of :const:`METH_FASTCALL | METH_KEYWORDS` supporting the *defining
+ class*, that is, the class that contains the method in question.
+ The defining class might be a superclass of ``Py_TYPE(self)``.
+
+ The method needs to be of type :c:type:`PyCMethod`, the same as for
+ ``METH_FASTCALL | METH_KEYWORDS`` with ``defining_class`` argument added after
+ ``self``.
+
+ .. versionadded:: 3.9
+
+
 .. data:: METH_NOARGS
 
  Methods without parameters don't need to check whether arguments are given if

Doc/c-api/type.rst

@@ -109,14 +109,38 @@ Type Objects
 
  .. versionadded:: 3.4
 
+.. c:function:: PyObject* PyType_GetModule(PyTypeObject *type)
+
+ Return the module object associated with the given type when the type was
+ created using :c:func:`PyType_FromModuleAndSpec`.
+
+ If no module is associated with the given type, sets :py:class:`TypeError`
+ and returns ``NULL``.
+
+ .. versionadded:: 3.9
+
+.. c:function:: void* PyType_GetModuleState(PyTypeObject *type)
+
+ Return the state of the module object associated with the given type.
+ This is a shortcut for calling :c:func:`PyModule_GetState()` on the result
+ of :c:func:`PyType_GetModule`.
+
+ If no module is associated with the given type, sets :py:class:`TypeError`
+ and returns ``NULL``.
+
+ If the *type* has an associated module but its state is ``NULL``,
+ returns ``NULL`` without setting an exception.
+
+ .. versionadded:: 3.9
+
 
 Creating Heap-Allocated Types
 .............................
 
 The following functions and structs are used to create
 :ref:`heap types <heap-types>`.
 
-.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+.. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
 
  Creates and returns a heap type object from the *spec*
  (:const:`Py_TPFLAGS_HEAPTYPE`).
@@ -127,8 +151,18 @@ The following functions and structs are used to create
  If *bases* is ``NULL``, the *Py_tp_base* slot is used instead.
  If that also is ``NULL``, the new type derives from :class:`object`.
 
+ The *module* must be a module object or ``NULL``.
+ If not ``NULL``, the module is associated with the new type and can later be
+ retreived with :c:func:`PyType_GetModule`.
+
  This function calls :c:func:`PyType_Ready` on the new type.
 
+ .. versionadded:: 3.9
+
+.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+
+ Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, bases)``.
+
  .. versionadded:: 3.3
 
 .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)

Include/cpython/methodobject.h

@@ -0,0 +1,32 @@
+#ifndef Py_CPYTHON_METHODOBJECT_H
+# error "this header file must not be included directly"
+#endif
+
+PyAPI_DATA(PyTypeObject) PyCMethod_Type;
+
+/* Macros for direct access to these values. Type checks are *not*
+ done, so use with care. */
+#define PyCFunction_GET_FUNCTION(func) \
+ (((PyCFunctionObject *)func) -> m_ml -> ml_meth)
+#define PyCFunction_GET_SELF(func) \
+ (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \
+ NULL : ((PyCFunctionObject *)func) -> m_self)
+#define PyCFunction_GET_FLAGS(func) \
+ (((PyCFunctionObject *)func) -> m_ml -> ml_flags)
+#define PyCFunction_GET_CLASS(func) \
+ (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_METHOD ? \
+ ((PyCMethodObject *)func) -> mm_class : NULL)
+
+typedef struct {
+ PyObject_HEAD
+ PyMethodDef *m_ml; /* Description of the C function to call */
+ PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */
+ PyObject *m_module; /* The __module__ attribute, can be anything */
+ PyObject *m_weakreflist; /* List of weak references */
+ vectorcallfunc vectorcall;
+} PyCFunctionObject;
+
+typedef struct {
+ PyCFunctionObject func;
+ PyTypeObject *mm_class; /* Class that defines this method */
+} PyCMethodObject;

Include/cpython/object.h

@@ -289,6 +289,7 @@ typedef struct _heaptypeobject {
  PyBufferProcs as_buffer;
  PyObject *ht_name, *ht_slots, *ht_qualname;
  struct _dictkeysobject *ht_cached_keys;
+ PyObject *ht_module;
  /* here are optional user slots, followed by the members. */
 } PyHeapTypeObject;
 

Include/methodobject.h

@@ -13,7 +13,7 @@ extern "C" {
 
 PyAPI_DATA(PyTypeObject) PyCFunction_Type;
 
-#define PyCFunction_Check(op) Py_IS_TYPE(op, &PyCFunction_Type)
+#define PyCFunction_Check(op) (Py_IS_TYPE(op, &PyCFunction_Type) || (PyType_IsSubtype(Py_TYPE(op), &PyCFunction_Type)))
 
 typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
 typedef PyObject *(*_PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t);
@@ -22,21 +22,13 @@ typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
 typedef PyObject *(*_PyCFunctionFastWithKeywords) (PyObject *,
  PyObject *const *, Py_ssize_t,
  PyObject *);
+typedef PyObject *(*PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *,
+ size_t, PyObject *);
+
 PyAPI_FUNC(PyCFunction) PyCFunction_GetFunction(PyObject *);
 PyAPI_FUNC(PyObject *) PyCFunction_GetSelf(PyObject *);
 PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *);
 
-/* Macros for direct access to these values. Type checks are *not*
- done, so use with care. */
-#ifndef Py_LIMITED_API
-#define PyCFunction_GET_FUNCTION(func) \
- (((PyCFunctionObject *)func) -> m_ml -> ml_meth)
-#define PyCFunction_GET_SELF(func) \
- (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \
- NULL : ((PyCFunctionObject *)func) -> m_self)
-#define PyCFunction_GET_FLAGS(func) \
- (((PyCFunctionObject *)func) -> m_ml -> ml_flags)
-#endif
 Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *);
 
 struct PyMethodDef {
@@ -52,6 +44,13 @@ typedef struct PyMethodDef PyMethodDef;
 PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
  PyObject *);
 
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
+#define PyCFunction_NewEx(ML, SELF, MOD) PyCMethod_New((ML), (SELF), (MOD), NULL)
+PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *,
+ PyObject *, PyTypeObject *);
+#endif
+
+
 /* Flag passed to newmethodobject */
 /* #define METH_OLDARGS 0x0000 -- unsupported now */
 #define METH_VARARGS 0x0001
@@ -84,15 +83,24 @@ PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
 #define METH_STACKLESS 0x0000
 #endif
 
+/* METH_METHOD means the function stores an
+ * additional reference to the class that defines it;
+ * both self and class are passed to it.
+ * It uses PyCMethodObject instead of PyCFunctionObject.
+ * May not be combined with METH_NOARGS, METH_O, METH_CLASS or METH_STATIC.
+ */
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
+#define METH_METHOD 0x0200
+#endif
+
+
 #ifndef Py_LIMITED_API
-typedef struct {
- PyObject_HEAD
- PyMethodDef *m_ml; /* Description of the C function to call */
- PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */
- PyObject *m_module; /* The __module__ attribute, can be anything */
- PyObject *m_weakreflist; /* List of weak references */
- vectorcallfunc vectorcall;
-} PyCFunctionObject;
+
+#define Py_CPYTHON_METHODOBJECT_H
+#include "cpython/methodobject.h"
+#undef Py_CPYTHON_METHODOBJECT_H
+
 #endif
 
 #ifdef __cplusplus

Include/object.h

@@ -213,6 +213,11 @@ PyAPI_FUNC(PyObject*) PyType_FromSpecWithBases(PyType_Spec*, PyObject*);
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03040000
 PyAPI_FUNC(void*) PyType_GetSlot(PyTypeObject*, int);
 #endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
+PyAPI_FUNC(PyObject*) PyType_FromModuleAndSpec(PyObject *, PyType_Spec *, PyObject *);
+PyAPI_FUNC(PyObject *) PyType_GetModule(struct _typeobject *);
+PyAPI_FUNC(void *) PyType_GetModuleState(struct _typeobject *);
+#endif
 
 /* Generic type check */
 PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *);

Lib/test/test_capi.py

@@ -13,6 +13,8 @@
 import time
 import unittest
 import weakref
+import importlib.machinery
+import importlib.util
 from test import support
 from test.support import MISSING_C_DOCSTRINGS
 from test.support.script_helper import assert_python_failure, assert_python_ok
@@ -774,5 +776,76 @@ class PyMemDefaultTests(PyMemDebugTests):
  PYTHONMALLOC = ''
 
 
+class Test_ModuleStateAccess(unittest.TestCase):
+ """Test access to module start (PEP 573)"""
+
+ # The C part of the tests lives in _testmultiphase, in a module called
+ # _testmultiphase_meth_state_access.
+ # This module has multi-phase initialization, unlike _testcapi.
+
+ def setUp(self):
+ fullname = '_testmultiphase_meth_state_access' # XXX
+ origin = importlib.util.find_spec('_testmultiphase').origin
+ loader = importlib.machinery.ExtensionFileLoader(fullname, origin)
+ spec = importlib.util.spec_from_loader(fullname, loader)
+ module = importlib.util.module_from_spec(spec)
+ loader.exec_module(module)
+ self.module = module
+
+ def test_subclass_get_module(self):
+ """PyType_GetModule for defining_class"""
+ class StateAccessType_Subclass(self.module.StateAccessType):
+ pass
+
+ instance = StateAccessType_Subclass()
+ self.assertIs(instance.get_defining_module(), self.module)
+
+ def test_subclass_get_module_with_super(self):
+ class StateAccessType_Subclass(self.module.StateAccessType):
+ def get_defining_module(self):
+ return super().get_defining_module()
+
+ instance = StateAccessType_Subclass()
+ self.assertIs(instance.get_defining_module(), self.module)
+
+ def test_state_access(self):
+ """Checks methods defined with and without argument clinic
+
+ This tests a no-arg method (get_count) and a method with
+ both a positional and keyword argument.
+ """
+
+ a = self.module.StateAccessType()
+ b = self.module.StateAccessType()
+
+ methods = {
+ 'clinic': a.increment_count_clinic,
+ 'noclinic': a.increment_count_noclinic,
+ }
+
+ for name, increment_count in methods.items():
+ with self.subTest(name):
+ self.assertEqual(a.get_count(), b.get_count())
+ self.assertEqual(a.get_count(), 0)
+
+ increment_count()
+ self.assertEqual(a.get_count(), b.get_count())
+ self.assertEqual(a.get_count(), 1)
+
+ increment_count(3)
+ self.assertEqual(a.get_count(), b.get_count())
+ self.assertEqual(a.get_count(), 4)
+
+ increment_count(-2, twice=True)
+ self.assertEqual(a.get_count(), b.get_count())
+ self.assertEqual(a.get_count(), 0)
+
+ with self.assertRaises(TypeError):
+ increment_count(thrice=3)
+
+ with self.assertRaises(TypeError):
+ increment_count(1, 2, 3)
+
+
 if __name__ == "__main__":
  unittest.main()

Lib/test/test_sys.py

@@ -1322,7 +1322,7 @@ def delx(self): del self.__x
  '3P' # PyMappingMethods
  '10P' # PySequenceMethods
  '2P' # PyBufferProcs
- '4P')
+ '5P')
  class newstyleclass(object): pass
  # Separate block for PyDictKeysObject with 8 keys and 5 entries
  check(newstyleclass, s + calcsize("2nP2n0P") + 8 + 5*calcsize("n2P"))

Makefile.pre.in

@@ -1104,6 +1104,7 @@ PYTHON_HEADERS= \
 $(srcdir)/Include/cpython/initconfig.h \
 $(srcdir)/Include/cpython/interpreteridobject.h \
 $(srcdir)/Include/cpython/listobject.h \
+$(srcdir)/Include/cpython/methodobject.h \
 $(srcdir)/Include/cpython/object.h \
 $(srcdir)/Include/cpython/objimpl.h \
 $(srcdir)/Include/cpython/pyerrors.h \

Misc/NEWS.d/next/C API/2020-01-22-12-38-59.bpo-38787.HUH6hd.rst

@@ -0,0 +1,2 @@
+Module C state is now accessible from C-defined heap type methods (:pep:`573`).
+Patch by Marcel Plch and Petr Viktorin.