bpo-46841: Quicken code in-place

Include/cpython/code.h

@@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT;
 // Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing:
 #define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode))
 
+// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
+// defined in this macro:
+#define _PyCode_DEF(SIZE) { \
+ PyObject_VAR_HEAD \
+ \
+ /* Note only the following fields are used in hash and/or comparisons \
+ * \
+ * - co_name \
+ * - co_argcount \
+ * - co_posonlyargcount \
+ * - co_kwonlyargcount \
+ * - co_nlocals \
+ * - co_stacksize \
+ * - co_flags \
+ * - co_firstlineno \
+ * - co_consts \
+ * - co_names \
+ * - co_localsplusnames \
+ * This is done to preserve the name and line number for tracebacks \
+ * and debuggers; otherwise, constant de-duplication would collapse \
+ * identical functions/lambdas defined on different lines. \
+ */ \
+ \
+ /* These fields are set with provided values on new code objects. */ \
+ \
+ /* The hottest fields (in the eval loop) are grouped here at the top. */ \
+ PyObject *co_consts; /* list (constants used) */ \
+ PyObject *co_names; /* list of strings (names used) */ \
+ PyObject *co_exceptiontable; /* Byte string encoding exception handling \
+ table */ \
+ int co_flags; /* CO_..., see below */ \
+ int co_warmup; /* Warmup counter for quickening */ \
+ \
+ /* The rest are not so impactful on performance. */ \
+ int co_argcount; /* #arguments, except *args */ \
+ int co_posonlyargcount; /* #positional only arguments */ \
+ int co_kwonlyargcount; /* #keyword only arguments */ \
+ int co_stacksize; /* #entries needed for evaluation stack */ \
+ int co_firstlineno; /* first source line number */ \
+ \
+ /* redundant values (derived from co_localsplusnames and \
+ co_localspluskinds) */ \
+ int co_nlocalsplus; /* number of local + cell + free variables \
+ */ \
+ int co_nlocals; /* number of local variables */ \
+ int co_nplaincellvars; /* number of non-arg cell variables */ \
+ int co_ncellvars; /* total number of cell variables */ \
+ int co_nfreevars; /* number of free variables */ \
+ \
+ PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \
+ PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \
+ per variable) */ \
+ PyObject *co_filename; /* unicode (where it was loaded from) */ \
+ PyObject *co_name; /* unicode (name, for reference) */ \
+ PyObject *co_qualname; /* unicode (qualname, for reference) */ \
+ PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) \
+ See Objects/lnotab_notes.txt for details. \
+ */ \
+ PyObject *co_endlinetable; /* bytes object that holds end lineno for \
+ instructions separated across different \
+ lines */ \
+ PyObject *co_columntable; /* bytes object that holds start/end column \
+ offset each instruction */ \
+ \
+ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \
+ /* Scratch space for extra data relating to the code object. \
+ Type is a void* to keep the format private in codeobject.c to force \
+ people to go through the proper APIs. */ \
+ void *co_extra; \
+ char co_code_adaptive[(SIZE)]; \
+}
 
 /* Bytecode object */
-struct PyCodeObject {
- PyObject_HEAD
-
- /* Note only the following fields are used in hash and/or comparisons
- *
- * - co_name
- * - co_argcount
- * - co_posonlyargcount
- * - co_kwonlyargcount
- * - co_nlocals
- * - co_stacksize
- * - co_flags
- * - co_firstlineno
- * - co_code
- * - co_consts
- * - co_names
- * - co_varnames
- * - co_freevars
- * - co_cellvars
- *
- * This is done to preserve the name and line number for tracebacks
- * and debuggers; otherwise, constant de-duplication would collapse
- * identical functions/lambdas defined on different lines.
- */
-
- /* These fields are set with provided values on new code objects. */
-
- // The hottest fields (in the eval loop) are grouped here at the top.
- PyObject *co_consts; /* list (constants used) */
- PyObject *co_names; /* list of strings (names used) */
- _Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
- Unlike the other "hot" fields, this one is
- actually derived from co_code. */
- PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
- int co_flags; /* CO_..., see below */
- int co_warmup; /* Warmup counter for quickening */
-
- // The rest are not so impactful on performance.
- int co_argcount; /* #arguments, except *args */
- int co_posonlyargcount; /* #positional only arguments */
- int co_kwonlyargcount; /* #keyword only arguments */
- int co_stacksize; /* #entries needed for evaluation stack */
- int co_firstlineno; /* first source line number */
- PyObject *co_code; /* instruction opcodes */
- PyObject *co_localsplusnames; /* tuple mapping offsets to names */
- PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
- PyObject *co_filename; /* unicode (where it was loaded from) */
- PyObject *co_name; /* unicode (name, for reference) */
- PyObject *co_qualname; /* unicode (qualname, for reference) */
- PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) See
- Objects/lnotab_notes.txt for details. */
- PyObject *co_endlinetable; /* bytes object that holds end lineno for
- instructions separated across different
- lines */
- PyObject *co_columntable; /* bytes object that holds start/end column
- offset each instruction */
-
- /* These fields are set with computed values on new code objects. */
-
- // redundant values (derived from co_localsplusnames and co_localspluskinds)
- int co_nlocalsplus; /* number of local + cell + free variables */
- int co_nlocals; /* number of local variables */
- int co_nplaincellvars; /* number of non-arg cell variables */
- int co_ncellvars; /* total number of cell variables */
- int co_nfreevars; /* number of free variables */
- // lazily-computed values
- PyObject *co_varnames; /* tuple of strings (local variable names) */
- PyObject *co_cellvars; /* tuple of strings (cell variable names) */
- PyObject *co_freevars; /* tuple of strings (free variable names) */
-
- /* The remaining fields are zeroed out on new code objects. */
-
- PyObject *co_weakreflist; /* to support weakrefs to code objects */
- /* Scratch space for extra data relating to the code object.
- Type is a void* to keep the format private in codeobject.c to force
- people to go through the proper APIs. */
- void *co_extra;
- /* Quickened instructions and cache, or NULL
- This should be treated as opaque by all code except the specializer and
- interpreter. */
- _Py_CODEUNIT *co_quickened;
-
-};
+struct PyCodeObject _PyCode_DEF(1);
 
 /* Masks for co_flags above */
 #define CO_OPTIMIZED 0x0001
@@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
 
 #define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
 #define PyCode_GetNumFree(op) ((op)->co_nfreevars)
+#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
+#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
 
 /* Public interface */
 PyAPI_FUNC(PyCodeObject *) PyCode_New(

Include/internal/pycore_code.h

@@ -92,30 +92,22 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
 
-/* Maximum size of code to quicken, in code units. */
-#define MAX_SIZE_TO_QUICKEN 10000
-
 #define QUICKENING_WARMUP_DELAY 8
 
 /* We want to compare to zero for efficiency, so we offset values accordingly */
 #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
-#define QUICKENING_WARMUP_COLDEST 1
 
-int _Py_Quicken(PyCodeObject *code);
+void _PyCode_Quicken(PyCodeObject *code);
 
-/* Returns 1 if quickening occurs.
- * -1 if an error occurs
- * 0 otherwise */
-static inline int
-_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
+static inline void
+_PyCode_Warmup(PyCodeObject *code)
 {
  if (code->co_warmup != 0) {
  code->co_warmup++;
  if (code->co_warmup == 0) {
- return _Py_Quicken(code) ? -1 : 1;
+ _PyCode_Quicken(code);
  }
  }
- return 0;
 }
 
 extern Py_ssize_t _Py_QuickenedCount;
@@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
 extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
 extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
 extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
+extern PyObject* _PyCode_GetCode(PyCodeObject *);
 
 /* Return the ending source code line number from a bytecode index. */
 extern int _PyCode_Addr2EndLine(PyCodeObject *, int);