pythonGH-113710: Add a tier 2 peephole optimization pass. (pythonGH-114487)

* Convert _LOAD_CONST to inline versions * Remove PEP 523 checks

Author: markshannon

Include/internal/pycore_uop_ids.h

@@ -202,6 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
  [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
  [_EXIT_TRACE] = HAS_DEOPT_FLAG,
  [_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
+ [_LOAD_CONST_INLINE] = 0,
  [_LOAD_CONST_INLINE_BORROW] = 0,
  [_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
 };
@@ -329,6 +330,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
  [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT",
  [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS",
  [_LOAD_CONST] = "_LOAD_CONST",
+ [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE",
  [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW",
  [_LOAD_DEREF] = "_LOAD_DEREF",
  [_LOAD_FAST] = "_LOAD_FAST",

Include/internal/pycore_uop_metadata.h

@@ -4070,6 +4070,10 @@ dummy_func(
  DEOPT_IF(!current_executor->vm_data.valid);
  }
 
+ op(_LOAD_CONST_INLINE, (ptr/4 -- value)) {
+ value = Py_NewRef(ptr);
+ }
+
  op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) {
  value = ptr;
  }

Python/bytecodes.c

@@ -588,6 +588,9 @@ translate_bytecode_to_trace(
  ADD_TO_TRACE(uop, oparg, operand, target);
  if (uop == _POP_FRAME) {
  TRACE_STACK_POP();
+ /* Set the operand to the code object returned to,
+ * to assist optimization passes */
+ trace[trace_length-1].operand = (uintptr_t)code;
  DPRINTF(2,
  "Returning to %s (%s:%d) at byte offset %d\n",
  PyUnicode_AsUTF8(code->co_qualname),
@@ -629,6 +632,9 @@ translate_bytecode_to_trace(
  instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1;
  TRACE_STACK_PUSH();
  _Py_BloomFilter_Add(dependencies, new_code);
+ /* Set the operand to the callee's code object,
+ * to assist optimization passes */
+ trace[trace_length-1].operand = (uintptr_t)new_code;
  code = new_code;
  instr = _PyCode_CODE(code);
  DPRINTF(2,

Python/executor_cases.c.h

@@ -12,6 +12,39 @@
 #include <stddef.h>
 #include "pycore_optimizer.h"
 
+static void
+peephole_opt(PyCodeObject *co, _PyUOpInstruction *buffer, int buffer_size)
+{
+ for (int pc = 0; pc < buffer_size; pc++) {
+ int opcode = buffer[pc].opcode;
+ switch(opcode) {
+ case _LOAD_CONST: {
+ assert(co != NULL);
+ PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg);
+ buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE;
+ buffer[pc].operand = (uintptr_t)val;
+ break;
+ }
+ case _CHECK_PEP_523:
+ {
+ /* Setting the eval frame function invalidates
+ * all executors, so no need to check dynamically */
+ if (_PyInterpreterState_GET()->eval_frame == NULL) {
+ buffer[pc].opcode = _NOP;
+ }
+ break;
+ }
+ case _PUSH_FRAME:
+ case _POP_FRAME:
+ co = (PyCodeObject *)buffer[pc].operand;
+ break;
+ case _JUMP_TO_TOP:
+ case _EXIT_TRACE:
+ return;
+ }
+ }
+}
+
 static void
 remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
 {
@@ -59,6 +92,7 @@ _Py_uop_analyze_and_optimize(
  int curr_stacklen
 )
 {
+ peephole_opt(co, buffer, buffer_size);
  remove_unneeded_uops(buffer, buffer_size);
  return 0;
 }

Python/optimizer.c

@@ -2608,11 +2608,15 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp,
  _PyFrameEvalFunction eval_frame)
 {
  if (eval_frame == _PyEval_EvalFrameDefault) {
- interp->eval_frame = NULL;
+ eval_frame = NULL;
  }
- else {
- interp->eval_frame = eval_frame;
+ if (eval_frame == interp->eval_frame) {
+ return;
+ }
+ if (eval_frame != NULL) {
+ _Py_Executors_InvalidateAll(interp);
  }
+ interp->eval_frame = eval_frame;
 }