split PyFunctionArgument to specialize Option (#5002)

* split `PyFunctionArgument` to specialize `Option` Adds const generic parameter to `PyFunctionArgument` to allow specialization. This allows `Option` wrapped extraction of some types that can't implement `FromPyObject(Bound)` because they require a `holder` to borrow from, most notably `&T` for `T: PyClass`. Closes #4965 * tests and changelog * clippy --------- Co-authored-by: David Hewitt <mail@davidhewitt.dev>

Author: Icxolu

guide/src/class.md

@@ -1390,7 +1390,7 @@ impl pyo3::PyClass for MyClass {
  type Frozen = pyo3::pyclass::boolean_struct::False;
 }
 
-impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a MyClass
+impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a MyClass
 {
  type Holder = ::std::option::Option<pyo3::PyRef<'py, MyClass>>;
 
@@ -1400,7 +1400,7 @@ impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a
  }
 }
 
-impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut MyClass
+impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a mut MyClass
 {
  type Holder = ::std::option::Option<pyo3::PyRefMut<'py, MyClass>>;
 

newsfragments/5002.fixed.md

@@ -0,0 +1 @@
+Fix compile failure with required `#[pyfunction]` arguments taking `Option<&str>` and `Option<&T>` (for `#[pyclass]` types).

pyo3-macros-backend/src/params.rs

@@ -1,4 +1,4 @@
-use crate::utils::{deprecated_from_py_with, Ctx};
+use crate::utils::{deprecated_from_py_with, Ctx, TypeExt as _};
 use crate::{
  attributes::FromPyWithAttribute,
  method::{FnArg, FnSpec, RegularArg},
@@ -200,7 +200,7 @@ fn impl_arg_param(
  let holder = holders.push_holder(arg.name.span());
  let name_str = arg.name.to_string();
  quote_spanned! { arg.name.span() =>
- #pyo3_path::impl_::extract_argument::extract_argument(
+ #pyo3_path::impl_::extract_argument::extract_argument::<_, false>(
  &_args,
  &mut #holder,
  #name_str
@@ -211,7 +211,7 @@ fn impl_arg_param(
  let holder = holders.push_holder(arg.name.span());
  let name_str = arg.name.to_string();
  quote_spanned! { arg.name.span() =>
- #pyo3_path::impl_::extract_argument::extract_optional_argument(
+ #pyo3_path::impl_::extract_argument::extract_optional_argument::<_, false>(
  _kwargs.as_deref(),
  &mut #holder,
  #name_str,
@@ -238,8 +238,9 @@ pub(crate) fn impl_regular_arg_param(
 
  // Use this macro inside this function, to ensure that all code generated here is associated
  // with the function argument
+ let use_probe = quote!(use #pyo3_path::impl_::pyclass::Probe as _;);
  macro_rules! quote_arg_span {
- ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => $($tokens)*) }
+ ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => { #use_probe $($tokens)* }) }
  }
 
  let name_str = arg.name.to_string();
@@ -251,6 +252,7 @@ pub(crate) fn impl_regular_arg_param(
  default = default.map(|tokens| some_wrap(tokens, ctx));
  }
 
+ let arg_ty = arg.ty.clone().elide_lifetimes();
  if let Some(FromPyWithAttribute { kw, .. }) = arg.from_py_with {
  let extractor = quote_spanned! { kw.span =>
  { let from_py_with: fn(_) -> _ = #from_py_with; from_py_with }
@@ -279,9 +281,13 @@ pub(crate) fn impl_regular_arg_param(
  }
  } else if let Some(default) = default {
  let holder = holders.push_holder(arg.name.span());
- if arg.option_wrapped_type.is_some() {
+ if let Some(arg_ty) = arg.option_wrapped_type {
+ let arg_ty = arg_ty.clone().elide_lifetimes();
  quote_arg_span! {
- #pyo3_path::impl_::extract_argument::extract_optional_argument(
+ #pyo3_path::impl_::extract_argument::extract_optional_argument::<
+ _,
+ { #pyo3_path::impl_::pyclass::IsOption::<#arg_ty>::VALUE }
+ >(
  #arg_value,
  &mut #holder,
  #name_str,
@@ -293,22 +299,28 @@ pub(crate) fn impl_regular_arg_param(
  }
  } else {
  quote_arg_span! {
- #pyo3_path::impl_::extract_argument::extract_argument_with_default(
- #arg_value,
- &mut #holder,
- #name_str,
- #[allow(clippy::redundant_closure)]
- {
- || #default
- }
- )?
+ #pyo3_path::impl_::extract_argument::extract_argument_with_default::<
+ _,
+ { #pyo3_path::impl_::pyclass::IsOption::<#arg_ty>::VALUE }
+ >(
+ #arg_value,
+ &mut #holder,
+ #name_str,
+ #[allow(clippy::redundant_closure)]
+ {
+ || #default
+ }
+ )?
  }
  }
  } else {
  let holder = holders.push_holder(arg.name.span());
  let unwrap = quote! {unsafe { #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value) }};
  quote_arg_span! {
- #pyo3_path::impl_::extract_argument::extract_argument(
+ #pyo3_path::impl_::extract_argument::extract_argument::<
+ _,
+ { #pyo3_path::impl_::pyclass::IsOption::<#arg_ty>::VALUE }
+ >(
  #unwrap,
  &mut #holder,
  #name_str

pyo3-macros-backend/src/pyclass.rs

@@ -2097,7 +2097,7 @@ impl<'a> PyClassImplsBuilder<'a> {
  let cls = self.cls;
  if self.attr.options.frozen.is_some() {
  quote! {
- impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls
+ impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a #cls
  {
  type Holder = ::std::option::Option<#pyo3_path::PyRef<'py, #cls>>;
 
@@ -2109,7 +2109,7 @@ impl<'a> PyClassImplsBuilder<'a> {
  }
  } else {
  quote! {
- impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls
+ impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a #cls
  {
  type Holder = ::std::option::Option<#pyo3_path::PyRef<'py, #cls>>;
 
@@ -2119,7 +2119,7 @@ impl<'a> PyClassImplsBuilder<'a> {
  }
  }
 
- impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut #cls
+ impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a mut #cls
  {
  type Holder = ::std::option::Option<#pyo3_path::PyRefMut<'py, #cls>>;
 

pyo3-macros-backend/src/pymethod.rs

@@ -4,7 +4,7 @@ use std::ffi::CString;
 use crate::attributes::{FromPyWithAttribute, NameAttribute, RenamingRule};
 use crate::method::{CallingConvention, ExtractErrorMode, PyArg};
 use crate::params::{impl_regular_arg_param, Holders};
-use crate::utils::{deprecated_from_py_with, PythonDoc};
+use crate::utils::{deprecated_from_py_with, PythonDoc, TypeExt as _};
 use crate::utils::{Ctx, LitCStr};
 use crate::{
  method::{FnArg, FnSpec, FnType, SelfType},
@@ -699,8 +699,13 @@ pub fn impl_py_setter_def(
  .unwrap_or_default();
 
  let holder = holders.push_holder(span);
+ let ty = field.ty.clone().elide_lifetimes();
  quote! {
- let _val = #pyo3_path::impl_::extract_argument::extract_argument(_value.into(), &mut #holder, #name)?;
+ use #pyo3_path::impl_::pyclass::Probe as _;
+ let _val = #pyo3_path::impl_::extract_argument::extract_argument::<
+ _,
+ { #pyo3_path::impl_::pyclass::IsOption::<#ty>::VALUE }
+ >(_value.into(), &mut #holder, #name)?;
  }
  }
  };
@@ -1198,13 +1203,18 @@ fn extract_object(
  }
  } else {
  let holder = holders.push_holder(Span::call_site());
- quote! {
- #pyo3_path::impl_::extract_argument::extract_argument(
+ let ty = arg.ty().clone().elide_lifetimes();
+ quote! {{
+ use #pyo3_path::impl_::pyclass::Probe as _;
+ #pyo3_path::impl_::extract_argument::extract_argument::<
+ _,
+ { #pyo3_path::impl_::pyclass::IsOption::<#ty>::VALUE }
+ >(
  unsafe { #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr).0 },
  &mut #holder,
  #name
  )
- }
+ }}
  };
 
  let extracted = extract_error_mode.handle_error(extract, ctx);

pyo3-macros-backend/src/utils.rs

@@ -337,3 +337,73 @@ pub(crate) fn deprecated_from_py_with(expr_path: &ExprPathWrap) -> Option<TokenS
  }
  })
 }
+
+pub(crate) trait TypeExt {
+ /// Replaces all explicit lifetimes in `self` with elided (`'_`) lifetimes
+ ///
+ /// This is useful if `Self` is used in `const` context, where explicit
+ /// lifetimes are not allowed (yet).
+ fn elide_lifetimes(self) -> Self;
+}
+
+impl TypeExt for syn::Type {
+ fn elide_lifetimes(mut self) -> Self {
+ fn elide_lifetimes(ty: &mut syn::Type) {
+ match ty {
+ syn::Type::Path(type_path) => {
+ if let Some(qself) = &mut type_path.qself {
+ elide_lifetimes(&mut qself.ty)
+ }
+ for seg in &mut type_path.path.segments {
+ if let syn::PathArguments::AngleBracketed(args) = &mut seg.arguments {
+ for generic_arg in &mut args.args {
+ match generic_arg {
+ syn::GenericArgument::Lifetime(lt) => {
+ *lt = syn::Lifetime::new("'_", lt.span());
+ }
+ syn::GenericArgument::Type(ty) => elide_lifetimes(ty),
+ syn::GenericArgument::AssocType(assoc) => {
+ elide_lifetimes(&mut assoc.ty)
+ }
+
+ syn::GenericArgument::Const(_)
+ | syn::GenericArgument::AssocConst(_)
+ | syn::GenericArgument::Constraint(_)
+ | _ => {}
+ }
+ }
+ }
+ }
+ }
+ syn::Type::Reference(type_ref) => {
+ if let Some(lt) = type_ref.lifetime.as_mut() {
+ *lt = syn::Lifetime::new("'_", lt.span());
+ }
+ elide_lifetimes(&mut type_ref.elem);
+ }
+ syn::Type::Tuple(type_tuple) => {
+ for ty in &mut type_tuple.elems {
+ elide_lifetimes(ty);
+ }
+ }
+ syn::Type::Array(type_array) => elide_lifetimes(&mut type_array.elem),
+ syn::Type::Slice(ty) => elide_lifetimes(&mut ty.elem),
+ syn::Type::Group(ty) => elide_lifetimes(&mut ty.elem),
+ syn::Type::Paren(ty) => elide_lifetimes(&mut ty.elem),
+ syn::Type::Ptr(ty) => elide_lifetimes(&mut ty.elem),
+
+ syn::Type::BareFn(_)
+ | syn::Type::ImplTrait(_)
+ | syn::Type::Infer(_)
+ | syn::Type::Macro(_)
+ | syn::Type::Never(_)
+ | syn::Type::TraitObject(_)
+ | syn::Type::Verbatim(_)
+ | _ => {}
+ }
+ }
+
+ elide_lifetimes(&mut self);
+ self
+ }
+}

src/impl_/extract_argument.rs

@@ -20,12 +20,12 @@ type PyArg<'py> = Borrowed<'py, 'py, PyAny>;
 /// will be dropped as soon as the pyfunction call ends.
 ///
 /// There exists a trivial blanket implementation for `T: FromPyObject` with `Holder = ()`.
-pub trait PyFunctionArgument<'a, 'py>: Sized + 'a {
+pub trait PyFunctionArgument<'a, 'py, const IS_OPTION: bool>: Sized + 'a {
  type Holder: FunctionArgumentHolder;
  fn extract(obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder) -> PyResult<Self>;
 }
 
-impl<'a, 'py, T> PyFunctionArgument<'a, 'py> for T
+impl<'a, 'py, T> PyFunctionArgument<'a, 'py, false> for T
 where
  T: FromPyObjectBound<'a, 'py> + 'a,
 {
@@ -37,7 +37,7 @@ where
  }
 }
 
-impl<'a, 'py, T: 'py> PyFunctionArgument<'a, 'py> for &'a Bound<'py, T>
+impl<'a, 'py, T: 'py> PyFunctionArgument<'a, 'py, false> for &'a Bound<'py, T>
 where
  T: PyTypeCheck,
 {
@@ -49,24 +49,24 @@ where
  }
 }
 
-impl<'a, 'py, T: 'py> PyFunctionArgument<'a, 'py> for Option<&'a Bound<'py, T>>
+impl<'a, 'py, T> PyFunctionArgument<'a, 'py, true> for Option<T>
 where
- T: PyTypeCheck,
+ T: PyFunctionArgument<'a, 'py, false>, // inner `Option`s will use `FromPyObject`
 {
- type Holder = ();
+ type Holder = T::Holder;
 
  #[inline]
- fn extract(obj: &'a Bound<'py, PyAny>, _: &'a mut ()) -> PyResult<Self> {
+ fn extract(obj: &'a Bound<'py, PyAny>, holder: &'a mut T::Holder) -> PyResult<Self> {
  if obj.is_none() {
  Ok(None)
  } else {
- Ok(Some(obj.downcast()?))
+ Ok(Some(T::extract(obj, holder)?))
  }
  }
 }
 
 #[cfg(all(Py_LIMITED_API, not(Py_3_10)))]
-impl<'a> PyFunctionArgument<'a, '_> for &'a str {
+impl<'a> PyFunctionArgument<'a, '_, false> for &'a str {
  type Holder = Option<std::borrow::Cow<'a, str>>;
 
  #[inline]
@@ -110,13 +110,13 @@ pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass<Frozen = False>>(
 
 /// The standard implementation of how PyO3 extracts a `#[pyfunction]` or `#[pymethod]` function argument.
 #[doc(hidden)]
-pub fn extract_argument<'a, 'py, T>(
+pub fn extract_argument<'a, 'py, T, const IS_OPTION: bool>(
  obj: &'a Bound<'py, PyAny>,
  holder: &'a mut T::Holder,
  arg_name: &str,
 ) -> PyResult<T>
 where
- T: PyFunctionArgument<'a, 'py>,
+ T: PyFunctionArgument<'a, 'py, IS_OPTION>,
 {
  match PyFunctionArgument::extract(obj, holder) {
  Ok(value) => Ok(value),
@@ -127,14 +127,14 @@ where
 /// Alternative to [`extract_argument`] used for `Option<T>` arguments. This is necessary because Option<&T>
 /// does not implement `PyFunctionArgument` for `T: PyClass`.
 #[doc(hidden)]
-pub fn extract_optional_argument<'a, 'py, T>(
+pub fn extract_optional_argument<'a, 'py, T, const IS_OPTION: bool>(
  obj: Option<&'a Bound<'py, PyAny>>,
  holder: &'a mut T::Holder,
  arg_name: &str,
  default: fn() -> Option<T>,
 ) -> PyResult<Option<T>>
 where
- T: PyFunctionArgument<'a, 'py>,
+ T: PyFunctionArgument<'a, 'py, IS_OPTION>,
 {
  match obj {
  Some(obj) => {
@@ -151,14 +151,14 @@ where
 
 /// Alternative to [`extract_argument`] used when the argument has a default value provided by an annotation.
 #[doc(hidden)]
-pub fn extract_argument_with_default<'a, 'py, T>(
+pub fn extract_argument_with_default<'a, 'py, T, const IS_OPTION: bool>(
  obj: Option<&'a Bound<'py, PyAny>>,
  holder: &'a mut T::Holder,
  arg_name: &str,
  default: fn() -> T,
 ) -> PyResult<T>
 where
- T: PyFunctionArgument<'a, 'py>,
+ T: PyFunctionArgument<'a, 'py, IS_OPTION>,
 {
  match obj {
  Some(obj) => extract_argument(obj, holder, arg_name),

src/impl_/pyclass/probes.rs

@@ -70,3 +70,9 @@ probe!(IsSync);
 impl<T: Sync> IsSync<T> {
  pub const VALUE: bool = true;
 }
+
+probe!(IsOption);
+
+impl<T> IsOption<Option<T>> {
+ pub const VALUE: bool = true;
+}