Auto merge of #148302 - folkertdev:non-rustic-unsized, r=bjorn3

error on non-rustic ABIs using unsized parameters tracking issue: #48055 This came up in #144529 (comment). The idea is that the layout of an unsized type is unstable (following the rust layout rules), and hence stable ABIs should not use unsized types. On stable, unsized types (or generics with a `?Sized` bound) are not accepted as parameters, so the errors introduced by this PR can only be observed when the unstable `unsized_fn_params` feature is enabled. r? `@bjorn3` cc `@RalfJung`

Author: bors

compiler/rustc_abi/src/canon_abi.rs

@@ -51,6 +51,20 @@ pub enum CanonAbi {
  X86(X86Call),
 }
 
+impl CanonAbi {
+ pub fn is_rustic_abi(self) -> bool {
+ match self {
+ CanonAbi::Rust | CanonAbi::RustCold => true,
+ CanonAbi::C
+ | CanonAbi::Custom
+ | CanonAbi::Arm(_)
+ | CanonAbi::GpuKernel
+ | CanonAbi::Interrupt(_)
+ | CanonAbi::X86(_) => false,
+ }
+ }
+}
+
 impl fmt::Display for CanonAbi {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  // convert to the ExternAbi that *shares a string* with this CanonAbi.

compiler/rustc_monomorphize/messages.ftl

@@ -12,6 +12,17 @@ monomorphize_abi_error_disabled_vector_type =
  } here
  .help = consider enabling it globally (`-C target-feature=+{$required_feature}`) or locally (`#[target_feature(enable="{$required_feature}")]`)
 
+monomorphize_abi_error_unsupported_unsized_parameter =
+ this function {$is_call ->
+ [true] call
+ *[false] definition
+ } uses unsized type `{$ty}` which is not supported with the chosen ABI
+ .label = function {$is_call ->
+ [true] called
+ *[false] defined
+ } here
+ .help = only rustic ABIs support unsized parameters
+
 monomorphize_abi_error_unsupported_vector_type =
  this function {$is_call ->
  [true] call

compiler/rustc_monomorphize/src/errors.rs

@@ -80,6 +80,18 @@ pub(crate) struct AbiErrorDisabledVectorType<'a> {
  pub is_call: bool,
 }
 
+#[derive(Diagnostic)]
+#[diag(monomorphize_abi_error_unsupported_unsized_parameter)]
+#[help]
+pub(crate) struct AbiErrorUnsupportedUnsizedParameter<'a> {
+ #[primary_span]
+ #[label]
+ pub span: Span,
+ pub ty: Ty<'a>,
+ /// Whether this is a problem at a call site or at a declaration.
+ pub is_call: bool,
+}
+
 #[derive(Diagnostic)]
 #[diag(monomorphize_abi_error_unsupported_vector_type)]
 pub(crate) struct AbiErrorUnsupportedVectorType<'a> {

compiler/rustc_monomorphize/src/mono_checks/abi_check.rs

@@ -78,8 +78,37 @@ fn do_check_simd_vector_abi<'tcx>(
  }
 }
 
-/// Checks that the ABI of a given instance of a function does not contain vector-passed arguments
-/// or return values for which the corresponding target feature is not enabled.
+/// Emit an error when a non-rustic ABI has unsized parameters.
+/// Unsized types do not have a stable layout, so should not be used with stable ABIs.
+/// `is_call` indicates whether this is a call-site check or a definition-site check;
+/// this is only relevant for the wording in the emitted error.
+fn do_check_unsized_params<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+ is_call: bool,
+ loc: impl Fn() -> (Span, HirId),
+) {
+ // Unsized parameters are allowed with the (unstable) "Rust" (and similar) ABIs.
+ if fn_abi.conv.is_rustic_abi() {
+ return;
+ }
+
+ for arg_abi in fn_abi.args.iter() {
+ if !arg_abi.layout.layout.is_sized() {
+ let (span, _hir_id) = loc();
+ tcx.dcx().emit_err(errors::AbiErrorUnsupportedUnsizedParameter {
+ span,
+ ty: arg_abi.layout.ty,
+ is_call,
+ });
+ }
+ }
+}
+
+/// Checks the ABI of an Instance, emitting an error when:
+///
+/// - a non-rustic ABI uses unsized parameters
+/// - the signature requires target features that are not enabled
 fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
  let typing_env = ty::TypingEnv::fully_monomorphized();
  let Ok(abi) = tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty())))
@@ -102,11 +131,14 @@ fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
  def_id.as_local().map(|did| tcx.local_def_id_to_hir_id(did)).unwrap_or(CRATE_HIR_ID),
  )
  };
+ do_check_unsized_params(tcx, abi, /*is_call*/ false, loc);
  do_check_simd_vector_abi(tcx, abi, instance.def_id(), /*is_call*/ false, loc);
 }
 
-/// Checks that a call expression does not try to pass a vector-passed argument which requires a
-/// target feature that the caller does not have, as doing so causes UB because of ABI mismatch.
+/// Check the ABI at a call site, emitting an error when:
+///
+/// - a non-rustic ABI uses unsized parameters
+/// - the signature requires target features that are not enabled
 fn check_call_site_abi<'tcx>(
  tcx: TyCtxt<'tcx>,
  callee: Ty<'tcx>,
@@ -140,6 +172,7 @@ fn check_call_site_abi<'tcx>(
  // ABI failed to compute; this will not get through codegen.
  return;
  };
+ do_check_unsized_params(tcx, callee_abi, /*is_call*/ true, loc);
  do_check_simd_vector_abi(tcx, callee_abi, caller.def_id(), /*is_call*/ true, loc);
 }
 

tests/ui/abi/non-rustic-unsized.rs

@@ -0,0 +1,66 @@
+//@ add-minicore
+//@ build-fail
+#![no_core]
+#![crate_type = "lib"]
+#![feature(no_core, unsized_fn_params)]
+#![allow(improper_ctypes_definitions, improper_ctypes)]
+
+extern crate minicore;
+use minicore::*;
+
+fn rust(_: [u8]) {}
+extern "C" fn c(_: [u8]) {}
+//~^ ERROR this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+extern "system" fn system(_: [u8]) {}
+//~^ ERROR this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+
+#[repr(C)]
+struct CustomUnsized {
+ a: i64,
+ b: [u8],
+}
+
+extern "C" fn c_custom_unsized(x: CustomUnsized) {}
+//~^ ERROR this function definition uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+
+#[unsafe(no_mangle)]
+fn entry(x: [u8], y: [u8], z: [u8], w: CustomUnsized) {
+ rust(x);
+ c(y);
+ //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+ system(z);
+ //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+ c_custom_unsized(w);
+ //~^ ERROR this function call uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+}
+
+#[unsafe(no_mangle)]
+fn test_fn_ptr(rust: extern "Rust" fn(_: [u8]), c: extern "C" fn(_: [u8]), x: [u8], y: [u8]) {
+ rust(x);
+ c(y);
+ //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+}
+
+#[unsafe(no_mangle)]
+fn test_extern(x: [u8], y: [u8]) {
+ unsafe extern "Rust" {
+ safe fn rust(_: [u8]);
+ }
+
+ unsafe extern "system" {
+ safe fn system(_: [u8]);
+ }
+
+ rust(x);
+ system(y);
+ //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+}
+
+extern "C" fn c_polymorphic<T: ?Sized>(_: T) {}
+//~^ ERROR this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+
+#[unsafe(no_mangle)]
+fn test_polymorphic(x: [u8]) {
+ c_polymorphic(x);
+ //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+}

tests/ui/abi/non-rustic-unsized.stderr

@@ -0,0 +1,88 @@
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:29:5
+ |
+LL | c(y);
+ | ^^^^ function called here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:31:5
+ |
+LL | system(z);
+ | ^^^^^^^^^ function called here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:33:5
+ |
+LL | c_custom_unsized(w);
+ | ^^^^^^^^^^^^^^^^^^^ function called here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:12:1
+ |
+LL | extern "C" fn c(_: [u8]) {}
+ | ^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:14:1
+ |
+LL | extern "system" fn system(_: [u8]) {}
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:23:1
+ |
+LL | extern "C" fn c_custom_unsized(x: CustomUnsized) {}
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:40:5
+ |
+LL | c(y);
+ | ^^^^ function called here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:55:5
+ |
+LL | system(y);
+ | ^^^^^^^^^ function called here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:64:5
+ |
+LL | c_polymorphic(x);
+ | ^^^^^^^^^^^^^^^^ function called here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+ --> $DIR/non-rustic-unsized.rs:59:1
+ |
+LL | extern "C" fn c_polymorphic<T: ?Sized>(_: T) {}
+ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+ |
+ = help: only rustic ABIs support unsized parameters
+
+note: the above error was encountered while instantiating `fn c_polymorphic::<[u8]>`
+ --> $DIR/non-rustic-unsized.rs:64:5
+ |
+LL | c_polymorphic(x);
+ | ^^^^^^^^^^^^^^^^
+
+error: aborting due to 10 previous errors
+