Rollup merge of rust-lang#118983 - Urgau:invalid_ref_casting-bigger-layout, r=oli-obk

Warn on references casting to bigger memory layout

This PR extends the [`invalid_reference_casting`](https://doc.rust-lang.org/rustc/lints/listing/deny-by-default.html#invalid-reference-casting) lint (*deny-by-default*) which currently lint on `&T -> &mut T` casting to also lint on `&(mut) A -> &(mut) B` where `size_of::<B>() > size_of::<A>()` (bigger memory layout requirement).

The goal is to detect such cases:

```rust
let u8_ref: &u8 = &0u8;
let u64_ref: &u64 = unsafe { &*(u8_ref as *const u8 as *const u64) };
//~^ ERROR casting references to a bigger memory layout is undefined behavior

let mat3 = Mat3 { a: Vec3(0i32, 0, 0), b: Vec3(0, 0, 0), c: Vec3(0, 0, 0) };
let mat3 = unsafe { &*(&mat3 as *const _ as *const [[i64; 3]; 3]) };
//~^ ERROR casting references to a bigger memory layout is undefined behavior
```

This is added to help people who write unsafe code, especially when people have matrix struct that they cast to simple array of arrays.

EDIT: One caveat, due to the [`&Header`](rust-lang/unsafe-code-guidelines#256) uncertainty the lint only fires when it can find the underline allocation.

~~I have manually tested all the new expressions that warn against Miri, and they all report immediate UB.~~

r? `@est31`

Author: matthiaskrgr

compiler/rustc_lint/messages.ftl

@@ -319,6 +319,11 @@ lint_invalid_nan_comparisons_lt_le_gt_ge = incorrect NaN comparison, NaN is not
 lint_invalid_reference_casting_assign_to_ref = assigning to `&T` is undefined behavior, consider using an `UnsafeCell`
     .label = casting happend here
 
+lint_invalid_reference_casting_bigger_layout = casting references to a bigger memory layout than the backing allocation is undefined behavior, even if the reference is unused
+    .label = casting happend here
+    .alloc = backing allocation comes from here
+    .layout = casting from `{$from_ty}` ({$from_size} bytes) to `{$to_ty}` ({$to_size} bytes)
+
 lint_invalid_reference_casting_borrow_as_mut = casting `&T` to `&mut T` is undefined behavior, even if the reference is unused, consider instead using an `UnsafeCell`
     .label = casting happend here
 

compiler/rustc_lint/src/lints.rs

@@ -716,7 +716,7 @@ pub enum InvalidFromUtf8Diag {
 
 // reference_casting.rs
 #[derive(LintDiagnostic)]
-pub enum InvalidReferenceCastingDiag {
+pub enum InvalidReferenceCastingDiag<'tcx> {
     #[diag(lint_invalid_reference_casting_borrow_as_mut)]
     #[note(lint_invalid_reference_casting_note_book)]
     BorrowAsMut {
@@ -733,6 +733,18 @@ pub enum InvalidReferenceCastingDiag {
         #[note(lint_invalid_reference_casting_note_ty_has_interior_mutability)]
         ty_has_interior_mutability: Option<()>,
     },
+    #[diag(lint_invalid_reference_casting_bigger_layout)]
+    #[note(lint_layout)]
+    BiggerLayout {
+        #[label]
+        orig_cast: Option<Span>,
+        #[label(lint_alloc)]
+        alloc: Span,
+        from_ty: Ty<'tcx>,
+        from_size: u64,
+        to_ty: Ty<'tcx>,
+        to_size: u64,
+    },
 }
 
 // hidden_unicode_codepoints.rs

compiler/rustc_lint/src/reference_casting.rs

@@ -1,6 +1,7 @@
 use rustc_ast::Mutability;
 use rustc_hir::{Expr, ExprKind, UnOp};
-use rustc_middle::ty::{self, TypeAndMut};
+use rustc_middle::ty::layout::LayoutOf as _;
+use rustc_middle::ty::{self, layout::TyAndLayout, TypeAndMut};
 use rustc_span::sym;
 
 use crate::{lints::InvalidReferenceCastingDiag, LateContext, LateLintPass, LintContext};
@@ -38,13 +39,19 @@ declare_lint_pass!(InvalidReferenceCasting => [INVALID_REFERENCE_CASTING]);
 impl<'tcx> LateLintPass<'tcx> for InvalidReferenceCasting {
     fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'tcx>) {
         if let Some((e, pat)) = borrow_or_assign(cx, expr) {
-            if matches!(pat, PatternKind::Borrow { mutbl: Mutability::Mut } | PatternKind::Assign) {
-                let init = cx.expr_or_init(e);
+            let init = cx.expr_or_init(e);
+            let orig_cast = if init.span != e.span { Some(init.span) } else { None };
+
+            // small cache to avoid recomputing needlesly computing peel_casts of init
+            let mut peel_casts = {
+                let mut peel_casts_cache = None;
+                move || *peel_casts_cache.get_or_insert_with(|| peel_casts(cx, init))
+            };
 
-                let Some(ty_has_interior_mutability) = is_cast_from_ref_to_mut_ptr(cx, init) else {
-                    return;
-                };
-                let orig_cast = if init.span != e.span { Some(init.span) } else { None };
+            if matches!(pat, PatternKind::Borrow { mutbl: Mutability::Mut } | PatternKind::Assign)
+                && let Some(ty_has_interior_mutability) =
+                    is_cast_from_ref_to_mut_ptr(cx, init, &mut peel_casts)
+            {
                 let ty_has_interior_mutability = ty_has_interior_mutability.then_some(());
 
                 cx.emit_span_lint(
@@ -63,6 +70,23 @@ impl<'tcx> LateLintPass<'tcx> for InvalidReferenceCasting {
                     },
                 );
             }
+
+            if let Some((from_ty_layout, to_ty_layout, e_alloc)) =
+                is_cast_to_bigger_memory_layout(cx, init, &mut peel_casts)
+            {
+                cx.emit_span_lint(
+                    INVALID_REFERENCE_CASTING,
+                    expr.span,
+                    InvalidReferenceCastingDiag::BiggerLayout {
+                        orig_cast,
+                        alloc: e_alloc.span,
+                        from_ty: from_ty_layout.ty,
+                        from_size: from_ty_layout.layout.size().bytes(),
+                        to_ty: to_ty_layout.ty,
+                        to_size: to_ty_layout.layout.size().bytes(),
+                    },
+                );
+            }
         }
     }
 }
@@ -124,6 +148,7 @@ fn borrow_or_assign<'tcx>(
 fn is_cast_from_ref_to_mut_ptr<'tcx>(
     cx: &LateContext<'tcx>,
     orig_expr: &'tcx Expr<'tcx>,
+    mut peel_casts: impl FnMut() -> (&'tcx Expr<'tcx>, bool),
 ) -> Option<bool> {
     let end_ty = cx.typeck_results().node_type(orig_expr.hir_id);
 
@@ -132,7 +157,7 @@ fn is_cast_from_ref_to_mut_ptr<'tcx>(
         return None;
     }
 
-    let (e, need_check_freeze) = peel_casts(cx, orig_expr);
+    let (e, need_check_freeze) = peel_casts();
 
     let start_ty = cx.typeck_results().node_type(e.hir_id);
     if let ty::Ref(_, inner_ty, Mutability::Not) = start_ty.kind() {
@@ -151,6 +176,49 @@ fn is_cast_from_ref_to_mut_ptr<'tcx>(
     }
 }
 
+fn is_cast_to_bigger_memory_layout<'tcx>(
+    cx: &LateContext<'tcx>,
+    orig_expr: &'tcx Expr<'tcx>,
+    mut peel_casts: impl FnMut() -> (&'tcx Expr<'tcx>, bool),
+) -> Option<(TyAndLayout<'tcx>, TyAndLayout<'tcx>, Expr<'tcx>)> {
+    let end_ty = cx.typeck_results().node_type(orig_expr.hir_id);
+
+    let ty::RawPtr(TypeAndMut { ty: inner_end_ty, mutbl: _ }) = end_ty.kind() else {
+        return None;
+    };
+
+    let (e, _) = peel_casts();
+    let start_ty = cx.typeck_results().node_type(e.hir_id);
+
+    let ty::Ref(_, inner_start_ty, _) = start_ty.kind() else {
+        return None;
+    };
+
+    // try to find the underlying allocation
+    let e_alloc = cx.expr_or_init(e);
+    let e_alloc =
+        if let ExprKind::AddrOf(_, _, inner_expr) = e_alloc.kind { inner_expr } else { e_alloc };
+    let alloc_ty = cx.typeck_results().node_type(e_alloc.hir_id);
+
+    // if we do not find it we bail out, as this may not be UB
+    // see https://github.com/rust-lang/unsafe-code-guidelines/issues/256
+    if alloc_ty.is_any_ptr() {
+        return None;
+    }
+
+    let from_layout = cx.layout_of(*inner_start_ty).ok()?;
+    let alloc_layout = cx.layout_of(alloc_ty).ok()?;
+    let to_layout = cx.layout_of(*inner_end_ty).ok()?;
+
+    if to_layout.layout.size() > from_layout.layout.size()
+        && to_layout.layout.size() > alloc_layout.layout.size()
+    {
+        Some((from_layout, to_layout, *e_alloc))
+    } else {
+        None
+    }
+}
+
 fn peel_casts<'tcx>(cx: &LateContext<'tcx>, mut e: &'tcx Expr<'tcx>) -> (&'tcx Expr<'tcx>, bool) {
     let mut gone_trough_unsafe_cell_raw_get = false;
 

src/tools/clippy/tests/ui/transmute_ptr_to_ptr.fixed

@@ -35,16 +35,16 @@ fn transmute_ptr_to_ptr() {
         // ref-ref transmutes; bad
         let _: &f32 = &*(&1u32 as *const u32 as *const f32);
         //~^ ERROR: transmute from a reference to a reference
-        let _: &f64 = &*(&1f32 as *const f32 as *const f64);
+        let _: &f32 = &*(&1f64 as *const f64 as *const f32);
         //~^ ERROR: transmute from a reference to a reference
         //:^ this test is here because both f32 and f64 are the same TypeVariant, but they are not
         // the same type
         let _: &mut f32 = &mut *(&mut 1u32 as *mut u32 as *mut f32);
         //~^ ERROR: transmute from a reference to a reference
         let _: &GenericParam<f32> = &*(&GenericParam { t: 1u32 } as *const GenericParam<u32> as *const GenericParam<f32>);
         //~^ ERROR: transmute from a reference to a reference
-        let u8_ref: &u8 = &0u8;
-        let u64_ref: &u64 = unsafe { &*(u8_ref as *const u8 as *const u64) };
+        let u64_ref: &u64 = &0u64;
+        let u8_ref: &u8 = unsafe { &*(u64_ref as *const u64 as *const u8) };
         //~^ ERROR: transmute from a reference to a reference
     }
 

src/tools/clippy/tests/ui/transmute_ptr_to_ptr.rs

@@ -35,16 +35,16 @@ fn transmute_ptr_to_ptr() {
         // ref-ref transmutes; bad
         let _: &f32 = std::mem::transmute(&1u32);
         //~^ ERROR: transmute from a reference to a reference
-        let _: &f64 = std::mem::transmute(&1f32);
+        let _: &f32 = std::mem::transmute(&1f64);
         //~^ ERROR: transmute from a reference to a reference
         //:^ this test is here because both f32 and f64 are the same TypeVariant, but they are not
         // the same type
         let _: &mut f32 = std::mem::transmute(&mut 1u32);
         //~^ ERROR: transmute from a reference to a reference
         let _: &GenericParam<f32> = std::mem::transmute(&GenericParam { t: 1u32 });
         //~^ ERROR: transmute from a reference to a reference
-        let u8_ref: &u8 = &0u8;
-        let u64_ref: &u64 = unsafe { std::mem::transmute(u8_ref) };
+        let u64_ref: &u64 = &0u64;
+        let u8_ref: &u8 = unsafe { std::mem::transmute(u64_ref) };
         //~^ ERROR: transmute from a reference to a reference
     }
 

src/tools/clippy/tests/ui/transmute_ptr_to_ptr.stderr

@@ -22,8 +22,8 @@ LL |         let _: &f32 = std::mem::transmute(&1u32);
 error: transmute from a reference to a reference
   --> $DIR/transmute_ptr_to_ptr.rs:38:23
    |
-LL |         let _: &f64 = std::mem::transmute(&1f32);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(&1f32 as *const f32 as *const f64)`
+LL |         let _: &f32 = std::mem::transmute(&1f64);
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(&1f64 as *const f64 as *const f32)`
 
 error: transmute from a reference to a reference
   --> $DIR/transmute_ptr_to_ptr.rs:42:27
@@ -38,10 +38,10 @@ LL |         let _: &GenericParam<f32> = std::mem::transmute(&GenericParam { t:
    |                                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(&GenericParam { t: 1u32 } as *const GenericParam<u32> as *const GenericParam<f32>)`
 
 error: transmute from a reference to a reference
-  --> $DIR/transmute_ptr_to_ptr.rs:47:38
+  --> $DIR/transmute_ptr_to_ptr.rs:47:36
    |
-LL |         let u64_ref: &u64 = unsafe { std::mem::transmute(u8_ref) };
-   |                                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(u8_ref as *const u8 as *const u64)`
+LL |         let u8_ref: &u8 = unsafe { std::mem::transmute(u64_ref) };
+   |                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(u64_ref as *const u64 as *const u8)`
 
 error: aborting due to 7 previous errors
 

src/tools/miri/tests/fail/unaligned_pointers/unaligned_ref_addr_of.rs

@@ -1,6 +1,8 @@
 // This should fail even without Stacked Borrows.
 //@compile-flags: -Zmiri-disable-stacked-borrows -Cdebug-assertions=no
 
+#![allow(invalid_reference_casting)] // for u16 -> u32
+
 fn main() {
     // Try many times as this might work by chance.
     for _ in 0..20 {

tests/ui/cast/cast-rfc0401.rs

@@ -81,14 +81,14 @@ fn main()
     assert_eq!(u as *const u16, p as *const u16);
 
     // ptr-ptr-cast (Length vtables)
-    let mut l : [u8; 2] = [0,1];
-    let w: *mut [u16; 2] = &mut l as *mut [u8; 2] as *mut _;
-    let w: *mut [u16] = unsafe {&mut *w};
-    let w_u8 : *const [u8] = w as *const [u8];
-    assert_eq!(unsafe{&*w_u8}, &l);
+    let mut l : [u16; 2] = [0,1];
+    let w: *mut [u8; 2] = &mut l as *mut [u16; 2] as *mut _;
+    let w: *mut [u8] = unsafe {&mut *w};
+    let w_u16 : *const [u16] = w as *const [u16];
+    assert_eq!(unsafe{&*w_u16}, &l);
 
     let s: *mut str = w as *mut str;
-    let l_via_str = unsafe{&*(s as *const [u8])};
+    let l_via_str = unsafe{&*(s as *const [u16])};
     assert_eq!(&l, l_via_str);
 
     // ptr-ptr-cast (Length vtables, check length is preserved)

tests/ui/lint/reference_casting.rs

@@ -30,7 +30,7 @@ unsafe fn ref_to_mut() {
     //~^ ERROR casting `&T` to `&mut T` is undefined behavior
     let _num = &mut *(num as *const i32).cast::<i32>().cast_mut().cast_const().cast_mut();
     //~^ ERROR casting `&T` to `&mut T` is undefined behavior
-    let _num = &mut *(std::ptr::from_ref(static_u8()) as *mut i32);
+    let _num = &mut *(std::ptr::from_ref(static_u8()) as *mut i8);
     //~^ ERROR casting `&T` to `&mut T` is undefined behavior
     let _num = &mut *std::mem::transmute::<_, *mut i32>(num);
     //~^ ERROR casting `&T` to `&mut T` is undefined behavior
@@ -141,6 +141,109 @@ unsafe fn assign_to_ref() {
     }
 }
 
+#[repr(align(16))]
+struct I64(i64);
+
+#[repr(C)]
+struct Mat3<T> {
+    a: Vec3<T>,
+    b: Vec3<T>,
+    c: Vec3<T>,
+}
+
+#[repr(C)]
+struct Vec3<T>(T, T, T);
+
+unsafe fn bigger_layout() {
+    {
+        let num = &mut 3i32;
+
+        let _num = &*(num as *const i32 as *const i64);
+        //~^ ERROR casting references to a bigger memory layout
+        let _num = &mut *(num as *mut i32 as *mut i64);
+        //~^ ERROR casting references to a bigger memory layout
+        let _num = &mut *(num as *mut i32 as *mut I64);
+        //~^ ERROR casting references to a bigger memory layout
+        std::ptr::write(num as *mut i32 as *mut i64, 2);
+        //~^ ERROR casting references to a bigger memory layout
+
+        let _num = &mut *(num as *mut i32);
+    }
+
+    {
+        let num = &mut [0i32; 3];
+
+        let _num = &mut *(num as *mut _ as *mut [i64; 2]);
+        //~^ ERROR casting references to a bigger memory layout
+        std::ptr::write_unaligned(num as *mut _ as *mut [i32; 4], [0, 0, 1, 1]);
+        //~^ ERROR casting references to a bigger memory layout
+
+        let _num = &mut *(num as *mut _ as *mut [u32; 3]);
+        let _num = &mut *(num as *mut _ as *mut [u32; 2]);
+    }
+
+    {
+        let num = &mut [0i32; 3] as &mut [i32];
+
+        let _num = &mut *(num as *mut _ as *mut i128);
+        //~^ ERROR casting references to a bigger memory layout
+        let _num = &mut *(num as *mut _ as *mut [i64; 4]);
+        //~^ ERROR casting references to a bigger memory layout
+
+        let _num = &mut *(num as *mut _ as *mut [u32]);
+        let _num = &mut *(num as *mut _ as *mut [i16]);
+    }
+
+    {
+        let mat3 = Mat3 { a: Vec3(0i32, 0, 0), b: Vec3(0, 0, 0), c: Vec3(0, 0, 0) };
+
+        let _num = &mut *(&mat3 as *const _ as *mut [[i64; 3]; 3]);
+        //~^ ERROR casting `&T` to `&mut T`
+        //~^^ ERROR casting references to a bigger memory layout
+        let _num = &*(&mat3 as *const _ as *mut [[i64; 3]; 3]);
+        //~^ ERROR casting references to a bigger memory layout
+
+        let _num = &*(&mat3 as *const _ as *mut [[i32; 3]; 3]);
+    }
+
+    {
+        let mut l: [u8; 2] = [0,1];
+        let w: *mut [u16; 2] = &mut l as *mut [u8; 2] as *mut _;
+        let w: *mut [u16] = unsafe {&mut *w};
+        //~^ ERROR casting references to a bigger memory layout
+    }
+
+    {
+        fn foo() -> [i32; 1] { todo!() }
+
+        let num = foo();
+        let _num = &*(&num as *const i32 as *const i64);
+        //~^ ERROR casting references to a bigger memory layout
+        let _num = &*(&foo() as *const i32 as *const i64);
+        //~^ ERROR casting references to a bigger memory layout
+    }
+
+    {
+        fn bar(_a: &[i32; 2]) -> &[i32; 1] { todo!() }
+
+        let num = bar(&[0, 0]);
+        let _num = &*(num as *const i32 as *const i64);
+        let _num = &*(bar(&[0, 0]) as *const i32 as *const i64);
+    }
+
+    {
+        fn foi<T>() -> T { todo!() }
+
+        let num = foi::<i32>();
+        let _num = &*(&num as *const i32 as *const i64);
+        //~^ ERROR casting references to a bigger memory layout
+    }
+
+    unsafe fn from_ref(this: &i32) -> &i64 {
+        &*(this as *const i32 as *const i64)
+    }
+}
+
 const RAW_PTR: *mut u8 = 1 as *mut u8;
 unsafe fn no_warn() {
     let num = &3i32;