More micro-opt

Author: saethlin

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -1,5 +1,6 @@
 use rustc_middle::mir;
 use rustc_middle::mir::NonDivergingIntrinsic;
+use rustc_session::config::OptLevel;
 
 use super::FunctionCx;
 use super::LocalRef;
@@ -67,8 +68,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_coverage(bx, coverage, statement.source_info.scope);
             }
             mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
-                let op_val = self.codegen_operand(bx, op);
-                bx.assume(op_val.immediate());
+                if !matches!(bx.tcx().sess.opts.optimize, OptLevel::No | OptLevel::Less) {
+                    let op_val = self.codegen_operand(bx, op);
+                    bx.assume(op_val.immediate());
+                }
             }
             mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
                 mir::CopyNonOverlapping { ref count, ref src, ref dst },

library/core/src/intrinsics.rs

@@ -56,7 +56,7 @@
 
 use crate::marker::DiscriminantKind;
 use crate::marker::Tuple;
-use crate::mem;
+use crate::mem::{self, align_of};
 
 pub mod mir;
 pub mod simd;
@@ -2599,10 +2599,9 @@ pub const unsafe fn is_val_statically_known<T: Copy>(_arg: T) -> bool {
 /// Check that the preconditions of an unsafe function are followed, if debug_assertions are on,
 /// and only at runtime.
 ///
-/// This macro should be called as `assert_unsafe_precondition!([Generics](name: Type) => Expression)`
+/// This macro should be called as `assert_unsafe_precondition!((name: Type) => Expression)`
 /// where the names specified will be moved into the macro as captured variables, and defines an item
-/// to call `const_eval_select` on. The tokens inside the square brackets are used to denote generics
-/// for the function declarations and can be omitted if there is no generics.
+/// to call `const_eval_select` on.
 ///
 /// # Safety
 ///
@@ -2618,27 +2617,22 @@ pub const unsafe fn is_val_statically_known<T: Copy>(_arg: T) -> bool {
 /// the occasional mistake, and this check should help them figure things out.
 #[allow_internal_unstable(const_eval_select, delayed_debug_assertions)] // permit this to be called in stably-const fn
 macro_rules! assert_unsafe_precondition {
-    ($name:expr, $([$($tt:tt)*])?($($i:ident:$ty:ty),*$(,)?) => $e:expr $(,)?) => {
+    ($name:expr, ($($i:ident:$ty:ty),*$(,)?) => $e:expr $(,)?) => {
         {
-        // allow non_snake_case to allow capturing const generics
-        #[allow(non_snake_case)]
-        #[inline(always)]
-        fn runtime$(<$($tt)*>)?($($i:$ty),*) {
-            #[cfg(miri)]
-            return;
+        #[inline(never)]
+        fn precondition_check($($i:$ty),*) {
             if !$e {
-                // don't unwind to reduce impact on code size
                 ::core::panicking::panic_nounwind(
                     concat!("unsafe precondition(s) violated: ", $name)
                 );
             }
         }
         #[allow(non_snake_case)]
         #[inline]
-        const fn comptime$(<$($tt)*>)?($(_:$ty),*) {}
+        const fn comptime($(_:$ty),*) {}
 
         if ::core::intrinsics::debug_assertions() {
-            ::core::intrinsics::const_eval_select(($($i,)*), comptime, runtime);
+            ::core::intrinsics::const_eval_select(($($i,)*), comptime, precondition_check);
         }
         }
     };
@@ -2648,8 +2642,8 @@ pub(crate) use assert_unsafe_precondition;
 /// Checks whether `ptr` is properly aligned with respect to
 /// `align_of::<T>()`.
 #[inline]
-pub(crate) fn is_aligned_and_not_null<T>(ptr: *const T) -> bool {
-    ((ptr.addr() & const { crate::mem::align_of::<T>() - 1 }) == 0) & (ptr.addr() != 0)
+pub(crate) fn is_aligned_and_not_null(ptr: *const (), align: usize) -> bool {
+    ptr.is_aligned_to(align) && !ptr.is_null()
 }
 
 /// Checks whether the regions of memory starting at `src` and `dst` of size
@@ -2658,9 +2652,12 @@ pub(crate) fn is_aligned_and_not_null<T>(ptr: *const T) -> bool {
 pub(crate) fn is_nonoverlapping<T>(src: *const T, dst: *const T, count: usize) -> bool {
     let src_usize = src.addr();
     let dst_usize = dst.addr();
-    let size = mem::size_of::<T>()
-        .checked_mul(count)
-        .expect("is_nonoverlapping: `size_of::<T>() * count` overflows a usize");
+    let Some(size) = mem::size_of::<T>().checked_mul(count) else {
+        // Use panic_nounwind instead of Option::expect, so that this function is nounwind.
+        crate::panicking::panic_nounwind(
+            "is_nonoverlapping: `size_of::<T>() * count` overflows a usize",
+        )
+    };
     let diff = src_usize.abs_diff(dst_usize);
     // If the absolute distance between the ptrs is at least as big as the size of the buffer,
     // they do not overlap.
@@ -2772,21 +2769,24 @@ pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: us
     // upheld by the caller.
     unsafe {
         if crate::intrinsics::debug_assertions() {
-            let args = (
-                src as *const (),
-                dst as *mut (),
-                crate::mem::size_of::<T>(),
-                crate::mem::align_of::<T>(),
-                count,
+            const_eval_select(
+                (
+                    src as *const (),
+                    dst as *mut (),
+                    crate::mem::size_of::<T>(),
+                    crate::mem::align_of::<T>(),
+                    count,
+                ),
+                compiletime,
+                check_aligned_nonoverlapping,
             );
-            const_eval_select(args, compiletime, check_valid_copy_nonoverlapping);
         }
         copy_nonoverlapping(src, dst, count)
     }
 }
 
 #[inline(never)]
-fn check_valid_copy_nonoverlapping(
+pub(crate) fn check_aligned_nonoverlapping(
     src: *const (),
     dst: *mut (),
     size: usize,
@@ -2805,9 +2805,11 @@ fn check_valid_copy_nonoverlapping(
     }
     let src_usize = src.addr();
     let dst_usize = dst.addr();
-    let size = size
-        .checked_mul(count)
-        .expect("is_nonoverlapping: `size_of::<T>() * count` overflows a usize");
+    let Some(size) = size.checked_mul(count) else {
+        crate::panicking::panic_nounwind(
+            "is_nonoverlapping: `size_of::<T>() * count` overflows a usize",
+        )
+    };
     let diff = src_usize.abs_diff(dst_usize);
     // If the absolute distance between the ptrs is at least as big as the size of the buffer,
     // they do not overlap.
@@ -2907,14 +2909,18 @@ pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) {
     // SAFETY: the safety contract for `copy` must be upheld by the caller.
     unsafe {
         if crate::intrinsics::debug_assertions() {
-            let args = (src as *const (), dst as *mut (), crate::mem::align_of::<T>());
-            const_eval_select(args, compiletime, check_valid_copy);
+            const_eval_select(
+                (src as *const (), dst as *mut (), align_of::<T>()),
+                compiletime,
+                check_valid_copy,
+            );
         }
         copy(src, dst, count)
     }
 }
 
 #[inline(never)]
+#[rustc_nounwind]
 fn check_valid_copy(src: *const (), dst: *mut (), align: usize) {
     if src.is_null() || dst.is_null() {
         crate::panicking::panic_nounwind(
@@ -2991,11 +2997,13 @@ pub const unsafe fn write_bytes<T>(dst: *mut T, val: u8, count: usize) {
         fn write_bytes<T>(dst: *mut T, val: u8, count: usize);
     }
 
+    let addr = dst as *const ();
+    let align = align_of::<T>();
     // SAFETY: the safety contract for `write_bytes` must be upheld by the caller.
     unsafe {
         assert_unsafe_precondition!(
             "ptr::write_bytes requires that the destination pointer is aligned and non-null",
-            [T](dst: *mut T) => is_aligned_and_not_null(dst)
+            (addr: *const (), align: usize) => is_aligned_and_not_null(addr, align)
         );
         write_bytes(dst, val, count)
     }

library/core/src/option.rs

@@ -1033,7 +1033,6 @@ impl<T> Option<T> {
     #[stable(feature = "option_result_unwrap_unchecked", since = "1.58.0")]
     #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")]
     pub const unsafe fn unwrap_unchecked(self) -> T {
-        debug_assert!(self.is_some());
         match self {
             Some(val) => val,
             // SAFETY: the safety contract must be upheld by the caller.

library/core/src/ptr/const_ptr.rs

@@ -806,13 +806,14 @@ impl<T: ?Sized> *const T {
     where
         T: Sized,
     {
-        let this = self;
+        let this = self as *const ();
+        let arg = origin as *const ();
         // SAFETY: The comparison has no side-effects, and the intrinsic
         // does this check internally in the CTFE implementation.
         unsafe {
             assert_unsafe_precondition!(
-                "ptr::sub_ptr requires `this >= origin`",
-                [T](this: *const T, origin: *const T) => this >= origin
+                "ptr::sub_ptr requires `self >= origin`",
+                (this: *const (), arg: *const ()) => this >= arg
             )
         };
 

library/core/src/ptr/mod.rs

@@ -381,11 +381,12 @@ use crate::cmp::Ordering;
 use crate::fmt;
 use crate::hash;
 use crate::intrinsics::{
-    self, assert_unsafe_precondition, is_aligned_and_not_null, is_nonoverlapping,
+    self, assert_unsafe_precondition, check_aligned_nonoverlapping, const_eval_select,
+    is_aligned_and_not_null,
 };
 use crate::marker::FnPtr;
 
-use crate::mem::{self, MaybeUninit};
+use crate::mem::{self, align_of, size_of, MaybeUninit};
 
 mod alignment;
 #[unstable(feature = "ptr_alignment_type", issue = "102070")]
@@ -961,17 +962,15 @@ pub const unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) {
         };
     }
 
+    const fn compiletime(_: *const (), _: *mut (), _: usize, _: usize, _: usize) {}
+
     // SAFETY: the caller must guarantee that `x` and `y` are
     // valid for writes and properly aligned.
     unsafe {
-        assert_unsafe_precondition!(
-            "ptr::swap_nonoverlapping requires that both pointer arguments are aligned and non-null \
-            and the specified memory ranges do not overlap",
-            [T](x: *mut T, y: *mut T, count: usize) =>
-            is_aligned_and_not_null(x)
-                && is_aligned_and_not_null(y)
-                && is_nonoverlapping(x, y, count)
-        );
+        if crate::intrinsics::debug_assertions() {
+            let args = (x as *const (), y as *mut (), size_of::<T>(), align_of::<T>(), count);
+            const_eval_select(args, compiletime, check_aligned_nonoverlapping);
+        }
     }
 
     // Split up the slice into small power-of-two-sized chunks that LLVM is able
@@ -1059,9 +1058,11 @@ pub const unsafe fn replace<T>(dst: *mut T, mut src: T) -> T {
     // and cannot overlap `src` since `dst` must point to a distinct
     // allocated object.
     unsafe {
+        let addr = dst as *const ();
+        let align = align_of::<T>();
         assert_unsafe_precondition!(
             "ptr::replace requires that the pointer argument is aligned and non-null",
-            [T](dst: *mut T) => is_aligned_and_not_null(dst)
+            (addr: *const (), align: usize) => is_aligned_and_not_null(addr, align)
         );
         mem::swap(&mut *dst, &mut src); // cannot overlap
     }
@@ -1580,9 +1581,11 @@ pub const unsafe fn write_unaligned<T>(dst: *mut T, src: T) {
 pub unsafe fn read_volatile<T>(src: *const T) -> T {
     // SAFETY: the caller must uphold the safety contract for `volatile_load`.
     unsafe {
+        let addr = src as *const ();
+        let align = align_of::<T>();
         assert_unsafe_precondition!(
             "ptr::read_volatile requires that the pointer argument is aligned and non-null",
-            [T](src: *const T) => is_aligned_and_not_null(src)
+            (addr: *const (), align: usize) => is_aligned_and_not_null(addr, align)
         );
         intrinsics::volatile_load(src)
     }
@@ -1655,9 +1658,11 @@ pub unsafe fn read_volatile<T>(src: *const T) -> T {
 pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
     // SAFETY: the caller must uphold the safety contract for `volatile_store`.
     unsafe {
+        let addr = dst as *mut ();
+        let align = crate::mem::align_of::<T>();
         assert_unsafe_precondition!(
             "ptr::write_volatile requires that the pointer argument is aligned and non-null",
-            [T](dst: *mut T) => is_aligned_and_not_null(dst)
+            (addr: *mut (), align: usize) => is_aligned_and_not_null(addr, align)
         );
         intrinsics::volatile_store(dst, src);
     }

library/core/src/slice/raw.rs

@@ -94,9 +94,11 @@ pub const unsafe fn from_raw_parts<'a, T>(data: *const T, len: usize) -> &'a [T]
     // SAFETY: the caller must uphold the safety contract for `from_raw_parts`.
     unsafe {
         if crate::intrinsics::debug_assertions() {
-            let args =
-                (data as *mut (), len, crate::mem::size_of::<T>(), crate::mem::align_of::<T>());
-            crate::intrinsics::const_eval_select(args, compiletime, check_valid_slice);
+            crate::intrinsics::const_eval_select(
+                (data as *mut (), len, crate::mem::size_of::<T>(), crate::mem::align_of::<T>()),
+                compiletime,
+                check_valid_slice,
+            );
         }
         &*ptr::slice_from_raw_parts(data, len)
     }
@@ -143,9 +145,11 @@ pub const unsafe fn from_raw_parts_mut<'a, T>(data: *mut T, len: usize) -> &'a m
     // SAFETY: the caller must uphold the safety contract for `from_raw_parts_mut`.
     unsafe {
         if crate::intrinsics::debug_assertions() {
-            let args =
-                (data as *mut (), len, crate::mem::size_of::<T>(), crate::mem::align_of::<T>());
-            crate::intrinsics::const_eval_select(args, compiletime, check_valid_slice);
+            crate::intrinsics::const_eval_select(
+                (data as *mut (), len, crate::mem::size_of::<T>(), crate::mem::align_of::<T>()),
+                compiletime,
+                check_valid_slice,
+            );
         }
         &mut *ptr::slice_from_raw_parts_mut(data, len)
     }

tests/assembly/is_aligned.rs

@@ -2,8 +2,8 @@
 // only-x86_64
 // ignore-sgx
 // revisions: opt-speed opt-size
-// [opt-speed] compile-flags: -Copt-level=1
-// [opt-size] compile-flags: -Copt-level=s
+// [opt-speed] compile-flags: -Copt-level=2 -Cdebug-assertions=no
+// [opt-size] compile-flags: -Copt-level=s -Cdebug-assertions=no
 #![crate_type="rlib"]
 
 #![feature(core_intrinsics)]

tests/mir-opt/inline/unwrap_unchecked.rs

@@ -2,7 +2,7 @@
 
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // ignore-debug: the debug assertions prevent the inlining we are testing for
-// compile-flags: -Zmir-opt-level=2 -Zinline-mir
+// compile-flags: -Zmir-opt-level=2 -Zinline-mir -Cdebug-assertions=no
 
 // EMIT_MIR unwrap_unchecked.unwrap_unchecked.Inline.diff
 // EMIT_MIR unwrap_unchecked.unwrap_unchecked.PreCodegen.after.mir