Optimize no-wrap niche discriminant cases

Author: scottmcm

Diff for: compiler/rustc_abi/src/lib.rs

@@ -43,7 +43,7 @@ use std::fmt;
 #[cfg(feature = "nightly")]
 use std::iter::Step;
 use std::num::{NonZeroUsize, ParseIntError};
-use std::ops::{Add, AddAssign, Mul, RangeInclusive, Sub};
+use std::ops::{Add, AddAssign, Mul, RangeFull, RangeInclusive, Sub};
 use std::str::FromStr;
 
 use bitflags::bitflags;
@@ -1162,12 +1162,45 @@ impl WrappingRange {
     }
 
     /// Returns `true` if `size` completely fills the range.
+    ///
+    /// Note that this is *not* the same as `self == WrappingRange::full(size)`.
+    /// Niche calculations can produce full ranges which are not the canonical one;
+    /// for example `Option<NonZero<u16>>` gets `valid_range: (..=0) | (1..)`.
     #[inline]
     fn is_full_for(&self, size: Size) -> bool {
         let max_value = size.unsigned_int_max();
         debug_assert!(self.start <= max_value && self.end <= max_value);
         self.start == (self.end.wrapping_add(1) & max_value)
     }
+
+    /// Checks whether this range is considered non-wrapping when the values are
+    /// interpreted as *unsigned* numbers of width `size`.
+    ///
+    /// Returns `Ok(true)` if there's no wrap-around, `Ok(false)` if there is,
+    /// and `Err(..)` if the range is full so it depends how you think about it.
+    #[inline]
+    pub fn no_unsigned_wraparound(&self, size: Size) -> Result<bool, RangeFull> {
+        if self.is_full_for(size) { Err(..) } else { Ok(self.start <= self.end) }
+    }
+
+    /// Checks whether this range is considered non-wrapping when the values are
+    /// interpreted as *signed* numbers of width `size`.
+    ///
+    /// This is heavily dependent on the `size`, as `100..=200` does wrap when
+    /// interpreted as `i8`, but doesn't when interpreted as `i16`.
+    ///
+    /// Returns `Ok(true)` if there's no wrap-around, `Ok(false)` if there is,
+    /// and `Err(..)` if the range is full so it depends how you think about it.
+    #[inline]
+    pub fn no_signed_wraparound(&self, size: Size) -> Result<bool, RangeFull> {
+        if self.is_full_for(size) {
+            Err(..)
+        } else {
+            let start: i128 = size.sign_extend(self.start);
+            let end: i128 = size.sign_extend(self.end);
+            Ok(start <= end)
+        }
+    }
 }
 
 impl fmt::Debug for WrappingRange {

Diff for: compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -3,7 +3,9 @@ use std::fmt;
 use arrayvec::ArrayVec;
 use either::Either;
 use rustc_abi as abi;
-use rustc_abi::{Align, BackendRepr, FIRST_VARIANT, Primitive, Size, TagEncoding, Variants};
+use rustc_abi::{
+    Align, BackendRepr, FIRST_VARIANT, Primitive, Size, TagEncoding, VariantIdx, Variants,
+};
 use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range};
 use rustc_middle::mir::{self, ConstValue};
 use rustc_middle::ty::Ty;
@@ -510,6 +512,8 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 );
 
                 let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
+                let tag_range = tag_scalar.valid_range(&dl);
+                let tag_size = tag_scalar.size(&dl);
 
                 // We have a subrange `niche_start..=niche_end` inside `range`.
                 // If the value of the tag is inside this subrange, it's a
@@ -525,53 +529,149 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 //     untagged_variant
                 // }
                 // However, we will likely be able to emit simpler code.
-                let (is_niche, tagged_discr, delta) = if relative_max == 0 {
-                    // Best case scenario: only one tagged variant. This will
-                    // likely become just a comparison and a jump.
-                    // The algorithm is:
-                    // is_niche = tag == niche_start
-                    // discr = if is_niche {
-                    //     niche_start
-                    // } else {
-                    //     untagged_variant
-                    // }
+
+                // First, the incredibly-common case of a two-variant enum (like
+                // `Option` or `Result`) where we only need one check.
+                if relative_max == 0 {
                     let niche_start = bx.cx().const_uint_big(tag_llty, niche_start);
-                    let is_niche = bx.icmp(IntPredicate::IntEQ, tag, niche_start);
-                    let tagged_discr =
-                        bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64);
-                    (is_niche, tagged_discr, 0)
-                } else {
-                    // The special cases don't apply, so we'll have to go with
-                    // the general algorithm.
-                    let relative_discr = bx.sub(tag, bx.cx().const_uint_big(tag_llty, niche_start));
-                    let cast_tag = bx.intcast(relative_discr, cast_to, false);
-                    let is_niche = bx.icmp(
-                        IntPredicate::IntULE,
-                        relative_discr,
-                        bx.cx().const_uint(tag_llty, relative_max as u64),
+                    let is_natural = bx.icmp(IntPredicate::IntNE, tag, niche_start);
+                    return if untagged_variant == VariantIdx::from_u32(1)
+                        && *niche_variants.start() == VariantIdx::from_u32(0)
+                    {
+                        // The polarity of the comparison above is picked so we can
+                        // just extend for `Option<T>`, which has these variants.
+                        bx.zext(is_natural, cast_to)
+                    } else {
+                        let tagged_discr =
+                            bx.cx().const_uint(cast_to, u64::from(niche_variants.start().as_u32()));
+                        let untagged_discr =
+                            bx.cx().const_uint(cast_to, u64::from(untagged_variant.as_u32()));
+                        bx.select(is_natural, untagged_discr, tagged_discr)
+                    };
+                }
+
+                let niche_end =
+                    tag_size.truncate(u128::from(relative_max).wrapping_add(niche_start));
+
+                // Next, the layout algorithm prefers to put the niches at one end,
+                // so look for cases where we don't need to calculate a relative_tag
+                // at all and can just look at the original tag value directly.
+                // This also lets us move any possibly-wrapping addition to the end
+                // where it's easiest to get rid of in the normal uses: it's easy
+                // to optimize `COMPLICATED + 7 == 2` to `COMPLICATED == (2 - 7)`.
+                {
+                    // Work in whichever size is wider, so we don't need to worry
+                    // about wrapping when computing `wide_niche_untagged`.
+                    let (wide_size, wide_ibty) = if cast_to_layout.size > tag_size {
+                        (cast_to_layout.size, cast_to)
+                    } else {
+                        (tag_size, tag_llty)
+                    };
+
+                    let wide_niche_to_variant =
+                        u128::from(niche_variants.start().as_u32()).wrapping_sub(niche_start);
+                    let wide_niche_untagged = wide_size.truncate(
+                        u128::from(untagged_variant.as_u32()).wrapping_sub(wide_niche_to_variant),
                     );
 
-                    // Thanks to parameter attributes and load metadata, LLVM already knows
-                    // the general valid range of the tag. It's possible, though, for there
-                    // to be an impossible value *in the middle*, which those ranges don't
-                    // communicate, so it's worth an `assume` to let the optimizer know.
-                    if niche_variants.contains(&untagged_variant)
-                        && bx.cx().sess().opts.optimize != OptLevel::No
+                    let wide_tag_and_is_niche_and_needs_assume =
+                        if tag_range.no_unsigned_wraparound(tag_size) == Ok(true) {
+                            let wide_tag = bx.zext(tag, wide_ibty);
+                            Some(if tag_range.start == niche_start {
+                                let end = bx.cx().const_uint_big(tag_llty, niche_end);
+                                (
+                                    wide_tag,
+                                    bx.icmp(IntPredicate::IntULE, tag, end),
+                                    wide_niche_untagged <= niche_end,
+                                )
+                            } else if tag_range.end == niche_end {
+                                let start = bx.cx().const_uint_big(tag_llty, niche_start);
+                                (
+                                    wide_tag,
+                                    bx.icmp(IntPredicate::IntUGE, tag, start),
+                                    wide_niche_untagged >= niche_start,
+                                )
+                            } else {
+                                bug!()
+                            })
+                        } else if tag_range.no_signed_wraparound(tag_size) == Ok(true) {
+                            let wide_tag = bx.sext(tag, wide_ibty);
+                            let wide_niche_untagged = wide_size.sign_extend(wide_niche_untagged);
+                            Some(if tag_range.start == niche_start {
+                                let end = bx.cx().const_uint_big(tag_llty, niche_end);
+                                let niche_end = wide_size.sign_extend(niche_end);
+                                (
+                                    wide_tag,
+                                    bx.icmp(IntPredicate::IntSLE, tag, end),
+                                    wide_niche_untagged <= niche_end,
+                                )
+                            } else if tag_range.end == niche_end {
+                                let start = bx.cx().const_uint_big(tag_llty, niche_start);
+                                let niche_start = wide_size.sign_extend(niche_start);
+                                (
+                                    wide_tag,
+                                    bx.icmp(IntPredicate::IntSGE, tag, start),
+                                    wide_niche_untagged >= niche_start,
+                                )
+                            } else {
+                                bug!()
+                            })
+                        } else {
+                            None
+                        };
+                    if let Some((wide_tag, is_niche, needs_assume)) =
+                        wide_tag_and_is_niche_and_needs_assume
                     {
-                        let impossible =
-                            u64::from(untagged_variant.as_u32() - niche_variants.start().as_u32());
-                        let impossible = bx.cx().const_uint(tag_llty, impossible);
-                        let ne = bx.icmp(IntPredicate::IntNE, relative_discr, impossible);
-                        bx.assume(ne);
+                        let wide_niche_untagged =
+                            bx.cx().const_uint_big(wide_ibty, wide_niche_untagged);
+                        if needs_assume && bx.cx().sess().opts.optimize != OptLevel::No {
+                            let not_untagged =
+                                bx.icmp(IntPredicate::IntNE, wide_tag, wide_niche_untagged);
+                            bx.assume(not_untagged);
+                        }
+
+                        let wide_niche = bx.select(is_niche, wide_tag, wide_niche_untagged);
+                        let cast_niche = bx.trunc(wide_niche, cast_to);
+                        let discr = if wide_niche_to_variant == 0 {
+                            cast_niche
+                        } else {
+                            let niche_to_variant =
+                                bx.cx().const_uint_big(cast_to, wide_niche_to_variant);
+                            bx.add(cast_niche, niche_to_variant)
+                        };
+                        return discr;
                     }
+                }
 
-                    (is_niche, cast_tag, niche_variants.start().as_u32() as u128)
-                };
+                // Otherwise the special cases don't apply,
+                // so we'll have to go with the general algorithm.
+                let relative_tag = bx.sub(tag, bx.cx().const_uint_big(tag_llty, niche_start));
+                let relative_discr = bx.intcast(relative_tag, cast_to, false);
+                let is_niche = bx.icmp(
+                    IntPredicate::IntULE,
+                    relative_tag,
+                    bx.cx().const_uint(tag_llty, u64::from(relative_max)),
+                );
+
+                // Thanks to parameter attributes and load metadata, LLVM already knows
+                // the general valid range of the tag. It's possible, though, for there
+                // to be an impossible value *in the middle*, which those ranges don't
+                // communicate, so it's worth an `assume` to let the optimizer know.
+                if niche_variants.contains(&untagged_variant)
+                    && bx.cx().sess().opts.optimize != OptLevel::No
+                {
+                    let impossible =
+                        u64::from(untagged_variant.as_u32() - niche_variants.start().as_u32());
+                    let impossible = bx.cx().const_uint(tag_llty, impossible);
+                    let ne = bx.icmp(IntPredicate::IntNE, relative_tag, impossible);
+                    bx.assume(ne);
+                }
 
+                let delta = niche_variants.start().as_u32();
                 let tagged_discr = if delta == 0 {
-                    tagged_discr
+                    relative_discr
                 } else {
-                    bx.add(tagged_discr, bx.cx().const_uint_big(cast_to, delta))
+                    bx.add(relative_discr, bx.cx().const_uint(cast_to, u64::from(delta)))
                 };
 
                 let discr = bx.select(

Diff for: tests/codegen/enum/enum-discriminant-assume.rs

@@ -0,0 +1,212 @@
+//@ compile-flags: -Copt-level=1 -C no-prepopulate-passes
+//@ only-64bit (because these discriminants are isize)
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+// Depending on the relative ordering of the variants, either
+//
+// 1. We want to `llvm.assume` to make it clear that the side with the niched
+//    tags can't actually have a value corresponding to the untagged one, or
+//
+// 2. The untagged variant would actually be on the side with the values,
+//    where it's critical that we *don't* assume since that could be one of
+//    the natural values, and thus we'd introduce UB.
+//
+// so these tests are particularly about *not* having assumes in the latter case.
+
+// See also `enum-discriminant-eq.rs`, which has payload-in-the-middle tests.
+// (That's not actually different in how it's detected during codegen compared
+//  to the cases here, but it's more relevant to how tests get optimized.)
+
+use std::cmp::Ordering;
+use std::intrinsics::discriminant_value;
+
+pub enum PayloadFirst<T> {
+    Payload(T),
+    After1,
+    After2,
+}
+
+pub enum PayloadLast<T> {
+    Before1,
+    Before2,
+    Payload(T),
+}
+
+// For a bool payload, the niches are 2 and 3.
+// - with the payload first, the payload variant equivalent is 1, which is a valid value.
+// - with the payload last, the payload variant equivalent is 4, which we assume away.
+
+#[unsafe(no_mangle)]
+pub fn payload_first_bool(a: PayloadFirst<bool>) -> isize {
+    // CHECK-LABEL: @payload_first_bool(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp uge i8 %a, 2
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 1
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -1
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_last_bool(a: PayloadLast<bool>) -> isize {
+    // CHECK-LABEL: @payload_last_bool(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp uge i8 %a, 2
+    // CHECK-NEXT: %[[ASSUME:.+]] = icmp ne i64 %0, 4
+    // CHECK-NEXT: call void @llvm.assume(i1 %[[ASSUME]])
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 4
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -2
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+// For a 7/8/9 payload, niches are 5 and 6, *before* the payload values.
+// - with the payload first, the payload variant equivalent is 4, which we assume away.
+// - with the payload last, the payload variant equivalent is 7, which is a valid value.
+
+pub enum SevenEightNine {
+    Seven = 7,
+    Eight = 8,
+    Nine = 9,
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_first_789(a: PayloadFirst<SevenEightNine>) -> isize {
+    // CHECK-LABEL: @payload_first_789(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ule i8 %a, 6
+    // CHECK-NEXT: %[[ASSUME:.+]] = icmp ne i64 %0, 4
+    // CHECK-NEXT: call void @llvm.assume(i1 %[[ASSUME]])
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 4
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -4
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_last_789(a: PayloadLast<SevenEightNine>) -> isize {
+    // CHECK-LABEL: @payload_last_789(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ule i8 %a, 6
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 7
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -5
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+// For a 2/3/4 payload, the zero niche gets prioritized so the niches are 0 and 1.
+// - with the payload first, the payload variant equivalent wraps to isize::MAX,
+//   which is actually on the value side again, so we don't assume it.
+// - with the payload last, the payload variant equivalent is 2, which is a valid value.
+//   (It also happens to have the tag equal to the discriminant, no adjustment needed.)
+
+pub enum TwoThreeFour {
+    Two = 2,
+    Three = 3,
+    Four = 4,
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_first_234(a: PayloadFirst<TwoThreeFour>) -> isize {
+    // CHECK-LABEL: @payload_first_234(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ule i8 %a, 1
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 -1
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], 1
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_last_234(a: PayloadLast<TwoThreeFour>) -> isize {
+    // CHECK-LABEL: @payload_last_234(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ule i8 %a, 1
+    // CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 2
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+// For an Ordering payload, the niches are 2 and 3 -- like `bool` but signed.
+
+#[unsafe(no_mangle)]
+pub fn payload_first_ordering(a: PayloadFirst<Ordering>) -> isize {
+    // CHECK-LABEL: @payload_first_ordering(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = sext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp sge i8 %a, 2
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 1
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -1
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_last_ordering(a: PayloadLast<Ordering>) -> isize {
+    // CHECK-LABEL: @payload_last_ordering(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = sext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp sge i8 %a, 2
+    // CHECK-NEXT: %[[ASSUME:.+]] = icmp ne i64 %0, 4
+    // CHECK-NEXT: call void @llvm.assume(i1 %[[ASSUME]])
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 4
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -2
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+// For a 1/2/3 payload, it would be nice if the zero niche were prioritized
+// (particularly as that would let this test the 4th case), but layout actually
+// puts the niches after the payload here too, so the niches are 4 and 5.
+// - with the payload first, the payload variant equivalent is 3, which is a valid value.
+// - with the payload last, the payload variant equivalent is 6, which we assume away.
+
+pub enum OneTwoThree {
+    One = 1,
+    Two = 2,
+    Three = 3,
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_first_123(a: PayloadFirst<OneTwoThree>) -> isize {
+    // CHECK-LABEL: @payload_first_123(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp uge i8 %a, 4
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 3
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -3
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}
+
+#[unsafe(no_mangle)]
+pub fn payload_last_123(a: PayloadLast<OneTwoThree>) -> isize {
+    // CHECK-LABEL: @payload_last_123(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: %[[A_EXT:.+]] = zext i8 %a to i64
+    // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp uge i8 %a, 4
+    // CHECK-NEXT: %[[ASSUME:.+]] = icmp ne i64 %0, 6
+    // CHECK-NEXT: call void @llvm.assume(i1 %[[ASSUME]])
+    // CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[A_EXT]], i64 6
+    // CHECK-NEXT: %[[DISCR:.+]] = add i64 %[[ADJ_DISCR]], -4
+    // CHECK-NEXT: ret i64 %[[DISCR]]
+
+    discriminant_value(&a) as _
+}

Diff for: tests/codegen/enum/enum-discriminant-eq.rs

@@ -4,17 +4,26 @@
 // so make sure we emit that in some reasonable way.
 
 #![crate_type = "lib"]
-#![feature(core_intrinsics)]
 #![feature(ascii_char)]
+#![feature(core_intrinsics)]
+#![feature(repr128)]
 
 use std::ascii::Char as AC;
 use std::cmp::Ordering;
 use std::intrinsics::discriminant_value;
 use std::num::NonZero;
 
+// A type that's bigger than `isize`, unlike the usual cases that have small tags.
+#[repr(u128)]
+pub enum Giant {
+    Two = 2,
+    Three = 3,
+    Four = 4,
+}
+
 #[unsafe(no_mangle)]
-pub fn opt_bool_eq(a: Option<bool>, b: Option<bool>) -> bool {
-    // CHECK-LABEL: @opt_bool_eq(
+pub fn opt_bool_eq_discr(a: Option<bool>, b: Option<bool>) -> bool {
+    // CHECK-LABEL: @opt_bool_eq_discr(
     // CHECK: %[[A:.+]] = icmp ne i8 %a, 2
     // CHECK: %[[B:.+]] = icmp eq i8 %b, 2
     // CHECK: %[[R:.+]] = xor i1 %[[A]], %[[B]]
@@ -24,8 +33,8 @@ pub fn opt_bool_eq(a: Option<bool>, b: Option<bool>) -> bool {
 }
 
 #[unsafe(no_mangle)]
-pub fn opt_ord_eq(a: Option<Ordering>, b: Option<Ordering>) -> bool {
-    // CHECK-LABEL: @opt_ord_eq(
+pub fn opt_ord_eq_discr(a: Option<Ordering>, b: Option<Ordering>) -> bool {
+    // CHECK-LABEL: @opt_ord_eq_discr(
     // CHECK: %[[A:.+]] = icmp ne i8 %a, 2
     // CHECK: %[[B:.+]] = icmp eq i8 %b, 2
     // CHECK: %[[R:.+]] = xor i1 %[[A]], %[[B]]
@@ -35,8 +44,8 @@ pub fn opt_ord_eq(a: Option<Ordering>, b: Option<Ordering>) -> bool {
 }
 
 #[unsafe(no_mangle)]
-pub fn opt_nz32_eq(a: Option<NonZero<u32>>, b: Option<NonZero<u32>>) -> bool {
-    // CHECK-LABEL: @opt_nz32_eq(
+pub fn opt_nz32_eq_discr(a: Option<NonZero<u32>>, b: Option<NonZero<u32>>) -> bool {
+    // CHECK-LABEL: @opt_nz32_eq_discr(
     // CHECK: %[[A:.+]] = icmp ne i32 %a, 0
     // CHECK: %[[B:.+]] = icmp eq i32 %b, 0
     // CHECK: %[[R:.+]] = xor i1 %[[A]], %[[B]]
@@ -46,8 +55,8 @@ pub fn opt_nz32_eq(a: Option<NonZero<u32>>, b: Option<NonZero<u32>>) -> bool {
 }
 
 #[unsafe(no_mangle)]
-pub fn opt_ac_eq(a: Option<AC>, b: Option<AC>) -> bool {
-    // CHECK-LABEL: @opt_ac_eq(
+pub fn opt_ac_eq_discr(a: Option<AC>, b: Option<AC>) -> bool {
+    // CHECK-LABEL: @opt_ac_eq_discr(
     // CHECK: %[[A:.+]] = icmp ne i8 %a, -128
     // CHECK: %[[B:.+]] = icmp eq i8 %b, -128
     // CHECK: %[[R:.+]] = xor i1 %[[A]], %[[B]]
@@ -56,58 +65,153 @@ pub fn opt_ac_eq(a: Option<AC>, b: Option<AC>) -> bool {
     discriminant_value(&a) == discriminant_value(&b)
 }
 
+#[unsafe(no_mangle)]
+pub fn opt_giant_eq_discr(a: Option<Giant>, b: Option<Giant>) -> bool {
+    // CHECK-LABEL: @opt_giant_eq_discr(
+    // CHECK: %[[A:.+]] = icmp ne i128 %a, 1
+    // CHECK: %[[B:.+]] = icmp eq i128 %b, 1
+    // CHECK: %[[R:.+]] = xor i1 %[[A]], %[[B]]
+    // CHECK: ret i1 %[[R]]
+
+    discriminant_value(&a) == discriminant_value(&b)
+}
+
 pub enum Mid<T> {
     Before,
     Thing(T),
     After,
 }
 
 #[unsafe(no_mangle)]
-pub fn mid_bool_eq(a: Mid<bool>, b: Mid<bool>) -> bool {
-    // CHECK-LABEL: @mid_bool_eq(
-    // CHECK: %[[AS:.+]] = add nsw i8 %a, -2
-    // CHECK: %[[AT:.+]] = icmp ult i8 %[[AS]], 3
-    // CHECK: %[[AD:.+]] = select i1 %[[AT]], i8 %[[AS]], i8 1
-    // CHECK: %[[BS:.+]] = add nsw i8 %b, -2
-    // CHECK: %[[BT:.+]] = icmp ult i8 %[[BS]], 3
-    // CHECK: %[[BD:.+]] = select i1 %[[BT]], i8 %[[BS]], i8 1
-    // CHECK: %[[R:.+]] = icmp eq i8 %[[AD]], %[[BD]]
+pub fn mid_bool_eq_discr(a: Mid<bool>, b: Mid<bool>) -> bool {
+    // CHECK-LABEL: @mid_bool_eq_discr(
+
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp samesign ugt i8 %a, 1
+    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %a, 3
+    // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
+    // CHECK: %[[A_ADJ_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %a, i8 3
+
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp samesign ugt i8 %b, 1
+    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %b, 3
+    // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
+    // CHECK: %[[B_ADJ_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %b, i8 3
+
+    // CHECK: %[[R:.+]] = icmp eq i8 %[[A_ADJ_DISCR]], %[[B_ADJ_DISCR]]
     // CHECK: ret i1 %[[R]]
     discriminant_value(&a) == discriminant_value(&b)
 }
 
 #[unsafe(no_mangle)]
-pub fn mid_ord_eq(a: Mid<Ordering>, b: Mid<Ordering>) -> bool {
-    // CHECK-LABEL: @mid_ord_eq(
-    // CHECK: %[[AS:.+]] = add nsw i8 %a, -2
-    // CHECK: %[[AT:.+]] = icmp ult i8 %[[AS]], 3
-    // CHECK: %[[AD:.+]] = select i1 %[[AT]], i8 %[[AS]], i8 1
-    // CHECK: %[[BS:.+]] = add nsw i8 %b, -2
-    // CHECK: %[[BT:.+]] = icmp ult i8 %[[BS]], 3
-    // CHECK: %[[BD:.+]] = select i1 %[[BT]], i8 %[[BS]], i8 1
-    // CHECK: %[[R:.+]] = icmp eq i8 %[[AD]], %[[BD]]
+pub fn mid_ord_eq_discr(a: Mid<Ordering>, b: Mid<Ordering>) -> bool {
+    // CHECK-LABEL: @mid_ord_eq_discr(
+
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp sgt i8 %a, 1
+    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %a, 3
+    // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
+    // CHECK: %[[A_ADJ_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %a, i8 3
+
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp sgt i8 %b, 1
+    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %b, 3
+    // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
+    // CHECK: %[[B_ADJ_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %b, i8 3
+
+    // CHECK: %[[R:.+]] = icmp eq i8 %[[A_ADJ_DISCR]], %[[B_ADJ_DISCR]]
     // CHECK: ret i1 %[[R]]
     discriminant_value(&a) == discriminant_value(&b)
 }
 
 #[unsafe(no_mangle)]
-pub fn mid_nz32_eq(a: Mid<NonZero<u32>>, b: Mid<NonZero<u32>>) -> bool {
-    // CHECK-LABEL: @mid_nz32_eq(
+pub fn mid_nz32_eq_discr(a: Mid<NonZero<u32>>, b: Mid<NonZero<u32>>) -> bool {
+    // CHECK-LABEL: @mid_nz32_eq_discr(
     // CHECK: %[[R:.+]] = icmp eq i32 %a.0, %b.0
     // CHECK: ret i1 %[[R]]
     discriminant_value(&a) == discriminant_value(&b)
 }
 
 #[unsafe(no_mangle)]
-pub fn mid_ac_eq(a: Mid<AC>, b: Mid<AC>) -> bool {
-    // CHECK-LABEL: @mid_ac_eq(
-    // CHECK: %[[AS:.+]] = xor i8 %a, -128
-    // CHECK: %[[AT:.+]] = icmp ult i8 %[[AS]], 3
-    // CHECK: %[[AD:.+]] = select i1 %[[AT]], i8 %[[AS]], i8 1
-    // CHECK: %[[BS:.+]] = xor i8 %b, -128
-    // CHECK: %[[BT:.+]] = icmp ult i8 %[[BS]], 3
-    // CHECK: %[[BD:.+]] = select i1 %[[BT]], i8 %[[BS]], i8 1
-    // CHECK: %[[R:.+]] = icmp eq i8 %[[AD]], %[[BD]]
+pub fn mid_ac_eq_discr(a: Mid<AC>, b: Mid<AC>) -> bool {
+    // CHECK-LABEL: @mid_ac_eq_discr(
+
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp slt i8 %a, 0
+    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %a, -127
+    // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
+    // CHECK: %[[A_ADJ_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %a, i8 -127
+
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp slt i8 %b, 0
+    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %b, -127
+    // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
+    // CHECK: %[[B_ADJ_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %b, i8 -127
+
+    // CHECK: %[[R:.+]] = icmp eq i8 %[[A_ADJ_DISCR]], %[[B_ADJ_DISCR]]
+    // CHECK: ret i1 %[[R]]
+    discriminant_value(&a) == discriminant_value(&b)
+}
+
+// Note that we emit the add after the select, where LLVM really ought to optimize
+// it out because it's on both sides of the `icmp eq`, but InstCombine sinks it
+// inside the `select`.  See <https://github.com/llvm/llvm-project/issues/134024>
+#[unsafe(no_mangle)]
+pub fn mid_giant_eq_discr(a: Mid<Giant>, b: Mid<Giant>) -> bool {
+    // CHECK-LABEL: @mid_giant_eq_discr(
+
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp samesign ugt i128 %a, 4
+    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i128 %a, 6
+    // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
+    // CHECK: %[[A_TRUNC:.+]] = trunc nuw nsw i128 %a to [[ISIZE:i32|i64]]
+    // CHECK: %[[A_NICHE_DISCR:.+]] = add nsw [[ISIZE]] %[[A_TRUNC]], -5
+    // CHECK: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], [[ISIZE]] %[[A_NICHE_DISCR]], [[ISIZE]] 1
+
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp samesign ugt i128 %b, 4
+    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i128 %b, 6
+    // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
+    // CHECK: %[[B_TRUNC:.+]] = trunc nuw nsw i128 %b to [[ISIZE]]
+    // CHECK: %[[B_NICHE_DISCR:.+]] = add nsw [[ISIZE]] %[[B_TRUNC]], -5
+    // CHECK: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], [[ISIZE]] %[[B_NICHE_DISCR]], [[ISIZE]] 1
+
+    // CHECK: %[[R:.+]] = icmp eq [[ISIZE]] %[[A_DISCR]], %[[B_DISCR]]
     // CHECK: ret i1 %[[R]]
     discriminant_value(&a) == discriminant_value(&b)
 }
+
+// In niche-encoded enums, testing for the untagged variant should optimize to a
+// straight-forward comparison looking for the natural range of the payload value.
+
+#[unsafe(no_mangle)]
+pub fn mid_bool_is_thing(a: Mid<bool>) -> bool {
+    // CHECK-LABEL: @mid_bool_is_thing(
+    // CHECK: %[[R:.+]] = icmp samesign ult i8 %a, 2
+    // CHECK: ret i1 %[[R]]
+    discriminant_value(&a) == 1
+}
+
+#[unsafe(no_mangle)]
+pub fn mid_ord_is_thing(a: Mid<Ordering>) -> bool {
+    // CHECK-LABEL: @mid_ord_is_thing(
+    // CHECK: %[[R:.+]] = icmp slt i8 %a, 2
+    // CHECK: ret i1 %[[R]]
+    discriminant_value(&a) == 1
+}
+
+#[unsafe(no_mangle)]
+pub fn mid_nz32_is_thing(a: Mid<NonZero<u32>>) -> bool {
+    // CHECK-LABEL: @mid_nz32_is_thing(
+    // CHECK: %[[R:.+]] = icmp eq i32 %a.0, 1
+    // CHECK: ret i1 %[[R]]
+    discriminant_value(&a) == 1
+}
+
+#[unsafe(no_mangle)]
+pub fn mid_ac_is_thing(a: Mid<AC>) -> bool {
+    // CHECK-LABEL: @mid_ac_is_thing(
+    // CHECK: %[[R:.+]] = icmp sgt i8 %a, -1
+    // CHECK: ret i1 %[[R]]
+    discriminant_value(&a) == 1
+}
+
+#[unsafe(no_mangle)]
+pub fn mid_giant_is_thing(a: Mid<Giant>) -> bool {
+    // CHECK-LABEL: @mid_giant_is_thing(
+    // CHECK: %[[R:.+]] = icmp samesign ult i128 %a, 5
+    // CHECK: ret i1 %[[R]]
+    discriminant_value(&a) == 1
+}

Diff for: tests/codegen/enum/enum-match.rs

@@ -39,12 +39,14 @@ pub enum Enum1 {
 
 // CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
-// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
-// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 2
-// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 1
-// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 0
-// CHECK-NEXT: switch i64 %[[DISCR]]
+// CHECK-NEXT: %[[ADJ_DISCR:.+]] = tail call i8 @llvm.umax.i8(i8 %0, i8 1)
+// CHECK-NEXT: switch i8 %[[ADJ_DISCR]], label %[[UNREACHABLE:.+]] [
+// CHECK-NEXT:   i8 1,
+// CHECK-NEXT:   i8 2,
+// CHECK-NEXT:   i8 3,
+// CHECK-NEXT: ]
+// CHECK: [[UNREACHABLE]]:
+// CHECK-NEXT: unreachable
 #[no_mangle]
 pub fn match1(e: Enum1) -> u8 {
     use Enum1::*;
@@ -147,12 +149,19 @@ pub enum MiddleNiche {
 
 // CHECK-LABEL: define noundef{{( range\(i8 -?[0-9]+, -?[0-9]+\))?}} i8 @match4(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
-// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 5
-// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp samesign ugt i8 %0, 1
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %0, 4
 // CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
-// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i8 %[[REL_VAR]], i8 2
-// CHECK-NEXT: switch i8 %[[DISCR]]
+// CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i8 %0, i8 4
+// CHECK-NEXT: switch i8 %[[ADJ_DISCR]], label %[[UNREACHABLE:.+]] [
+// CHECK-NEXT:   i8 2,
+// CHECK-NEXT:   i8 3,
+// CHECK-NEXT:   i8 4,
+// CHECK-NEXT:   i8 5,
+// CHECK-NEXT:   i8 6,
+// CHECK-NEXT: ]
+// CHECK: [[UNREACHABLE]]:
+// CHECK-NEXT: unreachable
 #[no_mangle]
 pub fn match4(e: MiddleNiche) -> u8 {
     use MiddleNiche::*;
@@ -167,9 +176,8 @@ pub fn match4(e: MiddleNiche) -> u8 {
 
 // CHECK-LABEL: define{{.+}}i1 @match4_is_c(i8{{.+}}%e)
 // CHECK-NEXT: start
-// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %e, -2
-// CHECK-NEXT: %[[NOT_NICHE:.+]] = icmp ugt i8 %[[REL_VAR]], 4
-// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2
+// CHECK-NEXT: %[[NOT_NICHE:.+]] = icmp samesign ult i8 %e, 2
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %e, 4
 // CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
 // CHECK-NEXT: ret i1 %[[NOT_NICHE]]
 #[no_mangle]
@@ -451,17 +459,17 @@ pub enum HugeVariantIndex {
 
 // CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match5(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
-// CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
-// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], 3
-// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 1
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp samesign ugt i8 %0, 1
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %0, 3
 // CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
-// CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 257
-// CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 258
-// CHECK-NEXT: switch i64 %[[DISCR]],
-// CHECK-NEXT:   i64 257,
-// CHECK-NEXT:   i64 258,
-// CHECK-NEXT:   i64 259,
+// CHECK-NEXT: %[[ADJ_DISCR:.+]] = select i1 %[[IS_NICHE]], i8 %0, i8 3
+// CHECK-NEXT: switch i8 %[[ADJ_DISCR]], label %[[UNREACHABLE:.+]] [
+// CHECK-NEXT:   i8 2,
+// CHECK-NEXT:   i8 3,
+// CHECK-NEXT:   i8 4,
+// CHECK-NEXT: ]
+// CHECK: [[UNREACHABLE]]:
+// CHECK-NEXT: unreachable
 #[no_mangle]
 pub fn match5(e: HugeVariantIndex) -> u8 {
     use HugeVariantIndex::*;
@@ -471,3 +479,283 @@ pub fn match5(e: HugeVariantIndex) -> u8 {
         Possible259 => 100,
     }
 }
+
+// Make an enum where the niche tags wrap both as signed and as unsigned, to hit
+// the most-fallback case where there's just nothing smart to do.
+
+pub enum E10Through65 {
+    D10 = 10,
+    D11 = 11,
+    D12 = 12,
+    D13 = 13,
+    D14 = 14,
+    D15 = 15,
+    D16 = 16,
+    D17 = 17,
+    D18 = 18,
+    D19 = 19,
+    D20 = 20,
+    D21 = 21,
+    D22 = 22,
+    D23 = 23,
+    D24 = 24,
+    D25 = 25,
+    D26 = 26,
+    D27 = 27,
+    D28 = 28,
+    D29 = 29,
+    D30 = 30,
+    D31 = 31,
+    D32 = 32,
+    D33 = 33,
+    D34 = 34,
+    D35 = 35,
+    D36 = 36,
+    D37 = 37,
+    D38 = 38,
+    D39 = 39,
+    D40 = 40,
+    D41 = 41,
+    D42 = 42,
+    D43 = 43,
+    D44 = 44,
+    D45 = 45,
+    D46 = 46,
+    D47 = 47,
+    D48 = 48,
+    D49 = 49,
+    D50 = 50,
+    D51 = 51,
+    D52 = 52,
+    D53 = 53,
+    D54 = 54,
+    D55 = 55,
+    D56 = 56,
+    D57 = 57,
+    D58 = 58,
+    D59 = 59,
+    D60 = 60,
+    D61 = 61,
+    D62 = 62,
+    D63 = 63,
+    D64 = 64,
+    D65 = 65,
+}
+
+pub enum Tricky {
+    Untagged(E10Through65),
+    V001,
+    V002,
+    V003,
+    V004,
+    V005,
+    V006,
+    V007,
+    V008,
+    V009,
+    V010,
+    V011,
+    V012,
+    V013,
+    V014,
+    V015,
+    V016,
+    V017,
+    V018,
+    V019,
+    V020,
+    V021,
+    V022,
+    V023,
+    V024,
+    V025,
+    V026,
+    V027,
+    V028,
+    V029,
+    V030,
+    V031,
+    V032,
+    V033,
+    V034,
+    V035,
+    V036,
+    V037,
+    V038,
+    V039,
+    V040,
+    V041,
+    V042,
+    V043,
+    V044,
+    V045,
+    V046,
+    V047,
+    V048,
+    V049,
+    V050,
+    V051,
+    V052,
+    V053,
+    V054,
+    V055,
+    V056,
+    V057,
+    V058,
+    V059,
+    V060,
+    V061,
+    V062,
+    V063,
+    V064,
+    V065,
+    V066,
+    V067,
+    V068,
+    V069,
+    V070,
+    V071,
+    V072,
+    V073,
+    V074,
+    V075,
+    V076,
+    V077,
+    V078,
+    V079,
+    V080,
+    V081,
+    V082,
+    V083,
+    V084,
+    V085,
+    V086,
+    V087,
+    V088,
+    V089,
+    V090,
+    V091,
+    V092,
+    V093,
+    V094,
+    V095,
+    V096,
+    V097,
+    V098,
+    V099,
+    V100,
+    V101,
+    V102,
+    V103,
+    V104,
+    V105,
+    V106,
+    V107,
+    V108,
+    V109,
+    V110,
+    V111,
+    V112,
+    V113,
+    V114,
+    V115,
+    V116,
+    V117,
+    V118,
+    V119,
+    V120,
+    V121,
+    V122,
+    V123,
+    V124,
+    V125,
+    V126,
+    V127,
+    V128,
+    V129,
+    V130,
+    V131,
+    V132,
+    V133,
+    V134,
+    V135,
+    V136,
+    V137,
+    V138,
+    V139,
+    V140,
+    V141,
+    V142,
+    V143,
+    V144,
+    V145,
+    V146,
+    V147,
+    V148,
+    V149,
+    V150,
+    V151,
+    V152,
+    V153,
+    V154,
+    V155,
+    V156,
+    V157,
+    V158,
+    V159,
+    V160,
+    V161,
+    V162,
+    V163,
+    V164,
+    V165,
+    V166,
+    V167,
+    V168,
+    V169,
+    V170,
+    V171,
+    V172,
+    V173,
+    V174,
+    V175,
+    V176,
+    V177,
+    V178,
+    V179,
+    V180,
+    V181,
+    V182,
+    V183,
+    V184,
+    V185,
+    V186,
+    V187,
+    V188,
+    V189,
+    V190,
+    V191,
+    V192,
+    V193,
+    V194,
+    V195,
+    V196,
+    V197,
+    V198,
+    V199,
+    V200,
+}
+
+const _: () = assert!(std::intrinsics::discriminant_value(&Tricky::V100) == 100);
+
+// CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @discriminant6(i8 noundef %e)
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[REL_VAR:.+]] = add i8 %e, -66
+// CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %0, -56
+// CHECK-NEXT: %[[TAGGED_DISCR:.+]] = add i8 %e, -65
+// CHECK-NEXT: %[[DISCR:.+]] = select i1 %1, i8 %2, i8 0
+// CHECK-NEXT: ret i8 %[[DISCR]]
+#[no_mangle]
+pub fn discriminant6(e: Tricky) -> u8 {
+    std::intrinsics::discriminant_value(&e) as _
+}

Diff for: tests/codegen/enum/enum-two-variants-match.rs

@@ -60,8 +60,8 @@ pub fn result_match(x: Result<u64, i64>) -> u16 {
 #[no_mangle]
 pub fn option_bool_match(x: Option<bool>) -> char {
     // CHECK: %[[RAW:.+]] = load i8, ptr %x
-    // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2
-    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[IS_SOME:.+]] = icmp ne i8 %[[RAW]], 2
+    // CHECK: %[[OPT_DISCR:.+]] = zext i1 %[[IS_SOME]] to i64
     // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
     // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
 
@@ -81,8 +81,8 @@ use std::cmp::Ordering::{self, *};
 #[no_mangle]
 pub fn option_ordering_match(x: Option<Ordering>) -> char {
     // CHECK: %[[RAW:.+]] = load i8, ptr %x
-    // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %[[RAW]], 2
-    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[IS_SOME:.+]] = icmp ne i8 %[[RAW]], 2
+    // CHECK: %[[OPT_DISCR:.+]] = zext i1 %[[IS_SOME]] to i64
     // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
     // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]
 
@@ -109,8 +109,8 @@ pub fn option_ordering_match(x: Option<Ordering>) -> char {
 pub fn option_nonzero_match(x: Option<std::num::NonZero<u16>>) -> u16 {
     // CHECK: %[[OUT:.+]] = alloca [2 x i8]
 
-    // CHECK: %[[IS_NONE:.+]] = icmp eq i16 %x, 0
-    // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1
+    // CHECK: %[[IS_SOME:.+]] = icmp ne i16 %x, 0
+    // CHECK: %[[OPT_DISCR:.+]] = zext i1 %[[IS_SOME]] to i64
     // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1
     // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]]