Add #[repr(no_niche)].

This repr-hint makes a struct/enum hide any niche within from its
surrounding type-construction context.

It is meant (at least initially) as an implementation detail for
resolving issue 68303. We will not stabilize the repr-hint unless
someone finds motivation for doing so.

(So, declaration of `no_niche` feature lives in section of file
where other internal implementation details are grouped, and
deliberately leaves out the tracking issue number.)

incorporated review feedback, and fixed post-rebase.

Author: pnkfelix

src/librustc/ty/layout.rs

@@ -356,12 +356,14 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
             debug!("univariant offset: {:?} field: {:#?}", offset, field);
             offsets[i as usize] = offset;
 
-            if let Some(mut niche) = field.largest_niche.clone() {
-                let available = niche.available(dl);
-                if available > largest_niche_available {
-                    largest_niche_available = available;
-                    niche.offset += offset;
-                    largest_niche = Some(niche);
+            if !repr.hide_niche() {
+                if let Some(mut niche) = field.largest_niche.clone() {
+                    let available = niche.available(dl);
+                    if available > largest_niche_available {
+                        largest_niche_available = available;
+                        niche.offset += offset;
+                        largest_niche = Some(niche);
+                    }
                 }
             }
 
@@ -838,7 +840,11 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                             }
 
                             // Update `largest_niche` if we have introduced a larger niche.
-                            let niche = Niche::from_scalar(dl, Size::ZERO, scalar.clone());
+                            let niche = if def.repr.hide_niche() {
+                                None
+                            } else {
+                                Niche::from_scalar(dl, Size::ZERO, scalar.clone())
+                            };
                             if let Some(niche) = niche {
                                 match &st.largest_niche {
                                     Some(largest_niche) => {
@@ -863,6 +869,11 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                     return Ok(tcx.intern_layout(st));
                 }
 
+                // At this point, we have handled all unions and
+                // structs. (We have also handled univariant enums
+                // that allow representation optimization.)
+                assert!(def.is_enum());
+
                 // The current code for niche-filling relies on variant indices
                 // instead of actual discriminants, so dataful enums with
                 // explicit discriminants (RFC #2363) would misbehave.

src/librustc/ty/mod.rs

@@ -2036,7 +2036,8 @@ bitflags! {
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
         const IS_LINEAR          = 1 << 3;
-
+        // If true, don't expose any niche to type's context.
+        const HIDE_NICHE         = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
         const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits |
                                    ReprFlags::IS_SIMD.bits |
@@ -2073,6 +2074,7 @@ impl ReprOptions {
                         ReprFlags::empty()
                     }
                     attr::ReprTransparent => ReprFlags::IS_TRANSPARENT,
+                    attr::ReprNoNiche => ReprFlags::HIDE_NICHE,
                     attr::ReprSimd => ReprFlags::IS_SIMD,
                     attr::ReprInt(i) => {
                         size = Some(i);
@@ -2113,6 +2115,10 @@ impl ReprOptions {
     pub fn linear(&self) -> bool {
         self.flags.contains(ReprFlags::IS_LINEAR)
     }
+    #[inline]
+    pub fn hide_niche(&self) -> bool {
+        self.flags.contains(ReprFlags::HIDE_NICHE)
+    }
 
     pub fn discr_type(&self) -> attr::IntType {
         self.int.unwrap_or(attr::SignedInt(ast::IntTy::Isize))

src/librustc_builtin_macros/deriving/generic/mod.rs

@@ -824,7 +824,8 @@ fn find_repr_type_name(sess: &ParseSess, type_attrs: &[ast::Attribute]) -> &'sta
                 attr::ReprPacked(_)
                 | attr::ReprSimd
                 | attr::ReprAlign(_)
-                | attr::ReprTransparent => continue,
+                | attr::ReprTransparent
+                | attr::ReprNoNiche => continue,
 
                 attr::ReprC => "i32",
 

src/librustc_feature/active.rs

@@ -204,6 +204,10 @@ declare_features! (
     /// Added for testing E0705; perma-unstable.
     (active, test_2018_feature, "1.31.0", None, Some(Edition::Edition2018)),
 
+    /// Allows `#[repr(no_niche)]` (an implementation detail of `rustc`,
+    /// it is not on path for eventual stabilization).
+    (active, no_niche, "1.42.0", None, None),
+
     // no-tracking-issue-end
 
     // -------------------------------------------------------------------------

src/librustc_passes/check_attr.rs

@@ -16,9 +16,10 @@ use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_hir::DUMMY_HIR_ID;
 use rustc_hir::{self, HirId, Item, ItemKind, TraitItem};
 use rustc_session::lint::builtin::{CONFLICTING_REPR_HINTS, UNUSED_ATTRIBUTES};
+use rustc_session::parse::feature_err;
 use rustc_span::symbol::sym;
 use rustc_span::Span;
-use syntax::ast::Attribute;
+use syntax::ast::{Attribute, NestedMetaItem};
 use syntax::attr;
 
 fn target_from_impl_item<'tcx>(tcx: TyCtxt<'tcx>, impl_item: &hir::ImplItem<'_>) -> Target {
@@ -287,6 +288,21 @@ impl CheckAttrVisitor<'tcx> {
                         _ => ("a", "struct, enum, or union"),
                     }
                 }
+                sym::no_niche => {
+                    if !self.tcx.features().enabled(sym::no_niche) {
+                        feature_err(
+                            &self.tcx.sess.parse_sess,
+                            sym::no_niche,
+                            hint.span(),
+                            "the attribute `repr(no_niche)` is currently unstable",
+                        )
+                        .emit();
+                    }
+                    match target {
+                        Target::Struct | Target::Enum => continue,
+                        _ => ("a", "struct or enum"),
+                    }
+                }
                 sym::i8
                 | sym::u8
                 | sym::i16
@@ -314,8 +330,10 @@ impl CheckAttrVisitor<'tcx> {
         // This is not ideal, but tracking precisely which ones are at fault is a huge hassle.
         let hint_spans = hints.iter().map(|hint| hint.span());
 
-        // Error on repr(transparent, <anything else>).
-        if is_transparent && hints.len() > 1 {
+        // Error on repr(transparent, <anything else apart from no_niche>).
+        let non_no_niche = |hint: &&NestedMetaItem| hint.name_or_empty() != sym::no_niche;
+        let non_no_niche_count = hints.iter().filter(non_no_niche).count();
+        if is_transparent && non_no_niche_count > 1 {
             let hint_spans: Vec<_> = hint_spans.clone().collect();
             struct_span_err!(
                 self.tcx.sess,

src/librustc_span/symbol.rs

@@ -487,6 +487,7 @@ symbols! {
         None,
         non_exhaustive,
         non_modrs_mods,
+        no_niche,
         no_stack_check,
         no_start,
         no_std,
@@ -583,6 +584,7 @@ symbols! {
         repr128,
         repr_align,
         repr_align_enum,
+        repr_no_niche,
         repr_packed,
         repr_simd,
         repr_transparent,

src/libsyntax/attr/builtin.rs

@@ -820,6 +820,7 @@ pub enum ReprAttr {
     ReprSimd,
     ReprTransparent,
     ReprAlign(u32),
+    ReprNoNiche,
 }
 
 #[derive(Eq, PartialEq, Debug, RustcEncodable, RustcDecodable, Copy, Clone, HashStable_Generic)]
@@ -875,6 +876,7 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> {
                         sym::packed => Some(ReprPacked(1)),
                         sym::simd => Some(ReprSimd),
                         sym::transparent => Some(ReprTransparent),
+                        sym::no_niche => Some(ReprNoNiche),
                         name => int_type_of_word(name).map(ReprInt),
                     };