Skip to content

[perf] cache type info for ParamEnv #123058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 7, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions compiler/rustc_hir_analysis/src/collect/item_bounds.rs
Original file line number Diff line number Diff line change
@@ -165,10 +165,7 @@ pub(super) fn explicit_item_bounds_with_filter(
ty::EarlyBinder::bind(bounds)
}

pub(super) fn item_bounds(
tcx: TyCtxt<'_>,
def_id: DefId,
) -> ty::EarlyBinder<&'_ ty::List<ty::Clause<'_>>> {
pub(super) fn item_bounds(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<ty::Clauses<'_>> {
tcx.explicit_item_bounds(def_id).map_bound(|bounds| {
tcx.mk_clauses_from_iter(util::elaborate(tcx, bounds.iter().map(|&(bound, _span)| bound)))
})
@@ -177,7 +174,7 @@ pub(super) fn item_bounds(
pub(super) fn item_super_predicates(
tcx: TyCtxt<'_>,
def_id: DefId,
) -> ty::EarlyBinder<&'_ ty::List<ty::Clause<'_>>> {
) -> ty::EarlyBinder<ty::Clauses<'_>> {
tcx.explicit_item_super_predicates(def_id).map_bound(|bounds| {
tcx.mk_clauses_from_iter(
util::elaborate(tcx, bounds.iter().map(|&(bound, _span)| bound)).filter_only_self(),
@@ -188,12 +185,12 @@ pub(super) fn item_super_predicates(
pub(super) fn item_non_self_assumptions(
tcx: TyCtxt<'_>,
def_id: DefId,
) -> ty::EarlyBinder<&'_ ty::List<ty::Clause<'_>>> {
) -> ty::EarlyBinder<ty::Clauses<'_>> {
let all_bounds: FxIndexSet<_> = tcx.item_bounds(def_id).skip_binder().iter().collect();
let own_bounds: FxIndexSet<_> =
tcx.item_super_predicates(def_id).skip_binder().iter().collect();
if all_bounds.len() == own_bounds.len() {
ty::EarlyBinder::bind(ty::List::empty())
ty::EarlyBinder::bind(ty::ListWithCachedTypeInfo::empty())
} else {
ty::EarlyBinder::bind(tcx.mk_clauses_from_iter(all_bounds.difference(&own_bounds).copied()))
}
4 changes: 4 additions & 0 deletions compiler/rustc_middle/src/query/erase.rs
Original file line number Diff line number Diff line change
@@ -67,6 +67,10 @@ impl<T> EraseType for &'_ ty::List<T> {
type Result = [u8; size_of::<&'static ty::List<()>>()];
}

impl<T> EraseType for &'_ ty::ListWithCachedTypeInfo<T> {
type Result = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()];
}

impl<I: rustc_index::Idx, T> EraseType for &'_ rustc_index::IndexSlice<I, T> {
type Result = [u8; size_of::<&'static rustc_index::IndexSlice<u32, ()>>()];
}
2 changes: 1 addition & 1 deletion compiler/rustc_middle/src/query/keys.rs
Original file line number Diff line number Diff line change
@@ -432,7 +432,7 @@ impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) {
}
}

impl<'tcx> Key for &'tcx ty::List<ty::Clause<'tcx>> {
impl<'tcx> Key for ty::Clauses<'tcx> {
type Cache<V> = DefaultCache<Self, V>;

fn default_span(&self, _: TyCtxt<'_>) -> Span {
8 changes: 4 additions & 4 deletions compiler/rustc_middle/src/query/mod.rs
Original file line number Diff line number Diff line change
@@ -398,15 +398,15 @@ rustc_queries! {
/// ```
///
/// Bounds from the parent (e.g. with nested impl trait) are not included.
query item_bounds(key: DefId) -> ty::EarlyBinder<&'tcx ty::List<ty::Clause<'tcx>>> {
query item_bounds(key: DefId) -> ty::EarlyBinder<ty::Clauses<'tcx>> {
desc { |tcx| "elaborating item bounds for `{}`", tcx.def_path_str(key) }
}

query item_super_predicates(key: DefId) -> ty::EarlyBinder<&'tcx ty::List<ty::Clause<'tcx>>> {
query item_super_predicates(key: DefId) -> ty::EarlyBinder<ty::Clauses<'tcx>> {
desc { |tcx| "elaborating item assumptions for `{}`", tcx.def_path_str(key) }
}

query item_non_self_assumptions(key: DefId) -> ty::EarlyBinder<&'tcx ty::List<ty::Clause<'tcx>>> {
query item_non_self_assumptions(key: DefId) -> ty::EarlyBinder<ty::Clauses<'tcx>> {
desc { |tcx| "elaborating item assumptions for `{}`", tcx.def_path_str(key) }
}

@@ -2156,7 +2156,7 @@ rustc_queries! {
desc { "resolving instance `{}`", ty::Instance::new(key.value.0, key.value.1) }
}

query reveal_opaque_types_in_bounds(key: &'tcx ty::List<ty::Clause<'tcx>>) -> &'tcx ty::List<ty::Clause<'tcx>> {
query reveal_opaque_types_in_bounds(key: ty::Clauses<'tcx>) -> ty::Clauses<'tcx> {
desc { "revealing opaque types in `{:?}`", key }
}

6 changes: 4 additions & 2 deletions compiler/rustc_middle/src/ty/codec.rs
Original file line number Diff line number Diff line change
@@ -414,7 +414,9 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty
}
}

impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty::Clause<'tcx>> {
impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
for ty::ListWithCachedTypeInfo<ty::Clause<'tcx>>
{
fn decode(decoder: &mut D) -> &'tcx Self {
let len = decoder.read_usize();
decoder.interner().mk_clauses_from_iter(
@@ -461,7 +463,7 @@ impl_decodable_via_ref! {
&'tcx mir::BorrowCheckResult<'tcx>,
&'tcx mir::coverage::CodeRegion,
&'tcx ty::List<ty::BoundVariableKind>,
&'tcx ty::List<ty::Clause<'tcx>>,
&'tcx ty::ListWithCachedTypeInfo<ty::Clause<'tcx>>,
&'tcx ty::List<FieldIdx>,
&'tcx ty::List<(VariantIdx, FieldIdx)>,
}
61 changes: 51 additions & 10 deletions compiler/rustc_middle/src/ty/context.rs
Original file line number Diff line number Diff line change
@@ -25,10 +25,10 @@ use crate::traits::solve::{
ExternalConstraints, ExternalConstraintsData, PredefinedOpaques, PredefinedOpaquesData,
};
use crate::ty::{
self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Const, ConstData, GenericParamDefKind,
ImplPolarity, List, ParamConst, ParamTy, PolyExistentialPredicate, PolyFnSig, Predicate,
PredicateKind, PredicatePolarity, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty,
TyKind, TyVid, TypeVisitable, Visibility,
self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Clauses, Const, ConstData,
GenericParamDefKind, ImplPolarity, List, ListWithCachedTypeInfo, ParamConst, ParamTy,
PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, PredicatePolarity, Region,
RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid, TypeVisitable, Visibility,
};
use crate::ty::{GenericArg, GenericArgs, GenericArgsRef};
use rustc_ast::{self as ast, attr};
@@ -130,6 +130,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
type SubtypePredicate = ty::SubtypePredicate<'tcx>;
type CoercePredicate = ty::CoercePredicate<'tcx>;
type ClosureKind = ty::ClosureKind;
type Clauses = ty::Clauses<'tcx>;

fn mk_canonical_var_infos(self, infos: &[ty::CanonicalVarInfo<Self>]) -> Self::CanonicalVars {
self.mk_canonical_var_infos(infos)
@@ -152,7 +153,7 @@ pub struct CtxtInterners<'tcx> {
region: InternedSet<'tcx, RegionKind<'tcx>>,
poly_existential_predicates: InternedSet<'tcx, List<PolyExistentialPredicate<'tcx>>>,
predicate: InternedSet<'tcx, WithCachedTypeInfo<ty::Binder<'tcx, PredicateKind<'tcx>>>>,
clauses: InternedSet<'tcx, List<Clause<'tcx>>>,
clauses: InternedSet<'tcx, ListWithCachedTypeInfo<Clause<'tcx>>>,
projs: InternedSet<'tcx, List<ProjectionKind>>,
place_elems: InternedSet<'tcx, List<PlaceElem<'tcx>>>,
const_: InternedSet<'tcx, WithCachedTypeInfo<ConstData<'tcx>>>,
@@ -286,6 +287,24 @@ impl<'tcx> CtxtInterners<'tcx> {
.0,
))
}

fn intern_clauses(&self, clauses: &[Clause<'tcx>]) -> Clauses<'tcx> {
if clauses.is_empty() {
ListWithCachedTypeInfo::empty()
} else {
self.clauses
.intern_ref(clauses, || {
let flags = super::flags::FlagComputation::for_clauses(clauses);

InternedInSet(ListWithCachedTypeInfo::from_arena(
&*self.arena,
flags.into(),
clauses,
))
})
.0
}
}
}

// For these preinterned values, an alternative would be to have
@@ -1775,6 +1794,29 @@ impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, List<T>> {
}
}

impl<'tcx, T> Borrow<[T]> for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {
fn borrow(&self) -> &[T] {
&self.0[..]
}
}

impl<'tcx, T: PartialEq> PartialEq for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {
fn eq(&self, other: &InternedInSet<'tcx, ListWithCachedTypeInfo<T>>) -> bool {
// The `Borrow` trait requires that `x.borrow() == y.borrow()` equals
// `x == y`.
self.0[..] == other.0[..]
}
}

impl<'tcx, T: Eq> Eq for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {}

impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, ListWithCachedTypeInfo<T>> {
fn hash<H: Hasher>(&self, s: &mut H) {
// The `Borrow` trait requires that `x.borrow().hash(s) == x.hash(s)`.
self.0[..].hash(s)
}
}

macro_rules! direct_interners {
($($name:ident: $vis:vis $method:ident($ty:ty): $ret_ctor:ident -> $ret_ty:ty,)+) => {
$(impl<'tcx> Borrow<$ty> for InternedInSet<'tcx, $ty> {
@@ -1833,7 +1875,7 @@ macro_rules! slice_interners {
List::empty()
} else {
self.interners.$field.intern_ref(v, || {
InternedInSet(List::from_arena(&*self.arena, v))
InternedInSet(List::from_arena(&*self.arena, (), v))
}).0
}
})+
@@ -1850,7 +1892,6 @@ slice_interners!(
type_lists: pub mk_type_list(Ty<'tcx>),
canonical_var_infos: pub mk_canonical_var_infos(CanonicalVarInfo<'tcx>),
poly_existential_predicates: intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>),
clauses: intern_clauses(Clause<'tcx>),
projs: pub mk_projs(ProjectionKind),
place_elems: pub mk_place_elems(PlaceElem<'tcx>),
bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind),
@@ -2155,11 +2196,11 @@ impl<'tcx> TyCtxt<'tcx> {
self.intern_poly_existential_predicates(eps)
}

pub fn mk_clauses(self, clauses: &[Clause<'tcx>]) -> &'tcx List<Clause<'tcx>> {
pub fn mk_clauses(self, clauses: &[Clause<'tcx>]) -> Clauses<'tcx> {
// FIXME consider asking the input slice to be sorted to avoid
// re-interning permutations, in which case that would be asserted
// here.
self.intern_clauses(clauses)
self.interners.intern_clauses(clauses)
}

pub fn mk_local_def_ids(self, clauses: &[LocalDefId]) -> &'tcx List<LocalDefId> {
@@ -2223,7 +2264,7 @@ impl<'tcx> TyCtxt<'tcx> {
pub fn mk_clauses_from_iter<I, T>(self, iter: I) -> T::Output
where
I: Iterator<Item = T>,
T: CollectAndApply<Clause<'tcx>, &'tcx List<Clause<'tcx>>>,
T: CollectAndApply<Clause<'tcx>, Clauses<'tcx>>,
{
T::collect_and_apply(iter, |xs| self.mk_clauses(xs))
}
9 changes: 9 additions & 0 deletions compiler/rustc_middle/src/ty/flags.rs
Original file line number Diff line number Diff line change
@@ -35,6 +35,15 @@ impl FlagComputation {
result
}

pub fn for_clauses(clauses: &[ty::Clause<'_>]) -> FlagComputation {
let mut result = FlagComputation::new();
for c in clauses {
result.add_flags(c.as_predicate().flags());
result.add_exclusive_binder(c.as_predicate().outer_exclusive_binder());
}
result
}

fn add_flags(&mut self, flags: TypeFlags) {
self.flags = self.flags | flags;
}
9 changes: 5 additions & 4 deletions compiler/rustc_middle/src/ty/impls_ty.rs
Original file line number Diff line number Diff line change
@@ -11,19 +11,20 @@ use rustc_data_structures::stable_hasher::HashingControls;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey};
use rustc_query_system::ich::StableHashingContext;
use std::cell::RefCell;
use std::ptr;

impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for &'tcx ty::List<T>
impl<'a, 'tcx, H, T> HashStable<StableHashingContext<'a>> for &'tcx ty::list::RawList<H, T>
where
T: HashStable<StableHashingContext<'a>>,
{
fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
thread_local! {
static CACHE: RefCell<FxHashMap<(usize, usize, HashingControls), Fingerprint>> =
static CACHE: RefCell<FxHashMap<(*const (), HashingControls), Fingerprint>> =
RefCell::new(Default::default());
}

let hash = CACHE.with(|cache| {
let key = (self.as_ptr() as usize, self.len(), hcx.hashing_controls());
let key = (ptr::from_ref(*self).cast::<()>(), hcx.hashing_controls());
if let Some(&hash) = cache.borrow().get(&key) {
return hash;
}
@@ -40,7 +41,7 @@ where
}
}

impl<'a, 'tcx, T> ToStableHashKey<StableHashingContext<'a>> for &'tcx ty::List<T>
impl<'a, 'tcx, H, T> ToStableHashKey<StableHashingContext<'a>> for &'tcx ty::list::RawList<H, T>
where
T: HashStable<StableHashingContext<'a>>,
{
200 changes: 135 additions & 65 deletions compiler/rustc_middle/src/ty/list.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use super::flags::FlagComputation;
use super::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, TyCtxt, TypeFlags, WithInfcx};
use crate::arena::Arena;
use rustc_data_structures::aligned::{align_of, Aligned};
use rustc_serialize::{Encodable, Encoder};
use rustc_type_ir::{InferCtxtLike, WithInfcx};
use std::alloc::Layout;
use std::cmp::Ordering;
use std::fmt;
@@ -12,6 +13,9 @@ use std::ops::Deref;
use std::ptr;
use std::slice;

#[cfg(parallel_compiler)]
use rustc_data_structures::sync::DynSync;

/// `List<T>` is a bit like `&[T]`, but with some critical differences.
/// - IMPORTANT: Every `List<T>` is *required* to have unique contents. The
/// type's correctness relies on this, *but it does not enforce it*.
@@ -28,15 +32,27 @@ use std::slice;
/// - `T` must be `Copy`. This lets `List<T>` be stored in a dropless arena and
/// iterators return a `T` rather than a `&T`.
/// - `T` must not be zero-sized.
pub type List<T> = RawList<(), T>;

/// A generic type that can be used to prepend a [`List`] with some header.
///
/// The header will be ignored for value-based operations like [`PartialEq`],
/// [`Hash`] and [`Encodable`].
#[repr(C)]
pub struct List<T> {
len: usize,
pub struct RawList<H, T> {
skel: ListSkeleton<H, T>,
opaque: OpaqueListContents,
}

/// A [`RawList`] without the unsized tail. This type is used for layout computation
/// and constructing empty lists.
#[repr(C)]
struct ListSkeleton<H, T> {
header: H,
len: usize,
/// Although this claims to be a zero-length array, in practice `len`
/// elements are actually present.
data: [T; 0],

opaque: OpaqueListContents,
}

extern "C" {
@@ -45,35 +61,17 @@ extern "C" {
type OpaqueListContents;
}

impl<T> List<T> {
/// Returns a reference to the (unique, static) empty list.
impl<H, T> RawList<H, T> {
#[inline(always)]
pub fn empty<'a>() -> &'a List<T> {
#[repr(align(64))]
struct MaxAlign;

assert!(mem::align_of::<T>() <= mem::align_of::<MaxAlign>());

#[repr(C)]
struct InOrder<T, U>(T, U);

// The empty slice is static and contains a single `0` usize (for the
// length) that is 64-byte aligned, thus featuring the necessary
// trailing padding for elements with up to 64-byte alignment.
static EMPTY_SLICE: InOrder<usize, MaxAlign> = InOrder(0, MaxAlign);
unsafe { &*(std::ptr::addr_of!(EMPTY_SLICE) as *const List<T>) }
}

pub fn len(&self) -> usize {
self.len
self.skel.len
}

#[inline(always)]
pub fn as_slice(&self) -> &[T] {
self
}
}

impl<T: Copy> List<T> {
/// Allocates a list from `arena` and copies the contents of `slice` into it.
///
/// WARNING: the contents *must be unique*, such that no list with these
@@ -84,20 +82,31 @@ impl<T: Copy> List<T> {
/// (because the empty list exists statically, and is available via
/// `empty()`).
#[inline]
pub(super) fn from_arena<'tcx>(arena: &'tcx Arena<'tcx>, slice: &[T]) -> &'tcx List<T> {
pub(super) fn from_arena<'tcx>(
arena: &'tcx Arena<'tcx>,
header: H,
slice: &[T],
) -> &'tcx RawList<H, T>
where
T: Copy,
{
assert!(!mem::needs_drop::<T>());
assert!(mem::size_of::<T>() != 0);
assert!(!slice.is_empty());

let (layout, _offset) =
Layout::new::<usize>().extend(Layout::for_value::<[T]>(slice)).unwrap();
let mem = arena.dropless.alloc_raw(layout) as *mut List<T>;
Layout::new::<ListSkeleton<H, T>>().extend(Layout::for_value::<[T]>(slice)).unwrap();

let mem = arena.dropless.alloc_raw(layout) as *mut RawList<H, T>;
unsafe {
// Write the header
ptr::addr_of_mut!((*mem).skel.header).write(header);

// Write the length
ptr::addr_of_mut!((*mem).len).write(slice.len());
ptr::addr_of_mut!((*mem).skel.len).write(slice.len());

// Write the elements
ptr::addr_of_mut!((*mem).data)
ptr::addr_of_mut!((*mem).skel.data)
.cast::<T>()
.copy_from_nonoverlapping(slice.as_ptr(), slice.len());

@@ -110,17 +119,44 @@ impl<T: Copy> List<T> {
//
// This would be weird, as `self.into_iter` iterates over `T` directly.
#[inline(always)]
pub fn iter(&self) -> <&'_ List<T> as IntoIterator>::IntoIter {
pub fn iter(&self) -> <&'_ RawList<H, T> as IntoIterator>::IntoIter
where
T: Copy,
{
self.into_iter()
}
}

impl<T: fmt::Debug> fmt::Debug for List<T> {
macro_rules! impl_list_empty {
($header_ty:ty, $header_init:expr) => {
impl<T> RawList<$header_ty, T> {
/// Returns a reference to the (per header unique, static) empty list.
#[inline(always)]
pub fn empty<'a>() -> &'a RawList<$header_ty, T> {
#[repr(align(64))]
struct MaxAlign;

static EMPTY: ListSkeleton<$header_ty, MaxAlign> =
ListSkeleton { header: $header_init, len: 0, data: [] };

assert!(mem::align_of::<T>() <= mem::align_of::<MaxAlign>());

// SAFETY: `EMPTY` is sufficiently aligned to be an empty list for all
// types with `align_of(T) <= align_of(MaxAlign)`, which we checked above.
unsafe { &*(std::ptr::addr_of!(EMPTY) as *const RawList<$header_ty, T>) }
}
}
};
}

impl_list_empty!((), ());

impl<H, T: fmt::Debug> fmt::Debug for RawList<H, T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
(**self).fmt(f)
}
}
impl<'tcx, T: super::DebugWithInfcx<TyCtxt<'tcx>>> super::DebugWithInfcx<TyCtxt<'tcx>> for List<T> {
impl<'tcx, H, T: DebugWithInfcx<TyCtxt<'tcx>>> DebugWithInfcx<TyCtxt<'tcx>> for RawList<H, T> {
fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
this: WithInfcx<'_, Infcx, &Self>,
f: &mut core::fmt::Formatter<'_>,
@@ -129,40 +165,40 @@ impl<'tcx, T: super::DebugWithInfcx<TyCtxt<'tcx>>> super::DebugWithInfcx<TyCtxt<
}
}

impl<S: Encoder, T: Encodable<S>> Encodable<S> for List<T> {
impl<H, S: Encoder, T: Encodable<S>> Encodable<S> for RawList<H, T> {
#[inline]
fn encode(&self, s: &mut S) {
(**self).encode(s);
}
}

impl<T: PartialEq> PartialEq for List<T> {
impl<H, T: PartialEq> PartialEq for RawList<H, T> {
#[inline]
fn eq(&self, other: &List<T>) -> bool {
fn eq(&self, other: &RawList<H, T>) -> bool {
// Pointer equality implies list equality (due to the unique contents
// assumption).
ptr::eq(self, other)
}
}

impl<T: Eq> Eq for List<T> {}
impl<H, T: Eq> Eq for RawList<H, T> {}

impl<T> Ord for List<T>
impl<H, T> Ord for RawList<H, T>
where
T: Ord,
{
fn cmp(&self, other: &List<T>) -> Ordering {
fn cmp(&self, other: &RawList<H, T>) -> Ordering {
// Pointer equality implies list equality (due to the unique contents
// assumption), but the contents must be compared otherwise.
if self == other { Ordering::Equal } else { <[T] as Ord>::cmp(&**self, &**other) }
}
}

impl<T> PartialOrd for List<T>
impl<H, T> PartialOrd for RawList<H, T>
where
T: PartialOrd,
{
fn partial_cmp(&self, other: &List<T>) -> Option<Ordering> {
fn partial_cmp(&self, other: &RawList<H, T>) -> Option<Ordering> {
// Pointer equality implies list equality (due to the unique contents
// assumption), but the contents must be compared otherwise.
if self == other {
@@ -173,31 +209,36 @@ where
}
}

impl<T> Hash for List<T> {
impl<Hdr, T> Hash for RawList<Hdr, T> {
#[inline]
fn hash<H: Hasher>(&self, s: &mut H) {
// Pointer hashing is sufficient (due to the unique contents
// assumption).
(self as *const List<T>).hash(s)
ptr::from_ref(self).hash(s)
}
}

impl<T> Deref for List<T> {
impl<H, T> Deref for RawList<H, T> {
type Target = [T];
#[inline(always)]
fn deref(&self) -> &[T] {
self.as_ref()
}
}

impl<T> AsRef<[T]> for List<T> {
impl<H, T> AsRef<[T]> for RawList<H, T> {
#[inline(always)]
fn as_ref(&self) -> &[T] {
unsafe { slice::from_raw_parts(self.data.as_ptr(), self.len) }
let data_ptr = ptr::addr_of!(self.skel.data).cast::<T>();
// SAFETY: `data_ptr` has the same provenance as `self` and can therefore
// access the `self.skel.len` elements stored at `self.skel.data`.
// Note that we specifically don't reborrow `&self.skel.data`, because that
// would give us a pointer with provenance over 0 bytes.
unsafe { slice::from_raw_parts(data_ptr, self.skel.len) }
}
}

impl<'a, T: Copy> IntoIterator for &'a List<T> {
impl<'a, H, T: Copy> IntoIterator for &'a RawList<H, T> {
type Item = T;
type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>;
#[inline(always)]
@@ -206,27 +247,56 @@ impl<'a, T: Copy> IntoIterator for &'a List<T> {
}
}

unsafe impl<T: Sync> Sync for List<T> {}
unsafe impl<H: Sync, T: Sync> Sync for RawList<H, T> {}

// We need this since `List` uses extern type `OpaqueListContents`.
#[cfg(parallel_compiler)]
use rustc_data_structures::sync::DynSync;

use super::TyCtxt;
#[cfg(parallel_compiler)]
unsafe impl<T: DynSync> DynSync for List<T> {}
unsafe impl<H: DynSync, T: DynSync> DynSync for RawList<H, T> {}

// Safety:
// Layouts of `Equivalent<T>` and `List<T>` are the same, modulo opaque tail,
// thus aligns of `Equivalent<T>` and `List<T>` must be the same.
unsafe impl<T> Aligned for List<T> {
const ALIGN: ptr::Alignment = {
#[repr(C)]
struct Equivalent<T> {
_len: usize,
_data: [T; 0],
}
// Layouts of `ListSkeleton<H, T>` and `RawList<H, T>` are the same, modulo opaque tail,
// thus aligns of `ListSkeleton<H, T>` and `RawList<H, T>` must be the same.
unsafe impl<H, T> Aligned for RawList<H, T> {
const ALIGN: ptr::Alignment = align_of::<ListSkeleton<H, T>>();
}

align_of::<Equivalent<T>>()
};
/// A [`List`] that additionally stores type information inline to speed up
/// [`TypeVisitableExt`](super::TypeVisitableExt) operations.
pub type ListWithCachedTypeInfo<T> = RawList<TypeInfo, T>;

impl<T> ListWithCachedTypeInfo<T> {
#[inline(always)]
pub fn flags(&self) -> TypeFlags {
self.skel.header.flags
}

#[inline(always)]
pub fn outer_exclusive_binder(&self) -> DebruijnIndex {
self.skel.header.outer_exclusive_binder
}
}

impl_list_empty!(TypeInfo, TypeInfo::empty());

/// The additional info that is stored in [`ListWithCachedTypeInfo`].
#[repr(C)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TypeInfo {
flags: TypeFlags,
outer_exclusive_binder: DebruijnIndex,
}

impl TypeInfo {
const fn empty() -> Self {
Self { flags: TypeFlags::empty(), outer_exclusive_binder: super::INNERMOST }
}
}

impl From<FlagComputation> for TypeInfo {
fn from(computation: FlagComputation) -> TypeInfo {
TypeInfo {
flags: computation.flags,
outer_exclusive_binder: computation.outer_exclusive_binder,
}
}
}
26 changes: 19 additions & 7 deletions compiler/rustc_middle/src/ty/mod.rs
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ pub use self::context::{
TyCtxt, TyCtxtFeed,
};
pub use self::instance::{Instance, InstanceDef, ReifyReason, ShortInstance, UnusedGenericParams};
pub use self::list::List;
pub use self::list::{List, ListWithCachedTypeInfo};
pub use self::parameterized::ParameterizedOverTcx;
pub use self::predicate::{
Clause, ClauseKind, CoercePredicate, ExistentialPredicate, ExistentialProjection,
@@ -1034,6 +1034,18 @@ impl PlaceholderLike for PlaceholderConst {
}
}

pub type Clauses<'tcx> = &'tcx ListWithCachedTypeInfo<Clause<'tcx>>;

impl<'tcx> rustc_type_ir::visit::Flags for Clauses<'tcx> {
fn flags(&self) -> TypeFlags {
(**self).flags()
}

fn outer_exclusive_binder(&self) -> DebruijnIndex {
(**self).outer_exclusive_binder()
}
}

/// When interacting with the type system we must provide information about the
/// environment. `ParamEnv` is the type that represents this information. See the
/// [dev guide chapter][param_env_guide] for more information.
@@ -1053,7 +1065,7 @@ pub struct ParamEnv<'tcx> {
/// want `Reveal::All`.
///
/// Note: This is packed, use the reveal() method to access it.
packed: CopyTaggedPtr<&'tcx List<Clause<'tcx>>, ParamTag, true>,
packed: CopyTaggedPtr<Clauses<'tcx>, ParamTag, true>,
}

#[derive(Copy, Clone)]
@@ -1112,11 +1124,11 @@ impl<'tcx> ParamEnv<'tcx> {
/// [param_env_guide]: https://rustc-dev-guide.rust-lang.org/param_env/param_env_summary.html
#[inline]
pub fn empty() -> Self {
Self::new(List::empty(), Reveal::UserFacing)
Self::new(ListWithCachedTypeInfo::empty(), Reveal::UserFacing)
}

#[inline]
pub fn caller_bounds(self) -> &'tcx List<Clause<'tcx>> {
pub fn caller_bounds(self) -> Clauses<'tcx> {
self.packed.pointer()
}

@@ -1134,12 +1146,12 @@ impl<'tcx> ParamEnv<'tcx> {
/// or invoke `param_env.with_reveal_all()`.
#[inline]
pub fn reveal_all() -> Self {
Self::new(List::empty(), Reveal::All)
Self::new(ListWithCachedTypeInfo::empty(), Reveal::All)
}

/// Construct a trait environment with the given set of predicates.
#[inline]
pub fn new(caller_bounds: &'tcx List<Clause<'tcx>>, reveal: Reveal) -> Self {
pub fn new(caller_bounds: Clauses<'tcx>, reveal: Reveal) -> Self {
ty::ParamEnv { packed: CopyTaggedPtr::new(caller_bounds, ParamTag { reveal }) }
}

@@ -1168,7 +1180,7 @@ impl<'tcx> ParamEnv<'tcx> {
/// Returns this same environment but with no caller bounds.
#[inline]
pub fn without_caller_bounds(self) -> Self {
Self::new(List::empty(), self.reveal())
Self::new(ListWithCachedTypeInfo::empty(), self.reveal())
}

/// Creates a pair of param-env and value for use in queries.
14 changes: 13 additions & 1 deletion compiler/rustc_middle/src/ty/structural_impls.rs
Original file line number Diff line number Diff line change
@@ -712,7 +712,19 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
}
}

impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::Clause<'tcx>> {
impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
visitor.visit_clauses(self)
}
}

impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
self.as_slice().visit_with(visitor)
}
}

impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
self,
folder: &mut F,
20 changes: 11 additions & 9 deletions compiler/rustc_middle/src/ty/util.rs
Original file line number Diff line number Diff line change
@@ -1608,25 +1608,27 @@ pub fn is_trivially_const_drop(ty: Ty<'_>) -> bool {
/// let v = self.iter().map(|p| p.fold_with(folder)).collect::<SmallVec<[_; 8]>>();
/// folder.tcx().intern_*(&v)
/// ```
pub fn fold_list<'tcx, F, T>(
list: &'tcx ty::List<T>,
pub fn fold_list<'tcx, F, L, T>(
list: L,
folder: &mut F,
intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> &'tcx ty::List<T>,
) -> Result<&'tcx ty::List<T>, F::Error>
intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> L,
) -> Result<L, F::Error>
where
F: FallibleTypeFolder<TyCtxt<'tcx>>,
L: AsRef<[T]>,
T: TypeFoldable<TyCtxt<'tcx>> + PartialEq + Copy,
{
let mut iter = list.iter();
let slice = list.as_ref();
let mut iter = slice.iter().copied();
// Look for the first element that changed
match iter.by_ref().enumerate().find_map(|(i, t)| match t.try_fold_with(folder) {
Ok(new_t) if new_t == t => None,
new_t => Some((i, new_t)),
}) {
Some((i, Ok(new_t))) => {
// An element changed, prepare to intern the resulting list
let mut new_list = SmallVec::<[_; 8]>::with_capacity(list.len());
new_list.extend_from_slice(&list[..i]);
let mut new_list = SmallVec::<[_; 8]>::with_capacity(slice.len());
new_list.extend_from_slice(&slice[..i]);
new_list.push(new_t);
for t in iter {
new_list.push(t.try_fold_with(folder)?)
@@ -1647,8 +1649,8 @@ pub struct AlwaysRequiresDrop;
/// with their underlying types.
pub fn reveal_opaque_types_in_bounds<'tcx>(
tcx: TyCtxt<'tcx>,
val: &'tcx ty::List<ty::Clause<'tcx>>,
) -> &'tcx ty::List<ty::Clause<'tcx>> {
val: ty::Clauses<'tcx>,
) -> ty::Clauses<'tcx> {
let mut visitor = OpaqueTypeExpander {
seen_opaque_tys: FxHashSet::default(),
expanded_cache: FxHashMap::default(),
1 change: 1 addition & 0 deletions compiler/rustc_type_ir/src/interner.rs
Original file line number Diff line number Diff line change
@@ -100,6 +100,7 @@ pub trait Interner: Sized + Copy {
type SubtypePredicate: Copy + Debug + Hash + Eq;
type CoercePredicate: Copy + Debug + Hash + Eq;
type ClosureKind: Copy + Debug + Hash + Eq;
type Clauses: Copy + Debug + Hash + Eq + TypeSuperVisitable<Self> + Flags;

fn mk_canonical_var_infos(self, infos: &[CanonicalVarInfo<Self>]) -> Self::CanonicalVars;
}
23 changes: 23 additions & 0 deletions compiler/rustc_type_ir/src/visit.rs
Original file line number Diff line number Diff line change
@@ -110,6 +110,10 @@ pub trait TypeVisitor<I: Interner>: Sized {
fn visit_predicate(&mut self, p: I::Predicate) -> Self::Result {
p.super_visit_with(self)
}

fn visit_clauses(&mut self, p: I::Clauses) -> Self::Result {
p.super_visit_with(self)
}
}

///////////////////////////////////////////////////////////////////////////
@@ -423,6 +427,16 @@ impl<I: Interner> TypeVisitor<I> for HasTypeFlagsVisitor {
ControlFlow::Continue(())
}
}

#[inline]
fn visit_clauses(&mut self, clauses: I::Clauses) -> Self::Result {
// Note: no `super_visit_with` call.
if clauses.flags().intersects(self.flags) {
ControlFlow::Break(FoundFlags)
} else {
ControlFlow::Continue(())
}
}
}

#[derive(Debug, PartialEq, Eq, Copy, Clone)]
@@ -515,6 +529,15 @@ impl<I: Interner> TypeVisitor<I> for HasEscapingVarsVisitor {
ControlFlow::Continue(())
}
}

#[inline]
fn visit_clauses(&mut self, clauses: I::Clauses) -> Self::Result {
if clauses.outer_exclusive_binder() > self.outer_index {
ControlFlow::Break(FoundEscapingVars)
} else {
ControlFlow::Continue(())
}
}
}

struct HasErrorVisitor;