Skip to content

Commit 5d47c91

Browse files
eddybeddyb
committed
rustc: extend the niche-filling enum optimization past 2 variants.
1 parent 53a75bc commit 5d47c91

File tree

4 files changed

+158
-105
lines changed

4 files changed

+158
-105
lines changed

‎src/librustc/ty/layout.rs

Lines changed: 114 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -799,19 +799,19 @@ pub enum Variants {
799799
variants: Vec<CachedLayout>,
800800
},
801801

802-
/// Two cases distinguished by a niche (a value invalid for a type):
802+
/// Multiple cases distinguished by a niche (values invalid for a type):
803803
/// the variant `dataful_variant` contains a niche at an arbitrary
804-
/// offset (field 0 of the enum), which is set to `niche_value`
805-
/// for the other variant.
804+
/// offset (field 0 of the enum), which for a variant with discriminant
805+
/// `d` is set to `(d - niche_variants.start).wrapping_add(niche_start)`.
806806
///
807807
/// For example, `Option<(usize, &T)>` is represented such that
808808
/// `None` has a null pointer for the second tuple field, and
809809
/// `Some` is the identity function (with a non-null reference).
810810
NicheFilling {
811811
dataful_variant: usize,
812-
niche_variant: usize,
812+
niche_variants: RangeInclusive<usize>,
813813
niche: Scalar,
814-
niche_value: u128,
814+
niche_start: u128,
815815
variants: Vec<CachedLayout>,
816816
}
817817
}
@@ -1358,11 +1358,11 @@ impl<'a, 'tcx> CachedLayout {
13581358
}).collect::<Result<Vec<_>, _>>()
13591359
}).collect::<Result<Vec<_>, _>>()?;
13601360

1361-
let (inh_first, inh_second, inh_third) = {
1361+
let (inh_first, inh_second) = {
13621362
let mut inh_variants = (0..variants.len()).filter(|&v| {
13631363
variants[v].iter().all(|f| f.abi != Abi::Uninhabited)
13641364
});
1365-
(inh_variants.next(), inh_variants.next(), inh_variants.next())
1365+
(inh_variants.next(), inh_variants.next())
13661366
};
13671367
if inh_first.is_none() {
13681368
// Uninhabited because it has no variants, or only uninhabited ones.
@@ -1458,53 +1458,79 @@ impl<'a, 'tcx> CachedLayout {
14581458
let no_explicit_discriminants = def.variants.iter().enumerate()
14591459
.all(|(i, v)| v.discr == ty::VariantDiscr::Relative(i));
14601460

1461-
if inh_second.is_some() && inh_third.is_none() &&
1462-
!def.repr.inhibit_enum_layout_opt() &&
1463-
no_explicit_discriminants {
1464-
// Nullable pointer optimization
1465-
let (a, b) = (inh_first.unwrap(), inh_second.unwrap());
1466-
for &(i, other) in &[(a, b), (b, a)] {
1467-
if !variants[other].iter().all(|f| f.is_zst()) {
1468-
continue;
1461+
// Niche-filling enum optimization.
1462+
if !def.repr.inhibit_enum_layout_opt() && no_explicit_discriminants {
1463+
let mut dataful_variant = None;
1464+
let mut niche_variants = usize::max_value()...0;
1465+
1466+
// Find one non-ZST variant.
1467+
'variants: for (v, fields) in variants.iter().enumerate() {
1468+
for f in fields {
1469+
if f.abi == Abi::Uninhabited {
1470+
continue 'variants;
1471+
}
1472+
if !f.is_zst() {
1473+
if dataful_variant.is_none() {
1474+
dataful_variant = Some(v);
1475+
continue 'variants;
1476+
} else {
1477+
dataful_variant = None;
1478+
break 'variants;
1479+
}
1480+
}
1481+
}
1482+
if niche_variants.start > v {
1483+
niche_variants.start = v;
14691484
}
1485+
niche_variants.end = v;
1486+
}
1487+
1488+
if niche_variants.start > niche_variants.end {
1489+
dataful_variant = None;
1490+
}
14701491

1492+
if let Some(i) = dataful_variant {
1493+
let count = (niche_variants.end - niche_variants.start + 1) as u128;
14711494
for (field_index, field) in variants[i].iter().enumerate() {
1472-
if let Some((offset, niche, niche_value)) = field.find_niche(cx)? {
1473-
let st = variants.iter().enumerate().map(|(j, v)| {
1474-
let mut st = univariant_uninterned(v,
1475-
&def.repr, StructKind::AlwaysSized)?;
1476-
st.variants = Variants::Single { index: j };
1477-
Ok(st)
1478-
}).collect::<Result<Vec<_>, _>>()?;
1479-
1480-
let offset = st[i].fields.offset(field_index) + offset;
1481-
let CachedLayout { size, align, primitive_align, .. } = st[i];
1482-
let abi = if offset.bytes() == 0 && niche.value.size(dl) == size {
1483-
Abi::Scalar(niche.clone())
1484-
} else {
1485-
Abi::Aggregate {
1486-
sized: true,
1487-
packed: st[i].abi.is_packed()
1488-
}
1495+
let (offset, niche, niche_start) =
1496+
match field.find_niche(cx, count)? {
1497+
Some(niche) => niche,
1498+
None => continue
14891499
};
1490-
return Ok(tcx.intern_layout(CachedLayout {
1491-
variants: Variants::NicheFilling {
1492-
dataful_variant: i,
1493-
niche_variant: other,
1494-
niche,
1495-
niche_value,
1496-
variants: st,
1497-
},
1498-
fields: FieldPlacement::Arbitrary {
1499-
offsets: vec![offset],
1500-
memory_index: vec![0]
1501-
},
1502-
abi,
1503-
size,
1504-
align,
1505-
primitive_align
1506-
}));
1507-
}
1500+
let st = variants.iter().enumerate().map(|(j, v)| {
1501+
let mut st = univariant_uninterned(v,
1502+
&def.repr, StructKind::AlwaysSized)?;
1503+
st.variants = Variants::Single { index: j };
1504+
Ok(st)
1505+
}).collect::<Result<Vec<_>, _>>()?;
1506+
1507+
let offset = st[i].fields.offset(field_index) + offset;
1508+
let CachedLayout { size, align, primitive_align, .. } = st[i];
1509+
let abi = if offset.bytes() == 0 && niche.value.size(dl) == size {
1510+
Abi::Scalar(niche.clone())
1511+
} else {
1512+
Abi::Aggregate {
1513+
sized: true,
1514+
packed: st[i].abi.is_packed()
1515+
}
1516+
};
1517+
return Ok(tcx.intern_layout(CachedLayout {
1518+
variants: Variants::NicheFilling {
1519+
dataful_variant: i,
1520+
niche_variants,
1521+
niche,
1522+
niche_start,
1523+
variants: st,
1524+
},
1525+
fields: FieldPlacement::Arbitrary {
1526+
offsets: vec![offset],
1527+
memory_index: vec![0]
1528+
},
1529+
abi,
1530+
size,
1531+
align,
1532+
primitive_align
1533+
}));
15081534
}
15091535
}
15101536
}
@@ -2234,50 +2260,50 @@ impl<'a, 'tcx> TyLayout<'tcx> {
22342260
}
22352261

22362262
/// Find the offset of a niche leaf field, starting from
2237-
/// the given type and recursing through aggregates.
2263+
/// the given type and recursing through aggregates, which
2264+
/// has at least `count` consecutive invalid values.
22382265
/// The tuple is `(offset, scalar, niche_value)`.
22392266
// FIXME(eddyb) traverse already optimized enums.
2240-
fn find_niche<C>(&self, cx: C)
2267+
fn find_niche<C>(&self, cx: C, count: u128)
22412268
-> Result<Option<(Size, Scalar, u128)>, LayoutError<'tcx>>
22422269
where C: LayoutOf<Ty<'tcx>, TyLayout = Result<Self, LayoutError<'tcx>>> +
22432270
HasTyCtxt<'tcx>
22442271
{
22452272
let scalar_component = |scalar: &Scalar, offset| {
2246-
// FIXME(eddyb) support negative/wrap-around discriminant ranges.
2247-
let Scalar { value, ref valid_range } = *scalar;
2248-
if valid_range.start < valid_range.end {
2249-
let bits = value.size(cx).bits();
2250-
assert!(bits <= 128);
2251-
let max_value = !0u128 >> (128 - bits);
2252-
if valid_range.start > 0 {
2253-
let niche = valid_range.start - 1;
2254-
Ok(Some((offset, Scalar {
2255-
value,
2256-
valid_range: niche...valid_range.end
2257-
}, niche)))
2258-
} else if valid_range.end < max_value {
2259-
let niche = valid_range.end + 1;
2260-
Ok(Some((offset, Scalar {
2261-
value,
2262-
valid_range: valid_range.start...niche
2263-
}, niche)))
2264-
} else {
2265-
Ok(None)
2266-
}
2273+
let Scalar { value, valid_range: ref v } = *scalar;
2274+
2275+
let bits = value.size(cx).bits();
2276+
assert!(bits <= 128);
2277+
let max_value = !0u128 >> (128 - bits);
2278+
2279+
// Find out how many values are outside the valid range.
2280+
let niches = if v.start <= v.end {
2281+
v.start + (max_value - v.end)
22672282
} else {
2268-
Ok(None)
2283+
v.start - v.end - 1
2284+
};
2285+
2286+
// Give up if we can't fit `count` consecutive niches.
2287+
if count > niches {
2288+
return None;
22692289
}
2290+
2291+
let niche_start = v.end.wrapping_add(1) & max_value;
2292+
let niche_end = v.end.wrapping_add(count) & max_value;
2293+
Some((offset, Scalar {
2294+
value,
2295+
valid_range: v.start...niche_end
2296+
}, niche_start))
22702297
};
22712298

22722299
match self.abi {
22732300
Abi::Scalar(ref scalar) => {
2274-
return scalar_component(scalar, Size::from_bytes(0));
2301+
return Ok(scalar_component(scalar, Size::from_bytes(0)));
22752302
}
22762303
Abi::ScalarPair(ref a, ref b) => {
2277-
if let Some(result) = scalar_component(a, Size::from_bytes(0))? {
2278-
return Ok(Some(result));
2279-
}
2280-
return scalar_component(b, a.value.size(cx).abi_align(b.value.align(cx)));
2304+
return Ok(scalar_component(a, Size::from_bytes(0)).or_else(|| {
2305+
scalar_component(b, a.value.size(cx).abi_align(b.value.align(cx)))
2306+
}));
22812307
}
22822308
_ => {}
22832309
}
@@ -2290,13 +2316,13 @@ impl<'a, 'tcx> TyLayout<'tcx> {
22902316
return Ok(None);
22912317
}
22922318
}
2293-
if let FieldPlacement::Array { count, .. } = self.fields {
2294-
if count > 0 {
2295-
return self.field(cx, 0)?.find_niche(cx);
2319+
if let FieldPlacement::Array { .. } = self.fields {
2320+
if self.fields.count() > 0 {
2321+
return self.field(cx, 0)?.find_niche(cx, count);
22962322
}
22972323
}
22982324
for i in 0..self.fields.count() {
2299-
let r = self.field(cx, i)?.find_niche(cx)?;
2325+
let r = self.field(cx, i)?.find_niche(cx, count)?;
23002326
if let Some((offset, scalar, niche_value)) = r {
23012327
let offset = self.fields.offset(i) + offset;
23022328
return Ok(Some((offset, scalar, niche_value)));
@@ -2326,15 +2352,16 @@ impl<'gcx> HashStable<StableHashingContext<'gcx>> for Variants {
23262352
}
23272353
NicheFilling {
23282354
dataful_variant,
2329-
niche_variant,
2355+
niche_variants: RangeInclusive { start, end },
23302356
ref niche,
2331-
niche_value,
2357+
niche_start,
23322358
ref variants,
23332359
} => {
23342360
dataful_variant.hash_stable(hcx, hasher);
2335-
niche_variant.hash_stable(hcx, hasher);
2361+
start.hash_stable(hcx, hasher);
2362+
end.hash_stable(hcx, hasher);
23362363
niche.hash_stable(hcx, hasher);
2337-
niche_value.hash_stable(hcx, hasher);
2364+
niche_start.hash_stable(hcx, hasher);
23382365
variants.hash_stable(hcx, hasher);
23392366
}
23402367
}

‎src/librustc_trans/debuginfo/metadata.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1144,7 +1144,7 @@ impl<'tcx> EnumMemberDescriptionFactory<'tcx> {
11441144
}
11451145
}).collect()
11461146
}
1147-
layout::Variants::NicheFilling { dataful_variant, niche_variant, .. } => {
1147+
layout::Variants::NicheFilling { dataful_variant, ref niche_variants, .. } => {
11481148
let variant = self.layout.for_variant(cx, dataful_variant);
11491149
// Create a description of the non-null variant
11501150
let (variant_type_metadata, member_description_factory) =
@@ -1167,6 +1167,8 @@ impl<'tcx> EnumMemberDescriptionFactory<'tcx> {
11671167
let mut name = String::from("RUST$ENCODED$ENUM$");
11681168
// HACK(eddyb) the debuggers should just handle offset+size
11691169
// of discriminant instead of us having to recover its path.
1170+
// Right now it's not even going to work for `niche_start > 0`,
1171+
// and for multiple niche variants it only supports the first.
11701172
fn compute_field_path<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>,
11711173
name: &mut String,
11721174
layout: TyLayout<'tcx>,
@@ -1189,7 +1191,7 @@ impl<'tcx> EnumMemberDescriptionFactory<'tcx> {
11891191
self.layout,
11901192
self.layout.fields.offset(0),
11911193
self.layout.field(cx, 0).size);
1192-
name.push_str(&adt.variants[niche_variant].name.as_str());
1194+
name.push_str(&adt.variants[niche_variants.start].name.as_str());
11931195

11941196
// Create the (singleton) list of descriptions of union members.
11951197
vec![

‎src/librustc_trans/mir/constant.rs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1107,12 +1107,19 @@ fn trans_const_adt<'a, 'tcx>(
11071107
build_const_struct(ccx, l.for_variant(ccx, variant_index), vals, Some(discr))
11081108
}
11091109
}
1110-
layout::Variants::NicheFilling { dataful_variant, niche_value, .. } => {
1110+
layout::Variants::NicheFilling {
1111+
dataful_variant,
1112+
ref niche_variants,
1113+
niche_start,
1114+
..
1115+
} => {
11111116
if variant_index == dataful_variant {
11121117
build_const_struct(ccx, l.for_variant(ccx, dataful_variant), vals, None)
11131118
} else {
11141119
let niche = l.field(ccx, 0);
11151120
let niche_llty = niche.llvm_type(ccx);
1121+
let niche_value = ((variant_index - niche_variants.start) as u128)
1122+
.wrapping_add(niche_start);
11161123
// FIXME(eddyb) Check the actual primitive type here.
11171124
let niche_llval = if niche_value == 0 {
11181125
// HACK(eddyb) Using `C_null` as it works on all types.

0 commit comments

Comments
 (0)