Skip to content

Commit 1af0330

Browse files
adwinwhiteadwinwhite
committed
make enum size not depend on the order of variants
1 parent c0838c8 commit 1af0330

File tree

7 files changed

+298
-141
lines changed

7 files changed

+298
-141
lines changed

‎compiler/rustc_abi/src/layout.rs

+10-4
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
192192
pub fn layout_of_struct_or_enum<
193193
'a,
194194
FieldIdx: Idx,
195-
VariantIdx: Idx,
195+
VariantIdx: Idx + PartialOrd,
196196
F: Deref<Target = &'a LayoutS<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
197197
>(
198198
&self,
@@ -464,7 +464,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
464464
fn layout_of_enum<
465465
'a,
466466
FieldIdx: Idx,
467-
VariantIdx: Idx,
467+
VariantIdx: Idx + PartialOrd,
468468
F: Deref<Target = &'a LayoutS<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
469469
>(
470470
&self,
@@ -524,8 +524,14 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
524524
let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
525525
..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
526526

527-
let count =
528-
(niche_variants.end().index() as u128 - niche_variants.start().index() as u128) + 1;
527+
let count = {
528+
let niche_variants_len = (niche_variants.end().index() as u128 - niche_variants.start().index() as u128) + 1;
529+
if niche_variants.contains(&largest_variant_index) {
530+
niche_variants_len - 1
531+
} else {
532+
niche_variants_len
533+
}
534+
};
529535

530536
// Use the largest niche in the largest variant.
531537
let niche = variant_layouts[largest_variant_index].largest_niche?;

‎compiler/rustc_abi/src/lib.rs

+50-6
Original file line numberDiff line numberDiff line change
@@ -1475,15 +1475,59 @@ pub enum TagEncoding<VariantIdx: Idx> {
14751475
Direct,
14761476

14771477
/// Niche (values invalid for a type) encoding the discriminant:
1478-
/// Discriminant and variant index coincide.
1478+
/// Discriminant and variant index doesn't always coincide.
1479+
///
14791480
/// The variant `untagged_variant` contains a niche at an arbitrary
14801481
/// offset (field `tag_field` of the enum), which for a variant with
1481-
/// discriminant `d` is set to
1482-
/// `(d - niche_variants.start).wrapping_add(niche_start)`.
1482+
/// discriminant `d` is set to `d.wrapping_add(niche_start)`.
14831483
///
1484-
/// For example, `Option<(usize, &T)>` is represented such that
1485-
/// `None` has a null pointer for the second tuple field, and
1486-
/// `Some` is the identity function (with a non-null reference).
1484+
/// As for how to compute the discriminant, we have an optimization here that we allocate discriminant
1485+
/// value starting from the variant after the `untagged_variant` when the `untagged_variant` is
1486+
/// contained in `niche_variants`' range. Thus the `untagged_variant` won't be allocated with a
1487+
/// unneeded discriminant. Motivation for this is issue #117238.
1488+
/// For example,
1489+
/// ```rust
1490+
/// enum {
1491+
/// A, // 1
1492+
/// B, // 2
1493+
/// C(bool), // untagged_variant, no discriminant
1494+
/// D, // has a discriminant of 0
1495+
/// }
1496+
/// ```
1497+
/// The algorithm is as follows:
1498+
/// ```rust
1499+
/// // We ignore leading and trailing variants that don't need discriminants.
1500+
/// adjusted_len = niche_variants.end - niche_variants.start + 1
1501+
/// adjusted_index = variant_index - niche_variants.start
1502+
/// d = if niche_variants.contains(untagged_variant) {
1503+
/// adjusted_untagged_index = untagged_variant - niche_variants.start
1504+
/// (adjusted_index + adjusted_len - adjusted_untagged_index) % adjusted_len - 1
1505+
/// } else {
1506+
/// adjusted_index
1507+
/// }
1508+
/// tag_value = d.wrapping_add(niche_start)
1509+
/// ```
1510+
/// To load variant index from tag value:
1511+
/// ```rust
1512+
/// adjusted_len = niche_variants.end - niche_variants.start + 1
1513+
/// d = tag_value.wrapping_sub(niche_start)
1514+
/// variant_index = if niche_variants.contains(untagged_variant) {
1515+
/// if d < adjusted_len - 1 {
1516+
/// adjusted_untagged_index = untagged_variant - niche_variants.start
1517+
/// (d + 1 + adjusted_untagged_index) % adjusted_len + niche_variants.start
1518+
/// } else {
1519+
/// // When the discriminant is larger than the number of variants having
1520+
/// // discriminant, we know it represents the untagged_variant.
1521+
/// untagged_variant
1522+
/// }
1523+
/// } else {
1524+
/// if d < adjusted_len {
1525+
/// d + niche_variants.start
1526+
/// } else {
1527+
/// untagged_variant
1528+
/// }
1529+
/// }
1530+
/// ```
14871531
Niche {
14881532
untagged_variant: VariantIdx,
14891533
niche_variants: RangeInclusive<VariantIdx>,

‎compiler/rustc_codegen_cranelift/src/discriminant.rs

+92-64
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,19 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
5151
variants: _,
5252
} => {
5353
if variant_index != untagged_variant {
54+
let discr_len = niche_variants.end().index() - niche_variants.start().index() + 1;
55+
let adj_idx = variant_index.index() - niche_variants.start().index();
56+
5457
let niche = place.place_field(fx, FieldIdx::new(tag_field));
5558
let niche_type = fx.clif_type(niche.layout().ty).unwrap();
56-
let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
57-
let niche_value = (niche_value as u128).wrapping_add(niche_start);
59+
60+
let discr = if niche_variants.contains(&untagged_variant) {
61+
let adj_untagged_idx = untagged_variant.index() - niche_variants.start().index();
62+
(adj_idx + discr_len - adj_untagged_idx) % discr_len - 1;
63+
} else {
64+
adj_idx
65+
};
66+
let niche_value = (discr as u128).wrapping_add(niche_start);
5867
let niche_value = match niche_type {
5968
types::I128 => {
6069
let lsb = fx.bcx.ins().iconst(types::I64, niche_value as u64 as i64);
@@ -130,73 +139,92 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
130139
dest.write_cvalue(fx, res);
131140
}
132141
TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
133-
let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
134-
135-
// We have a subrange `niche_start..=niche_end` inside `range`.
136-
// If the value of the tag is inside this subrange, it's a
137-
// "niche value", an increment of the discriminant. Otherwise it
138-
// indicates the untagged variant.
139-
// A general algorithm to extract the discriminant from the tag
140-
// is:
141-
// relative_tag = tag - niche_start
142-
// is_niche = relative_tag <= (ule) relative_max
143-
// discr = if is_niche {
144-
// cast(relative_tag) + niche_variants.start()
145-
// } else {
146-
// untagged_variant
147-
// }
148-
// However, we will likely be able to emit simpler code.
149-
150-
let (is_niche, tagged_discr, delta) = if relative_max == 0 {
151-
// Best case scenario: only one tagged variant. This will
152-
// likely become just a comparison and a jump.
153-
// The algorithm is:
154-
// is_niche = tag == niche_start
155-
// discr = if is_niche {
156-
// niche_start
157-
// } else {
158-
// untagged_variant
159-
// }
160-
let is_niche = codegen_icmp_imm(fx, IntCC::Equal, tag, niche_start as i128);
142+
// See the algorithm explanation in the definition of `TagEncoding::Niche`.
143+
let discr_len = niche_variants.end().index() - niche_variants.start().index() + 1;
144+
145+
let niche_start = match fx.bcx.func.dfg.value_type(tag) {
146+
types::I128 => {
147+
let lsb = fx.bcx.ins().iconst(types::I64, niche_start as u64 as i64);
148+
let msb =
149+
fx.bcx.ins().iconst(types::I64, (niche_start >> 64) as u64 as i64);
150+
fx.bcx.ins().iconcat(lsb, msb)
151+
}
152+
ty => fx.bcx.ins().iconst(ty, niche_start as i64),
153+
};
154+
155+
let (is_niche, tagged_discr) = if discr_len == 1 {
156+
// Special case where we only have a single tagged variant.
157+
// The untagged variant can't be contained in niche_variant's range in this case.
158+
// Thus the discriminant of the only tagged variant is 0 and its variant index
159+
// is the start of niche_variants.
160+
let is_niche = codegen_icmp_imm(fx, IntCC::Equal, tag, niche_start);
161161
let tagged_discr =
162162
fx.bcx.ins().iconst(cast_to, niche_variants.start().as_u32() as i64);
163-
(is_niche, tagged_discr, 0)
163+
(is_niche, tagged_discr)
164164
} else {
165-
// The special cases don't apply, so we'll have to go with
166-
// the general algorithm.
167-
let niche_start = match fx.bcx.func.dfg.value_type(tag) {
168-
types::I128 => {
169-
let lsb = fx.bcx.ins().iconst(types::I64, niche_start as u64 as i64);
170-
let msb =
171-
fx.bcx.ins().iconst(types::I64, (niche_start >> 64) as u64 as i64);
172-
fx.bcx.ins().iconcat(lsb, msb)
173-
}
174-
ty => fx.bcx.ins().iconst(ty, niche_start as i64),
175-
};
176-
let relative_discr = fx.bcx.ins().isub(tag, niche_start);
177-
let cast_tag = clif_intcast(fx, relative_discr, cast_to, false);
178-
let is_niche = crate::common::codegen_icmp_imm(
179-
fx,
180-
IntCC::UnsignedLessThanOrEqual,
181-
relative_discr,
182-
i128::from(relative_max),
183-
);
184-
(is_niche, cast_tag, niche_variants.start().as_u32() as u128)
185-
};
165+
// General case.
166+
let discr = fx.bcx.ins().isub(tag, niche_start);
167+
let tagged_discr = clif_intcast(fx, relative_discr, cast_to, false);
168+
if niche_variants.contains(&untagged_variant) {
169+
let is_niche = crate::common::codegen_icmp_imm(
170+
fx,
171+
IntCC::UnsignedLessThan,
172+
discr,
173+
i128::from(discr_len - 1),
174+
);
175+
let adj_untagged_idx = untagged_variant.index() - niche_variants.start().index();
176+
let untagged_delta = 1 + adj_untagged_idx;
177+
let untagged_delta = match cast_to {
178+
types::I128 => {
179+
let lsb = fx.bcx.ins().iconst(types::I64, untagged_delta as u64 as i64);
180+
let msb = fx.bcx.ins().iconst(types::I64, (untagged_delta >> 64) as u64 as i64);
181+
fx.bcx.ins().iconcat(lsb, msb)
182+
}
183+
ty => fx.bcx.ins().iconst(ty, untagged_delta as i64),
184+
};
185+
let tagged_discr = fx.bcx.ins().iadd(tagged_discr, untagged_delta);
186186

187-
let tagged_discr = if delta == 0 {
188-
tagged_discr
189-
} else {
190-
let delta = match cast_to {
191-
types::I128 => {
192-
let lsb = fx.bcx.ins().iconst(types::I64, delta as u64 as i64);
193-
let msb = fx.bcx.ins().iconst(types::I64, (delta >> 64) as u64 as i64);
194-
fx.bcx.ins().iconcat(lsb, msb)
195-
}
196-
ty => fx.bcx.ins().iconst(ty, delta as i64),
187+
let discr_len = match cast_to {
188+
types::I128 => {
189+
let lsb = fx.bcx.ins().iconst(types::I64, discr_len as u64 as i64);
190+
let msb = fx.bcx.ins().iconst(types::I64, (discr_len >> 64) as u64 as i64);
191+
fx.bcx.ins().iconcat(lsb, msb)
192+
}
193+
ty => fx.bcx.ins().iconst(ty, discr_len as i64),
194+
};
195+
let tagged_discr = fx.bcx.ins().urem(tagged_discr, discr_len);
196+
197+
let niche_variants_start = niche_variants.start().index();
198+
let niche_variants_start = match cast_to {
199+
types::I128 => {
200+
let lsb = fx.bcx.ins().iconst(types::I64, niche_variants_start as u64 as i64);
201+
let msb = fx.bcx.ins().iconst(types::I64, (niche_variants_start >> 64) as u64 as i64);
202+
fx.bcx.ins().iconcat(lsb, msb)
203+
}
204+
ty => fx.bcx.ins().iconst(ty, niche_variants_start as i64),
205+
};
206+
let tagged_discr = fx.bcx.ins().iadd(tagged_discr, niche_variants_start);
207+
(is_niche, tagged_discr)
208+
} else {
209+
let is_niche = crate::common::codegen_icmp_imm(
210+
fx,
211+
IntCC::UnsignedLessThan,
212+
discr,
213+
i128::from(discr_len - 1),
214+
);
215+
let niche_variants_start = niche_variants.start().index();
216+
let niche_variants_start = match cast_to {
217+
types::I128 => {
218+
let lsb = fx.bcx.ins().iconst(types::I64, niche_variants_start as u64 as i64);
219+
let msb = fx.bcx.ins().iconst(types::I64, (niche_variants_start >> 64) as u64 as i64);
220+
fx.bcx.ins().iconcat(lsb, msb)
221+
}
222+
ty => fx.bcx.ins().iconst(ty, niche_variants_start as i64),
223+
};
224+
let tagged_discr = fx.bcx.ins().iadd(tagged_discr, niche_variants_start);
225+
(is_niche, tagged_discr)
197226
};
198-
fx.bcx.ins().iadd(tagged_discr, delta)
199-
};
227+
}
200228

201229
let untagged_variant = if cast_to == types::I128 {
202230
let zero = fx.bcx.ins().iconst(types::I64, 0);

‎compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs

+10-3
Original file line numberDiff line numberDiff line change
@@ -391,9 +391,16 @@ fn compute_discriminant_value<'ll, 'tcx>(
391391

392392
DiscrResult::Range(min, max)
393393
} else {
394-
let value = (variant_index.as_u32() as u128)
395-
.wrapping_sub(niche_variants.start().as_u32() as u128)
396-
.wrapping_add(niche_start);
394+
let discr_len = niche_variants.end().index() - niche_variants.start().index() + 1;
395+
let adj_idx = variant_index.index() - niche_variants.start().index();
396+
397+
let discr = if niche_variants.contains(&untagged_variant) {
398+
let adj_untagged_idx = untagged_variant.index() - niche_variants.start().index();
399+
(adj_idx + discr_len - adj_untagged_idx) % discr_len - 1
400+
} else {
401+
adj_idx
402+
};
403+
let value = (discr as u128).wrapping_add(niche_start);
397404
let value = tag.size(cx).truncate(value);
398405
DiscrResult::Value(value)
399406
}

0 commit comments

Comments
 (0)