Skip to content

Commit 725f521

Browse files
beetreesbeetrees
committed
Fix RISC-V C function ABI when passing/returning structs containing floats
1 parent 82eb03e commit 725f521

File tree

13 files changed

+514
-131
lines changed

13 files changed

+514
-131
lines changed

‎compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

Lines changed: 42 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,18 @@ fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -
3939
param
4040
}
4141

42-
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
42+
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[(Size, AbiParam); 2]> {
43+
if let Some(offset_from_start) = cast.rest_offset {
44+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
45+
assert_eq!(cast.rest.unit.size, cast.rest.total);
46+
let first = cast.prefix[0].unwrap();
47+
let second = cast.rest.unit;
48+
return smallvec![
49+
(Size::ZERO, reg_to_abi_param(first)),
50+
(offset_from_start, reg_to_abi_param(second))
51+
];
52+
}
53+
4354
let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 {
4455
(0, 0)
4556
} else {
@@ -54,25 +65,31 @@ fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
5465
// different types in Cranelift IR. Instead a single array of primitive types is used.
5566

5667
// Create list of fields in the main structure
57-
let mut args = cast
68+
let args = cast
5869
.prefix
5970
.iter()
6071
.flatten()
6172
.map(|&reg| reg_to_abi_param(reg))
62-
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
63-
.collect::<SmallVec<_>>();
73+
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)));
6474

6575
// Append final integer
66-
if rem_bytes != 0 {
76+
let args = args.chain(if rem_bytes != 0 {
6777
// Only integers can be really split further.
6878
assert_eq!(cast.rest.unit.kind, RegKind::Integer);
69-
args.push(reg_to_abi_param(Reg {
70-
kind: RegKind::Integer,
71-
size: Size::from_bytes(rem_bytes),
72-
}));
79+
Some(reg_to_abi_param(Reg { kind: RegKind::Integer, size: Size::from_bytes(rem_bytes) }))
80+
} else {
81+
None
82+
});
83+
84+
let mut res = SmallVec::new();
85+
let mut offset = Size::ZERO;
86+
87+
for arg in args {
88+
res.push((offset, arg));
89+
offset += Size::from_bytes(arg.value_type.bytes());
7390
}
7491

75-
args
92+
res
7693
}
7794

7895
impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
@@ -103,7 +120,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
103120
},
104121
PassMode::Cast { ref cast, pad_i32 } => {
105122
assert!(!pad_i32, "padding support not yet implemented");
106-
cast_target_to_abi_params(cast)
123+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
107124
}
108125
PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
109126
if on_stack {
@@ -149,9 +166,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
149166
}
150167
_ => unreachable!("{:?}", self.layout.backend_repr),
151168
},
152-
PassMode::Cast { ref cast, .. } => {
153-
(None, cast_target_to_abi_params(cast).into_iter().collect())
154-
}
169+
PassMode::Cast { ref cast, .. } => (
170+
None,
171+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(),
172+
),
155173
PassMode::Indirect { attrs: _, meta_attrs: None, on_stack } => {
156174
assert!(!on_stack);
157175
(Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
@@ -170,12 +188,14 @@ pub(super) fn to_casted_value<'tcx>(
170188
) -> SmallVec<[Value; 2]> {
171189
let (ptr, meta) = arg.force_stack(fx);
172190
assert!(meta.is_none());
173-
let mut offset = 0;
174191
cast_target_to_abi_params(cast)
175192
.into_iter()
176-
.map(|param| {
177-
let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new());
178-
offset += i64::from(param.value_type.bytes());
193+
.map(|(offset, param)| {
194+
let val = ptr.offset_i64(fx, offset.bytes() as i64).load(
195+
fx,
196+
param.value_type,
197+
MemFlags::new(),
198+
);
179199
val
180200
})
181201
.collect()
@@ -188,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>(
188208
cast: &CastTarget,
189209
) -> CValue<'tcx> {
190210
let abi_params = cast_target_to_abi_params(cast);
191-
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
211+
let abi_param_size: u32 = abi_params.iter().map(|(_, param)| param.value_type.bytes()).sum();
192212
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
193213
let ptr = fx.create_stack_slot(
194214
// Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`.
@@ -197,16 +217,13 @@ pub(super) fn from_casted_value<'tcx>(
197217
std::cmp::max(abi_param_size, layout_size),
198218
u32::try_from(layout.align.abi.bytes()).unwrap(),
199219
);
200-
let mut offset = 0;
201220
let mut block_params_iter = block_params.iter().copied();
202-
for param in abi_params {
203-
let val = ptr.offset_i64(fx, offset).store(
221+
for (offset, _) in abi_params {
222+
ptr.offset_i64(fx, offset.bytes() as i64).store(
204223
fx,
205224
block_params_iter.next().unwrap(),
206225
MemFlags::new(),
207-
);
208-
offset += i64::from(param.value_type.bytes());
209-
val
226+
)
210227
}
211228
assert_eq!(block_params_iter.next(), None, "Leftover block param");
212229
CValue::by_ref(ptr, layout)

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -551,7 +551,23 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
551551
bx.lifetime_start(llscratch, scratch_size);
552552

553553
// ... where we first store the value...
554-
bx.store(val, llscratch, scratch_align);
554+
if let Some(offset_from_start) = cast.rest_offset {
555+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
556+
assert_eq!(cast.rest.unit.size, cast.rest.total);
557+
assert!(cast.prefix[0].is_some());
558+
let first = bx.extract_value(val, 0);
559+
let second = bx.extract_value(val, 1);
560+
bx.store(first, llscratch, scratch_align);
561+
let second_ptr =
562+
bx.inbounds_ptradd(llscratch, bx.const_usize(offset_from_start.bytes()));
563+
bx.store(
564+
second,
565+
second_ptr,
566+
scratch_align.restrict_for_offset(offset_from_start),
567+
);
568+
} else {
569+
bx.store(val, llscratch, scratch_align);
570+
};
555571

556572
// ... and then memcpy it to the intended destination.
557573
bx.memcpy(

‎compiler/rustc_codegen_llvm/src/abi.rs

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,23 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
234234
let llscratch = bx.alloca(scratch_size, scratch_align);
235235
bx.lifetime_start(llscratch, scratch_size);
236236
// ...store the value...
237-
bx.store(val, llscratch, scratch_align);
237+
if let Some(offset_from_start) = cast.rest_offset {
238+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
239+
assert_eq!(cast.rest.unit.size, cast.rest.total);
240+
assert!(cast.prefix[0].is_some());
241+
let first = bx.extract_value(val, 0);
242+
let second = bx.extract_value(val, 1);
243+
bx.store(first, llscratch, scratch_align);
244+
let second_ptr =
245+
bx.inbounds_ptradd(llscratch, bx.const_usize(offset_from_start.bytes()));
246+
bx.store(
247+
second,
248+
second_ptr,
249+
scratch_align.restrict_for_offset(offset_from_start),
250+
);
251+
} else {
252+
bx.store(val, llscratch, scratch_align);
253+
};
238254
// ... and then memcpy it to the intended destination.
239255
bx.memcpy(
240256
dst.val.llval,

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -559,7 +559,25 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
559559
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
560560
};
561561
let ty = bx.cast_backend_type(cast_ty);
562-
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
562+
if let Some(offset_from_start) = cast_ty.rest_offset {
563+
assert!(cast_ty.prefix[1..].iter().all(|p| p.is_none()));
564+
assert_eq!(cast_ty.rest.unit.size, cast_ty.rest.total);
565+
let first_ty = bx.reg_backend_type(&cast_ty.prefix[0].unwrap());
566+
let second_ty = bx.reg_backend_type(&cast_ty.rest.unit);
567+
let first = bx.load(first_ty, llslot, self.fn_abi.ret.layout.align.abi);
568+
let second_ptr =
569+
bx.inbounds_ptradd(llslot, bx.const_usize(offset_from_start.bytes()));
570+
let second = bx.load(
571+
second_ty,
572+
second_ptr,
573+
self.fn_abi.ret.layout.align.abi.restrict_for_offset(offset_from_start),
574+
);
575+
let res = bx.cx().const_poison(ty);
576+
let res = bx.insert_value(res, first, 0);
577+
bx.insert_value(res, second, 1)
578+
} else {
579+
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
580+
}
563581
}
564582
};
565583
bx.ret(llval);
@@ -1588,7 +1606,25 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
15881606
);
15891607
// ...and then load it with the ABI type.
15901608
let cast_ty = bx.cast_backend_type(cast);
1591-
llval = bx.load(cast_ty, llscratch, scratch_align);
1609+
llval = if let Some(offset_from_start) = cast.rest_offset {
1610+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
1611+
assert_eq!(cast.rest.unit.size, cast.rest.total);
1612+
let first_ty = bx.reg_backend_type(&cast.prefix[0].unwrap());
1613+
let second_ty = bx.reg_backend_type(&cast.rest.unit);
1614+
let first = bx.load(first_ty, llscratch, self.fn_abi.ret.layout.align.abi);
1615+
let second_ptr =
1616+
bx.inbounds_ptradd(llscratch, bx.const_usize(offset_from_start.bytes()));
1617+
let second = bx.load(
1618+
second_ty,
1619+
second_ptr,
1620+
scratch_align.restrict_for_offset(offset_from_start),
1621+
);
1622+
let res = bx.cx().const_poison(cast_ty);
1623+
let res = bx.insert_value(res, first, 0);
1624+
bx.insert_value(res, second, 1)
1625+
} else {
1626+
bx.load(cast_ty, llscratch, scratch_align)
1627+
};
15921628
bx.lifetime_end(llscratch, scratch_size);
15931629
} else {
15941630
// We can't use `PlaceRef::load` here because the argument

‎compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
254254
}
255255
PassMode::Cast { ref cast, .. } => {
256256
debug!("alloc: {:?} (return place) -> place", local);
257-
let size = cast.size(&start_bx);
257+
let size = cast.size(&start_bx).max(layout.size);
258258
return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
259259
}
260260
_ => {}

‎compiler/rustc_target/src/callconv/mips64.rs

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@ use rustc_abi::{
22
BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Size, TyAbiInterface,
33
};
44

5-
use crate::callconv::{
6-
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, Uniform,
7-
};
5+
use crate::callconv::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Uniform};
86

97
fn extend_integer_width_mips<Ty>(arg: &mut ArgAbi<'_, Ty>, bits: u64) {
108
// Always sign extend u32 values on 64-bit mips
@@ -140,16 +138,7 @@ where
140138

141139
// Extract first 8 chunks as the prefix
142140
let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
143-
arg.cast_to(CastTarget {
144-
prefix,
145-
rest: Uniform::new(Reg::i64(), rest_size),
146-
attrs: ArgAttributes {
147-
regular: ArgAttribute::default(),
148-
arg_ext: ArgExtension::None,
149-
pointee_size: Size::ZERO,
150-
pointee_align: None,
151-
},
152-
});
141+
arg.cast_to(CastTarget::prefixed(prefix, Uniform::new(Reg::i64(), rest_size)));
153142
}
154143

155144
pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)

‎compiler/rustc_target/src/callconv/mod.rs

Lines changed: 57 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,17 @@ impl ArgAttributes {
196196
}
197197
}
198198

199+
impl From<ArgAttribute> for ArgAttributes {
200+
fn from(value: ArgAttribute) -> Self {
201+
Self {
202+
regular: value,
203+
arg_ext: ArgExtension::None,
204+
pointee_size: Size::ZERO,
205+
pointee_align: None,
206+
}
207+
}
208+
}
209+
199210
/// An argument passed entirely registers with the
200211
/// same kind (e.g., HFA / HVA on PPC64 and AArch64).
201212
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
@@ -250,6 +261,9 @@ impl Uniform {
250261
#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
251262
pub struct CastTarget {
252263
pub prefix: [Option<Reg>; 8],
264+
/// The offset of `rest` from the start of the value. Currently only implemented for a `Reg`
265+
/// pair created by the `offset_pair` method.
266+
pub rest_offset: Option<Size>,
253267
pub rest: Uniform,
254268
pub attrs: ArgAttributes,
255269
}
@@ -262,42 +276,45 @@ impl From<Reg> for CastTarget {
262276

263277
impl From<Uniform> for CastTarget {
264278
fn from(uniform: Uniform) -> CastTarget {
265-
CastTarget {
266-
prefix: [None; 8],
267-
rest: uniform,
268-
attrs: ArgAttributes {
269-
regular: ArgAttribute::default(),
270-
arg_ext: ArgExtension::None,
271-
pointee_size: Size::ZERO,
272-
pointee_align: None,
273-
},
274-
}
279+
Self::prefixed([None; 8], uniform)
275280
}
276281
}
277282

278283
impl CastTarget {
279-
pub fn pair(a: Reg, b: Reg) -> CastTarget {
280-
CastTarget {
284+
pub fn prefixed(prefix: [Option<Reg>; 8], rest: Uniform) -> Self {
285+
Self { prefix, rest_offset: None, rest, attrs: ArgAttributes::new() }
286+
}
287+
288+
pub fn offset_pair(a: Reg, offset_from_start: Size, b: Reg) -> Self {
289+
Self {
281290
prefix: [Some(a), None, None, None, None, None, None, None],
282-
rest: Uniform::from(b),
283-
attrs: ArgAttributes {
284-
regular: ArgAttribute::default(),
285-
arg_ext: ArgExtension::None,
286-
pointee_size: Size::ZERO,
287-
pointee_align: None,
288-
},
291+
rest_offset: Some(offset_from_start),
292+
rest: b.into(),
293+
attrs: ArgAttributes::new(),
289294
}
290295
}
291296

297+
pub fn with_attrs(mut self, attrs: ArgAttributes) -> Self {
298+
self.attrs = attrs;
299+
self
300+
}
301+
302+
pub fn pair(a: Reg, b: Reg) -> CastTarget {
303+
Self::prefixed([Some(a), None, None, None, None, None, None, None], Uniform::from(b))
304+
}
305+
292306
/// When you only access the range containing valid data, you can use this unaligned size;
293307
/// otherwise, use the safer `size` method.
294308
pub fn unaligned_size<C: HasDataLayout>(&self, _cx: &C) -> Size {
295309
// Prefix arguments are passed in specific designated registers
296-
let prefix_size = self
297-
.prefix
298-
.iter()
299-
.filter_map(|x| x.map(|reg| reg.size))
300-
.fold(Size::ZERO, |acc, size| acc + size);
310+
let prefix_size = if let Some(offset_from_start) = self.rest_offset {
311+
offset_from_start
312+
} else {
313+
self.prefix
314+
.iter()
315+
.filter_map(|x| x.map(|reg| reg.size))
316+
.fold(Size::ZERO, |acc, size| acc + size)
317+
};
301318
// Remaining arguments are passed in chunks of the unit size
302319
let rest_size =
303320
self.rest.unit.size * self.rest.total.bytes().div_ceil(self.rest.unit.size.bytes());
@@ -321,9 +338,22 @@ impl CastTarget {
321338
/// Checks if these two `CastTarget` are equal enough to be considered "the same for all
322339
/// function call ABIs".
323340
pub fn eq_abi(&self, other: &Self) -> bool {
324-
let CastTarget { prefix: prefix_l, rest: rest_l, attrs: attrs_l } = self;
325-
let CastTarget { prefix: prefix_r, rest: rest_r, attrs: attrs_r } = other;
326-
prefix_l == prefix_r && rest_l == rest_r && attrs_l.eq_abi(attrs_r)
341+
let CastTarget {
342+
prefix: prefix_l,
343+
rest_offset: rest_offset_l,
344+
rest: rest_l,
345+
attrs: attrs_l,
346+
} = self;
347+
let CastTarget {
348+
prefix: prefix_r,
349+
rest_offset: rest_offset_r,
350+
rest: rest_r,
351+
attrs: attrs_r,
352+
} = other;
353+
prefix_l == prefix_r
354+
&& rest_offset_l == rest_offset_r
355+
&& rest_l == rest_r
356+
&& attrs_l.eq_abi(attrs_r)
327357
}
328358
}
329359

0 commit comments

Comments
 (0)