Skip to content

Commit c9d478a

Browse files
AbhayKanhereAbhayKanhere
committed
[CodeGen][AArch64] ptrauth intrinsic to safely construct relative pointer for swift coroutines
 A ptrauth intrinsic for swift co-routine support that allows creation of signed pointer from offset stored at address relative to the pointer. Following C-like pseudo code (ignoring keys,discriminators) explains its operation: let rawptr = PACauth(inputptr); return PACsign( rawptr + *(int32*)(rawptr+addend) ) What: Authenticate a signed pointer, load a 32bit value at offset 'addend' from pointer, add this value to pointer, sign this new pointer. builtin: __builtin_ptrauth_auth_load_relative_and_sign intrinsic: ptrauth_auth_resign_load_relative note: conflicts resolved llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp due to upstream change in args for emitPtrauthAuthResign
1 parent 9912ccb commit c9d478a

File tree

10 files changed

+732
-21
lines changed

10 files changed

+732
-21
lines changed

‎clang/include/clang/Basic/Builtins.td

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4643,6 +4643,12 @@ def PtrauthAuthAndResign : Builtin {
46434643
let Prototype = "void*(void*,int,void*,int,void*)";
46444644
}
46454645

4646+
def PtrauthAuthLoadRelativeAndSign : Builtin {
4647+
let Spellings = ["__builtin_ptrauth_auth_load_relative_and_sign"];
4648+
let Attributes = [CustomTypeChecking, NoThrow];
4649+
let Prototype = "void*(void*,int,void*,int,void*,ptrdiff_t)";
4650+
}
4651+
46464652
def PtrauthAuth : Builtin {
46474653
let Spellings = ["__builtin_ptrauth_auth"];
46484654
let Attributes = [CustomTypeChecking, NoThrow];

‎clang/lib/CodeGen/CGBuiltin.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5584,12 +5584,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
55845584

55855585
case Builtin::BI__builtin_ptrauth_auth:
55865586
case Builtin::BI__builtin_ptrauth_auth_and_resign:
5587+
case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
55875588
case Builtin::BI__builtin_ptrauth_blend_discriminator:
55885589
case Builtin::BI__builtin_ptrauth_sign_generic_data:
55895590
case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
55905591
case Builtin::BI__builtin_ptrauth_strip: {
55915592
// Emit the arguments.
5592-
SmallVector<llvm::Value *, 5> Args;
5593+
SmallVector<llvm::Value *, 6> Args;
55935594
for (auto argExpr : E->arguments())
55945595
Args.push_back(EmitScalarExpr(argExpr));
55955596

@@ -5600,6 +5601,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
56005601

56015602
switch (BuiltinID) {
56025603
case Builtin::BI__builtin_ptrauth_auth_and_resign:
5604+
case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
56035605
if (Args[4]->getType()->isPointerTy())
56045606
Args[4] = Builder.CreatePtrToInt(Args[4], IntPtrTy);
56055607
[[fallthrough]];
@@ -5627,6 +5629,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
56275629
return Intrinsic::ptrauth_auth;
56285630
case Builtin::BI__builtin_ptrauth_auth_and_resign:
56295631
return Intrinsic::ptrauth_resign;
5632+
case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
5633+
return Intrinsic::ptrauth_resign_load_relative;
56305634
case Builtin::BI__builtin_ptrauth_blend_discriminator:
56315635
return Intrinsic::ptrauth_blend;
56325636
case Builtin::BI__builtin_ptrauth_sign_generic_data:

‎clang/lib/Headers/ptrauth.h

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,31 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
178178
__builtin_ptrauth_auth_and_resign(__value, __old_key, __old_data, __new_key, \
179179
__new_data)
180180

181+
/* Authenticate a pointer using one scheme, load 32bit value at offset addend
182+
from the pointer, and add this value to the pointer, sign using specified
183+
scheme.
184+
185+
If the result is subsequently authenticated using the new scheme, that
186+
authentication is guaranteed to fail if and only if the initial
187+
authentication failed.
188+
189+
The value must be an expression of pointer type.
190+
The key must be a constant expression of type ptrauth_key.
191+
The extra data must be an expression of pointer or integer type;
192+
if an integer, it will be coerced to ptrauth_extra_data_t.
193+
The addend must be an immediate ptrdiff_t value.
194+
The result will have the same type as the original value.
195+
196+
This operation is guaranteed to not leave the intermediate value
197+
available for attack before it is re-signed.
198+
199+
Do not pass a null pointer to this function. A null pointer
200+
will not successfully authenticate. */
201+
#define ptrauth_auth_load_relative_and_sign(__value, __old_key, __old_data, \
202+
__new_key, __new_data, __addend) \
203+
__builtin_ptrauth_auth_load_relative_and_sign( \
204+
__value, __old_key, __old_data, __new_key, __new_data, __addend)
205+
181206
/* Authenticate a pointer using one scheme and resign it as a C
182207
function pointer.
183208
@@ -380,7 +405,6 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
380405
((ptrauth_generic_signature_t)0); \
381406
})
382407

383-
384408
#define ptrauth_cxx_vtable_pointer(key, address_discrimination, \
385409
extra_discrimination...)
386410

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2818,6 +2818,19 @@ def int_ptrauth_resign : Intrinsic<[llvm_i64_ty],
28182818
[IntrNoMem, ImmArg<ArgIndex<1>>,
28192819
ImmArg<ArgIndex<3>>]>;
28202820

2821+
// Authenticate a signed pointer, load 32bit value at offset from pointer, add
2822+
// both, and sign it. The second (key) and third (discriminator) arguments
2823+
// specify the signing schema used for authenticating. The fourth and fifth
2824+
// arguments specify the schema used for signing. The sixth argument is addend
2825+
// added to pointer to load the relative offset. The signature must be valid.
2826+
// This is a combined form of int_ptrauth_resign for relative pointers
2827+
def int_ptrauth_resign_load_relative
2828+
: Intrinsic<[llvm_i64_ty],
2829+
[llvm_i64_ty, llvm_i32_ty, llvm_i64_ty, llvm_i32_ty,
2830+
llvm_i64_ty, llvm_i64_ty],
2831+
[IntrReadMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<3>>,
2832+
ImmArg<ArgIndex<5>>]>;
2833+
28212834
// Strip the embedded signature out of a signed pointer.
28222835
// The second argument specifies the key.
28232836
// This behaves like @llvm.ptrauth.auth, but doesn't require the signature to

‎llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

Lines changed: 83 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -168,13 +168,14 @@ class AArch64AsmPrinter : public AsmPrinter {
168168
// Check authenticated LR before tail calling.
169169
void emitPtrauthTailCallHardening(const MachineInstr *TC);
170170

171-
// Emit the sequence for AUT or AUTPAC.
171+
// Emit the sequence for AUT or AUTPAC. Addend if AUTRELLOADPAC
172172
void emitPtrauthAuthResign(Register AUTVal, AArch64PACKey::ID AUTKey,
173173
uint64_t AUTDisc,
174174
const MachineOperand *AUTAddrDisc,
175175
Register Scratch,
176176
std::optional<AArch64PACKey::ID> PACKey,
177-
uint64_t PACDisc, Register PACAddrDisc);
177+
uint64_t PACDisc, Register PACAddrDisc,
178+
std::optional<uint64_t> Addend);
178179

179180
// Emit the sequence to compute the discriminator.
180181
//
@@ -2075,9 +2076,9 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(
20752076
Register AUTVal, AArch64PACKey::ID AUTKey, uint64_t AUTDisc,
20762077
const MachineOperand *AUTAddrDisc, Register Scratch,
20772078
std::optional<AArch64PACKey::ID> PACKey, uint64_t PACDisc,
2078-
Register PACAddrDisc) {
2079+
Register PACAddrDisc, std::optional<uint64_t> OptAddend) {
20792080
const bool IsAUTPAC = PACKey.has_value();
2080-
2081+
const bool HasLoad = OptAddend.has_value();
20812082
// We expand AUT/AUTPAC into a sequence of the form
20822083
//
20832084
// ; authenticate x16
@@ -2148,12 +2149,76 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(
21482149
}
21492150

21502151
// We already emitted unchecked and checked-but-non-trapping AUTs.
2151-
// That left us with trapping AUTs, and AUTPACs.
2152+
// That left us with trapping AUTs, and AUTPA/AUTRELLOADPACs.
21522153
// Trapping AUTs don't need PAC: we're done.
21532154
if (!IsAUTPAC)
21542155
return;
21552156

2156-
// Compute pac discriminator
2157+
if (HasLoad) {
2158+
int64_t Addend = *OptAddend;
2159+
// incoming rawpointer in X16, X17 is not live at this point.
2160+
// LDSRWpre x17, x16, simm9 ; note: x16+simm9 used later.
2161+
if (isInt<9>(Addend)) {
2162+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWpre)
2163+
.addReg(AArch64::X16)
2164+
.addReg(AArch64::X17)
2165+
.addReg(AArch64::X16)
2166+
.addImm(/*simm9:*/ Addend));
2167+
} else {
2168+
// x16 = x16 + Addend computation has 2 variants
2169+
if (isUInt<24>(Addend)) {
2170+
// variant 1: add x16, x16, Addend >> shift12 ls shift12
2171+
// This can take upto 2 instructions.
2172+
for (int BitPos = 0; BitPos != 24 && (Addend >> BitPos); BitPos += 12) {
2173+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri)
2174+
.addReg(AArch64::X16)
2175+
.addReg(AArch64::X16)
2176+
.addImm((Addend >> BitPos) & 0xfff)
2177+
.addImm(AArch64_AM::getShifterImm(
2178+
AArch64_AM::LSL, BitPos)));
2179+
}
2180+
} else {
2181+
// variant 2: accumulate constant in X17 16 bits at a time, and add to
2182+
// X16 This can take 2-5 instructions.
2183+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
2184+
.addReg(AArch64::X17)
2185+
.addImm(Addend & 0xffff)
2186+
.addImm(AArch64_AM::getShifterImm(
2187+
AArch64_AM::LSL, 0)));
2188+
2189+
for (int Offset = 16; Offset < 64; Offset += 16) {
2190+
uint16_t Fragment = static_cast<uint16_t>(Addend >> Offset);
2191+
if (!Fragment)
2192+
continue;
2193+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
2194+
.addReg(AArch64::X17)
2195+
.addReg(AArch64::X17)
2196+
.addImm(Fragment)
2197+
.addImm(/*shift:*/ Offset));
2198+
}
2199+
// addx x16, x16, x17
2200+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
2201+
.addReg(AArch64::X16)
2202+
.addReg(AArch64::X16)
2203+
.addReg(AArch64::X17)
2204+
.addImm(0));
2205+
}
2206+
// ldrsw x17,x16(0)
2207+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWui)
2208+
.addReg(AArch64::X17)
2209+
.addReg(AArch64::X16)
2210+
.addImm(0));
2211+
}
2212+
// addx x16, x16, x17
2213+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
2214+
.addReg(AArch64::X16)
2215+
.addReg(AArch64::X16)
2216+
.addReg(AArch64::X17)
2217+
.addImm(0));
2218+
2219+
} /* HasLoad == true */
2220+
2221+
// Compute pac discriminator into x17
21572222
assert(isUInt<16>(PACDisc));
21582223
Register PACDiscReg =
21592224
emitPtrauthDiscriminator(PACDisc, PACAddrDisc, Scratch);
@@ -2872,22 +2937,31 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
28722937
emitPtrauthAuthResign(AArch64::X16,
28732938
(AArch64PACKey::ID)MI->getOperand(0).getImm(),
28742939
MI->getOperand(1).getImm(), &MI->getOperand(2),
2875-
AArch64::X17, std::nullopt, 0, 0);
2940+
AArch64::X17, std::nullopt, 0, 0, std::nullopt);
28762941
return;
28772942

28782943
case AArch64::AUTxMxN:
28792944
emitPtrauthAuthResign(MI->getOperand(0).getReg(),
28802945
(AArch64PACKey::ID)MI->getOperand(3).getImm(),
28812946
MI->getOperand(4).getImm(), &MI->getOperand(5),
2882-
MI->getOperand(1).getReg(), std::nullopt, 0, 0);
2947+
MI->getOperand(1).getReg(), std::nullopt, 0, 0,
2948+
std::nullopt);
28832949
return;
28842950

2951+
case AArch64::AUTRELLOADPAC:
2952+
emitPtrauthAuthResign(
2953+
AArch64::X16, (AArch64PACKey::ID)MI->getOperand(0).getImm(),
2954+
MI->getOperand(1).getImm(), &MI->getOperand(2), AArch64::X17,
2955+
(AArch64PACKey::ID)MI->getOperand(3).getImm(),
2956+
MI->getOperand(4).getImm(), MI->getOperand(5).getReg(),
2957+
MI->getOperand(6).getImm());
2958+
return;
28852959
case AArch64::AUTPAC:
28862960
emitPtrauthAuthResign(
28872961
AArch64::X16, (AArch64PACKey::ID)MI->getOperand(0).getImm(),
28882962
MI->getOperand(1).getImm(), &MI->getOperand(2), AArch64::X17,
28892963
(AArch64PACKey::ID)MI->getOperand(3).getImm(),
2890-
MI->getOperand(4).getImm(), MI->getOperand(5).getReg());
2964+
MI->getOperand(4).getImm(), MI->getOperand(5).getReg(), std::nullopt);
28912965
return;
28922966

28932967
case AArch64::LOADauthptrstatic:

‎llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1552,12 +1552,15 @@ void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
15521552

15531553
void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
15541554
SDLoc DL(N);
1555-
// IntrinsicID is operand #0
1556-
SDValue Val = N->getOperand(1);
1557-
SDValue AUTKey = N->getOperand(2);
1558-
SDValue AUTDisc = N->getOperand(3);
1559-
SDValue PACKey = N->getOperand(4);
1560-
SDValue PACDisc = N->getOperand(5);
1555+
// IntrinsicID is operand #0, if W_CHAIN it is #1
1556+
int OffsetBase = N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 1 : 0;
1557+
SDValue Val = N->getOperand(OffsetBase + 1);
1558+
SDValue AUTKey = N->getOperand(OffsetBase + 2);
1559+
SDValue AUTDisc = N->getOperand(OffsetBase + 3);
1560+
SDValue PACKey = N->getOperand(OffsetBase + 4);
1561+
SDValue PACDisc = N->getOperand(OffsetBase + 5);
1562+
uint32_t IntNum = N->getConstantOperandVal(OffsetBase + 0);
1563+
bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
15611564

15621565
unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
15631566
unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
@@ -1576,11 +1579,22 @@ void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
15761579
SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
15771580
AArch64::X16, Val, SDValue());
15781581

1579-
SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1580-
PACConstDisc, PACAddrDisc, X16Copy.getValue(1)};
1582+
if (HasLoad) {
1583+
SDValue Addend = N->getOperand(OffsetBase + 6);
1584+
SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
1585+
PACKey, PACConstDisc, PACAddrDisc,
1586+
Addend, X16Copy.getValue(1)};
15811587

1582-
SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC, DL, MVT::i64, Ops);
1583-
ReplaceNode(N, AUTPAC);
1588+
SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(AArch64::AUTRELLOADPAC, DL,
1589+
MVT::i64, MVT::Other, Ops);
1590+
ReplaceNode(N, AUTRELLOADPAC);
1591+
} else {
1592+
SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1593+
PACConstDisc, PACAddrDisc, X16Copy.getValue(1)};
1594+
1595+
SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC, DL, MVT::i64, Ops);
1596+
ReplaceNode(N, AUTPAC);
1597+
}
15841598
}
15851599

15861600
bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
@@ -5777,6 +5791,9 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
57775791
{AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
57785792
SelectCVTIntrinsicFP8(Node, 2, Opc);
57795793
return;
5794+
case Intrinsic::ptrauth_resign_load_relative:
5795+
SelectPtrauthResign(Node);
5796+
return;
57805797
}
57815798
} break;
57825799
case ISD::INTRINSIC_WO_CHAIN: {

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2176,6 +2176,26 @@ let Predicates = [HasPAuth] in {
21762176
let Uses = [X16];
21772177
}
21782178

2179+
// Similiar to AUTPAC, except a 32bit value is loaded at Addend offset from
2180+
// pointer and this value is added to the pointer before signing. This
2181+
// directly manipulates x16/x17, which are the only registers the OS
2182+
// guarantees are safe to use for sensitive operations.
2183+
def AUTRELLOADPAC
2184+
: Pseudo<(outs),
2185+
(ins i32imm:$AUTKey, i64imm:$AUTDisc, GPR64:$AUTAddrDisc,
2186+
i32imm:$PACKey, i64imm:$PACDisc, GPR64noip:$PACAddrDisc,
2187+
i64imm:$Addend),
2188+
[]>,
2189+
Sched<[WriteI, ReadI]> {
2190+
let isCodeGenOnly = 1;
2191+
let hasSideEffects = 1;
2192+
let mayStore = 0;
2193+
let mayLoad = 1;
2194+
let Size = 84;
2195+
let Defs = [X16, X17, NZCV];
2196+
let Uses = [X16];
2197+
}
2198+
21792199
// Materialize a signed global address, with adrp+add and PAC.
21802200
def MOVaddrPAC : Pseudo<(outs),
21812201
(ins i64imm:$Addr, i32imm:$Key,

‎llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6665,6 +6665,42 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
66656665
constrainSelectedInstRegOperands(*Memset, TII, TRI, RBI);
66666666
break;
66676667
}
6668+
case Intrinsic::ptrauth_resign_load_relative: {
6669+
Register DstReg = I.getOperand(0).getReg();
6670+
Register ValReg = I.getOperand(2).getReg();
6671+
uint64_t AUTKey = I.getOperand(3).getImm();
6672+
Register AUTDisc = I.getOperand(4).getReg();
6673+
uint64_t PACKey = I.getOperand(5).getImm();
6674+
Register PACDisc = I.getOperand(6).getReg();
6675+
int64_t Addend = I.getOperand(7).getImm();
6676+
6677+
Register AUTAddrDisc = AUTDisc;
6678+
uint16_t AUTConstDiscC = 0;
6679+
std::tie(AUTConstDiscC, AUTAddrDisc) =
6680+
extractPtrauthBlendDiscriminators(AUTDisc, MRI);
6681+
6682+
Register PACAddrDisc = PACDisc;
6683+
uint16_t PACConstDiscC = 0;
6684+
std::tie(PACConstDiscC, PACAddrDisc) =
6685+
extractPtrauthBlendDiscriminators(PACDisc, MRI);
6686+
6687+
MIB.buildCopy({AArch64::X16}, {ValReg});
6688+
6689+
MIB.buildInstr(AArch64::AUTRELLOADPAC)
6690+
.addImm(AUTKey)
6691+
.addImm(AUTConstDiscC)
6692+
.addUse(AUTAddrDisc)
6693+
.addImm(PACKey)
6694+
.addImm(PACConstDiscC)
6695+
.addUse(PACAddrDisc)
6696+
.addImm(Addend)
6697+
.constrainAllUses(TII, TRI, RBI);
6698+
MIB.buildCopy({DstReg}, Register(AArch64::X16));
6699+
6700+
RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
6701+
I.eraseFromParent();
6702+
return true;
6703+
}
66686704
}
66696705

66706706
I.eraseFromParent();

‎llvm/lib/Transforms/Utils/Local.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,7 @@ bool llvm::wouldInstructionBeTriviallyDead(const Instruction *I,
457457
case Intrinsic::wasm_trunc_unsigned:
458458
case Intrinsic::ptrauth_auth:
459459
case Intrinsic::ptrauth_resign:
460+
case Intrinsic::ptrauth_resign_load_relative:
460461
return true;
461462
default:
462463
return false;

0 commit comments

Comments
 (0)