@@ -488,13 +488,36 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
488
488
CGF.Builder .SetInsertPoint (ContBB);
489
489
}
490
490
491
+ // / Duplicate the atomic min/max operation in conventional IR for the builtin
492
+ // / variants that return the new rather than the original value.
493
+ static llvm::Value *EmitPostAtomicMinMax (CGBuilderTy &Builder,
494
+ AtomicExpr::AtomicOp Op,
495
+ bool IsSigned,
496
+ llvm::Value *OldVal,
497
+ llvm::Value *RHS) {
498
+ llvm::CmpInst::Predicate Pred;
499
+ switch (Op) {
500
+ default :
501
+ llvm_unreachable (" Unexpected min/max operation" );
502
+ case AtomicExpr::AO__atomic_max_fetch:
503
+ Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
504
+ break ;
505
+ case AtomicExpr::AO__atomic_min_fetch:
506
+ Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
507
+ break ;
508
+ }
509
+ llvm::Value *Cmp = Builder.CreateICmp (Pred, OldVal, RHS, " tst" );
510
+ return Builder.CreateSelect (Cmp, OldVal, RHS, " newval" );
511
+ }
512
+
491
513
static void EmitAtomicOp (CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
492
514
Address Ptr , Address Val1, Address Val2,
493
515
llvm::Value *IsWeak, llvm::Value *FailureOrder,
494
516
uint64_t Size , llvm::AtomicOrdering Order,
495
517
llvm::SyncScope::ID Scope) {
496
518
llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
497
- llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0 ;
519
+ bool PostOpMinMax = false ;
520
+ unsigned PostOp = 0 ;
498
521
499
522
switch (E->getOp ()) {
500
523
case AtomicExpr::AO__c11_atomic_init:
@@ -588,12 +611,20 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
588
611
Op = llvm::AtomicRMWInst::Sub;
589
612
break ;
590
613
614
+ case AtomicExpr::AO__atomic_min_fetch:
615
+ PostOpMinMax = true ;
616
+ LLVM_FALLTHROUGH;
617
+ case AtomicExpr::AO__c11_atomic_fetch_min:
591
618
case AtomicExpr::AO__opencl_atomic_fetch_min:
592
619
case AtomicExpr::AO__atomic_fetch_min:
593
620
Op = E->getValueType ()->isSignedIntegerType () ? llvm::AtomicRMWInst::Min
594
621
: llvm::AtomicRMWInst::UMin;
595
622
break ;
596
623
624
+ case AtomicExpr::AO__atomic_max_fetch:
625
+ PostOpMinMax = true ;
626
+ LLVM_FALLTHROUGH;
627
+ case AtomicExpr::AO__c11_atomic_fetch_max:
597
628
case AtomicExpr::AO__opencl_atomic_fetch_max:
598
629
case AtomicExpr::AO__atomic_fetch_max:
599
630
Op = E->getValueType ()->isSignedIntegerType () ? llvm::AtomicRMWInst::Max
@@ -643,8 +674,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
643
674
// For __atomic_*_fetch operations, perform the operation again to
644
675
// determine the value which was written.
645
676
llvm::Value *Result = RMWI;
646
- if (PostOp)
647
- Result = CGF.Builder .CreateBinOp (PostOp, RMWI, LoadVal1);
677
+ if (PostOpMinMax)
678
+ Result = EmitPostAtomicMinMax (CGF.Builder , E->getOp (),
679
+ E->getValueType ()->isSignedIntegerType (),
680
+ RMWI, LoadVal1);
681
+ else if (PostOp)
682
+ Result = CGF.Builder .CreateBinOp ((llvm::Instruction::BinaryOps)PostOp, RMWI,
683
+ LoadVal1);
648
684
if (E->getOp () == AtomicExpr::AO__atomic_nand_fetch)
649
685
Result = CGF.Builder .CreateNot (Result);
650
686
CGF.Builder .CreateStore (Result, Dest);
@@ -853,6 +889,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
853
889
case AtomicExpr::AO__c11_atomic_fetch_and:
854
890
case AtomicExpr::AO__c11_atomic_fetch_or:
855
891
case AtomicExpr::AO__c11_atomic_fetch_xor:
892
+ case AtomicExpr::AO__c11_atomic_fetch_max:
893
+ case AtomicExpr::AO__c11_atomic_fetch_min:
856
894
case AtomicExpr::AO__opencl_atomic_fetch_and:
857
895
case AtomicExpr::AO__opencl_atomic_fetch_or:
858
896
case AtomicExpr::AO__opencl_atomic_fetch_xor:
@@ -866,8 +904,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
866
904
case AtomicExpr::AO__atomic_or_fetch:
867
905
case AtomicExpr::AO__atomic_xor_fetch:
868
906
case AtomicExpr::AO__atomic_nand_fetch:
869
- case AtomicExpr::AO__atomic_fetch_min:
907
+ case AtomicExpr::AO__atomic_max_fetch:
908
+ case AtomicExpr::AO__atomic_min_fetch:
870
909
case AtomicExpr::AO__atomic_fetch_max:
910
+ case AtomicExpr::AO__atomic_fetch_min:
871
911
Val1 = EmitValToTemp (*this , E->getVal1 ());
872
912
break ;
873
913
}
@@ -916,14 +956,18 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
916
956
case AtomicExpr::AO__opencl_atomic_fetch_min:
917
957
case AtomicExpr::AO__opencl_atomic_fetch_max:
918
958
case AtomicExpr::AO__atomic_fetch_xor:
959
+ case AtomicExpr::AO__c11_atomic_fetch_max:
960
+ case AtomicExpr::AO__c11_atomic_fetch_min:
919
961
case AtomicExpr::AO__atomic_add_fetch:
920
962
case AtomicExpr::AO__atomic_and_fetch:
921
963
case AtomicExpr::AO__atomic_nand_fetch:
922
964
case AtomicExpr::AO__atomic_or_fetch:
923
965
case AtomicExpr::AO__atomic_sub_fetch:
924
966
case AtomicExpr::AO__atomic_xor_fetch:
925
- case AtomicExpr::AO__atomic_fetch_min:
926
967
case AtomicExpr::AO__atomic_fetch_max:
968
+ case AtomicExpr::AO__atomic_fetch_min:
969
+ case AtomicExpr::AO__atomic_max_fetch:
970
+ case AtomicExpr::AO__atomic_min_fetch:
927
971
// For these, only library calls for certain sizes exist.
928
972
UseOptimizedLibcall = true ;
929
973
break ;
@@ -991,6 +1035,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
991
1035
QualType RetTy;
992
1036
bool HaveRetTy = false ;
993
1037
llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0 ;
1038
+ bool PostOpMinMax = false ;
994
1039
switch (E->getOp ()) {
995
1040
case AtomicExpr::AO__c11_atomic_init:
996
1041
case AtomicExpr::AO__opencl_atomic_init:
@@ -1112,6 +1157,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
1112
1157
AddDirectArgument (*this , Args, UseOptimizedLibcall, Val1.getPointer (),
1113
1158
MemTy, E->getExprLoc (), sizeChars);
1114
1159
break ;
1160
+ case AtomicExpr::AO__atomic_min_fetch:
1161
+ PostOpMinMax = true ;
1162
+ LLVM_FALLTHROUGH;
1163
+ case AtomicExpr::AO__c11_atomic_fetch_min:
1115
1164
case AtomicExpr::AO__atomic_fetch_min:
1116
1165
case AtomicExpr::AO__opencl_atomic_fetch_min:
1117
1166
LibCallName = E->getValueType ()->isSignedIntegerType ()
@@ -1120,6 +1169,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
1120
1169
AddDirectArgument (*this , Args, UseOptimizedLibcall, Val1.getPointer (),
1121
1170
LoweredMemTy, E->getExprLoc (), sizeChars);
1122
1171
break ;
1172
+ case AtomicExpr::AO__atomic_max_fetch:
1173
+ PostOpMinMax = true ;
1174
+ LLVM_FALLTHROUGH;
1175
+ case AtomicExpr::AO__c11_atomic_fetch_max:
1123
1176
case AtomicExpr::AO__atomic_fetch_max:
1124
1177
case AtomicExpr::AO__opencl_atomic_fetch_max:
1125
1178
LibCallName = E->getValueType ()->isSignedIntegerType ()
@@ -1171,7 +1224,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
1171
1224
// PostOp is only needed for the atomic_*_fetch operations, and
1172
1225
// thus is only needed for and implemented in the
1173
1226
// UseOptimizedLibcall codepath.
1174
- assert (UseOptimizedLibcall || !PostOp);
1227
+ assert (UseOptimizedLibcall || ( !PostOp && !PostOpMinMax) );
1175
1228
1176
1229
RValue Res = emitAtomicLibcall (*this , LibCallName, RetTy, Args);
1177
1230
// The value is returned directly from the libcall.
@@ -1182,7 +1235,12 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
1182
1235
// provided an out-param.
1183
1236
if (UseOptimizedLibcall && Res.getScalarVal ()) {
1184
1237
llvm::Value *ResVal = Res.getScalarVal ();
1185
- if (PostOp) {
1238
+ if (PostOpMinMax) {
1239
+ llvm::Value *LoadVal1 = Args[1 ].getRValue (*this ).getScalarVal ();
1240
+ ResVal = EmitPostAtomicMinMax (Builder, E->getOp (),
1241
+ E->getValueType ()->isSignedIntegerType (),
1242
+ ResVal, LoadVal1);
1243
+ } else if (PostOp) {
1186
1244
llvm::Value *LoadVal1 = Args[1 ].getRValue (*this ).getScalarVal ();
1187
1245
ResVal = Builder.CreateBinOp (PostOp, ResVal, LoadVal1);
1188
1246
}
0 commit comments