Skip to content

Commit c5cfc13

Browse files
jofrnjofrn
committed
[SelectionDAG][X86] Remove unused elements from atomic vector.
After splitting, all elements are created. The two components must be found by looking at the upper and lower half of EXTRACT_ELEMENT. This change extends EltsFromConsecutiveLoads to understand AtomicSDNode so that unused elements can be removed. commit-id:b83937a8
1 parent 4783f04 commit c5cfc13

File tree

5 files changed

+104
-172
lines changed

5 files changed

+104
-172
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAG.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1843,7 +1843,7 @@ class SelectionDAG {
18431843
/// chain to the token factor. This ensures that the new memory node will have
18441844
/// the same relative memory dependency position as the old load. Returns the
18451845
/// new merged load chain.
1846-
SDValue makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, SDValue NewMemOp);
1846+
SDValue makeEquivalentMemoryOrdering(MemSDNode *OldLoad, SDValue NewMemOp);
18471847

18481848
/// Topological-sort the AllNodes list and a
18491849
/// assign a unique node id for each node in the DAG based on their
@@ -2281,7 +2281,7 @@ class SelectionDAG {
22812281
/// merged. Check that both are nonvolatile and if LD is loading
22822282
/// 'Bytes' bytes from a location that is 'Dist' units away from the
22832283
/// location that the 'Base' load is loading from.
2284-
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base,
2284+
bool areNonVolatileConsecutiveLoads(MemSDNode *LD, MemSDNode *Base,
22852285
unsigned Bytes, int Dist) const;
22862286

22872287
/// Infer alignment of a load / store address. Return std::nullopt if it

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12218,7 +12218,7 @@ SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
1221812218
return TokenFactor;
1221912219
}
1222012220

12221-
SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
12221+
SDValue SelectionDAG::makeEquivalentMemoryOrdering(MemSDNode *OldLoad,
1222212222
SDValue NewMemOp) {
1222312223
assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
1222412224
SDValue OldChain = SDValue(OldLoad, 1);
@@ -12911,17 +12911,21 @@ std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
1291112911
getBuildVector(NewOvVT, dl, OvScalars));
1291212912
}
1291312913

12914-
bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
12915-
LoadSDNode *Base,
12914+
bool SelectionDAG::areNonVolatileConsecutiveLoads(MemSDNode *LD,
12915+
MemSDNode *Base,
1291612916
unsigned Bytes,
1291712917
int Dist) const {
1291812918
if (LD->isVolatile() || Base->isVolatile())
1291912919
return false;
12920-
// TODO: probably too restrictive for atomics, revisit
12921-
if (!LD->isSimple())
12922-
return false;
12923-
if (LD->isIndexed() || Base->isIndexed())
12924-
return false;
12920+
if (auto Ld = dyn_cast<LoadSDNode>(LD)) {
12921+
if (!Ld->isSimple())
12922+
return false;
12923+
if (Ld->isIndexed())
12924+
return false;
12925+
}
12926+
if (auto Ld = dyn_cast<LoadSDNode>(Base))
12927+
if (Ld->isIndexed())
12928+
return false;
1292512929
if (LD->getChain() != Base->getChain())
1292612930
return false;
1292712931
EVT VT = LD->getMemoryVT();

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,8 @@ bool BaseIndexOffset::contains(const SelectionDAG &DAG, int64_t BitSize,
195195
}
196196

197197
/// Parses tree in Ptr for base, index, offset addresses.
198-
static BaseIndexOffset matchLSNode(const LSBaseSDNode *N,
199-
const SelectionDAG &DAG) {
198+
template <typename T>
199+
static BaseIndexOffset matchSDNode(const T *N, const SelectionDAG &DAG) {
200200
SDValue Ptr = N->getBasePtr();
201201

202202
// (((B + I*M) + c)) + c ...
@@ -206,16 +206,18 @@ static BaseIndexOffset matchLSNode(const LSBaseSDNode *N,
206206
bool IsIndexSignExt = false;
207207

208208
// pre-inc/pre-dec ops are components of EA.
209-
if (N->getAddressingMode() == ISD::PRE_INC) {
210-
if (auto *C = dyn_cast<ConstantSDNode>(N->getOffset()))
211-
Offset += C->getSExtValue();
212-
else // If unknown, give up now.
213-
return BaseIndexOffset(SDValue(), SDValue(), 0, false);
214-
} else if (N->getAddressingMode() == ISD::PRE_DEC) {
215-
if (auto *C = dyn_cast<ConstantSDNode>(N->getOffset()))
216-
Offset -= C->getSExtValue();
217-
else // If unknown, give up now.
218-
return BaseIndexOffset(SDValue(), SDValue(), 0, false);
209+
if constexpr (std::is_same_v<T, LSBaseSDNode>) {
210+
if (N->getAddressingMode() == ISD::PRE_INC) {
211+
if (auto *C = dyn_cast<ConstantSDNode>(N->getOffset()))
212+
Offset += C->getSExtValue();
213+
else // If unknown, give up now.
214+
return BaseIndexOffset(SDValue(), SDValue(), 0, false);
215+
} else if (N->getAddressingMode() == ISD::PRE_DEC) {
216+
if (auto *C = dyn_cast<ConstantSDNode>(N->getOffset()))
217+
Offset -= C->getSExtValue();
218+
else // If unknown, give up now.
219+
return BaseIndexOffset(SDValue(), SDValue(), 0, false);
220+
}
219221
}
220222

221223
// Consume constant adds & ors with appropriate masking.
@@ -300,8 +302,10 @@ static BaseIndexOffset matchLSNode(const LSBaseSDNode *N,
300302

301303
BaseIndexOffset BaseIndexOffset::match(const SDNode *N,
302304
const SelectionDAG &DAG) {
305+
if (const auto *AN = dyn_cast<AtomicSDNode>(N))
306+
return matchSDNode(AN, DAG);
303307
if (const auto *LS0 = dyn_cast<LSBaseSDNode>(N))
304-
return matchLSNode(LS0, DAG);
308+
return matchSDNode(LS0, DAG);
305309
if (const auto *LN = dyn_cast<LifetimeSDNode>(N)) {
306310
if (LN->hasOffset())
307311
return BaseIndexOffset(LN->getOperand(1), SDValue(), LN->getOffset(),

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 61 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7193,15 +7193,19 @@ static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
71937193
}
71947194

71957195
// Recurse to find a LoadSDNode source and the accumulated ByteOffest.
7196-
static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
7197-
if (ISD::isNON_EXTLoad(Elt.getNode())) {
7198-
auto *BaseLd = cast<LoadSDNode>(Elt);
7199-
if (!BaseLd->isSimple())
7200-
return false;
7196+
static bool findEltLoadSrc(SDValue Elt, MemSDNode *&Ld, int64_t &ByteOffset) {
7197+
if (auto *BaseLd = dyn_cast<AtomicSDNode>(Elt)) {
72017198
Ld = BaseLd;
72027199
ByteOffset = 0;
72037200
return true;
7204-
}
7201+
} else if (auto *BaseLd = dyn_cast<LoadSDNode>(Elt))
7202+
if (ISD::isNON_EXTLoad(Elt.getNode())) {
7203+
if (!BaseLd->isSimple())
7204+
return false;
7205+
Ld = BaseLd;
7206+
ByteOffset = 0;
7207+
return true;
7208+
}
72057209

72067210
switch (Elt.getOpcode()) {
72077211
case ISD::BITCAST:
@@ -7230,6 +7234,20 @@ static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
72307234
}
72317235
}
72327236
break;
7237+
case ISD::EXTRACT_ELEMENT:
7238+
if (auto *IdxC = dyn_cast<ConstantSDNode>(Elt.getOperand(1))) {
7239+
SDValue Src = Elt.getOperand(0);
7240+
unsigned SrcSizeInBits = Src.getScalarValueSizeInBits();
7241+
unsigned DstSizeInBits = Elt.getScalarValueSizeInBits();
7242+
if (2 * DstSizeInBits == SrcSizeInBits && (SrcSizeInBits % 8) == 0 &&
7243+
findEltLoadSrc(Src, Ld, ByteOffset)) {
7244+
uint64_t Idx = IdxC->getZExtValue();
7245+
if (Idx == 1) // Get the upper half.
7246+
ByteOffset += SrcSizeInBits / 8 / 2;
7247+
return true;
7248+
}
7249+
}
7250+
break;
72337251
}
72347252

72357253
return false;
@@ -7254,7 +7272,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
72547272
APInt ZeroMask = APInt::getZero(NumElems);
72557273
APInt UndefMask = APInt::getZero(NumElems);
72567274

7257-
SmallVector<LoadSDNode*, 8> Loads(NumElems, nullptr);
7275+
SmallVector<MemSDNode *, 8> Loads(NumElems, nullptr);
72587276
SmallVector<int64_t, 8> ByteOffsets(NumElems, 0);
72597277

72607278
// For each element in the initializer, see if we've found a load, zero or an
@@ -7304,7 +7322,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
73047322
EVT EltBaseVT = EltBase.getValueType();
73057323
assert(EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits() &&
73067324
"Register/Memory size mismatch");
7307-
LoadSDNode *LDBase = Loads[FirstLoadedElt];
7325+
MemSDNode *LDBase = Loads[FirstLoadedElt];
73087326
assert(LDBase && "Did not find base load for merging consecutive loads");
73097327
unsigned BaseSizeInBits = EltBaseVT.getStoreSizeInBits();
73107328
unsigned BaseSizeInBytes = BaseSizeInBits / 8;
@@ -7318,8 +7336,8 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
73187336

73197337
// Check to see if the element's load is consecutive to the base load
73207338
// or offset from a previous (already checked) load.
7321-
auto CheckConsecutiveLoad = [&](LoadSDNode *Base, int EltIdx) {
7322-
LoadSDNode *Ld = Loads[EltIdx];
7339+
auto CheckConsecutiveLoad = [&](MemSDNode *Base, int EltIdx) {
7340+
MemSDNode *Ld = Loads[EltIdx];
73237341
int64_t ByteOffset = ByteOffsets[EltIdx];
73247342
if (ByteOffset && (ByteOffset % BaseSizeInBytes) == 0) {
73257343
int64_t BaseIdx = EltIdx - (ByteOffset / BaseSizeInBytes);
@@ -7347,7 +7365,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
73477365
}
73487366
}
73497367

7350-
auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
7368+
auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, MemSDNode *LDBase) {
73517369
auto MMOFlags = LDBase->getMemOperand()->getFlags();
73527370
assert(LDBase->isSimple() &&
73537371
"Cannot merge volatile or atomic loads.");
@@ -9452,8 +9470,9 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
94529470
{
94539471
SmallVector<SDValue, 64> Ops(Op->ops().take_front(NumElems));
94549472
if (SDValue LD =
9455-
EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))
9473+
EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false)) {
94569474
return LD;
9475+
}
94579476
}
94589477

94599478
// If this is a splat of pairs of 32-bit elements, we can use a narrower
@@ -60381,6 +60400,35 @@ static SDValue combineINTRINSIC_VOID(SDNode *N, SelectionDAG &DAG,
6038160400
return SDValue();
6038260401
}
6038360402

60403+
static SDValue combineVZEXT_LOAD(SDNode *N, SelectionDAG &DAG,
60404+
TargetLowering::DAGCombinerInfo &DCI) {
60405+
// Find the TokenFactor to locate the associated AtomicLoad.
60406+
SDNode *ALD = nullptr;
60407+
for (auto &TF : DAG.allnodes())
60408+
if (TF.getOpcode() == ISD::TokenFactor) {
60409+
SDValue L = TF.getOperand(0);
60410+
SDValue R = TF.getOperand(1);
60411+
if (L.getNode() == N)
60412+
ALD = R.getNode();
60413+
else if (R.getNode() == N)
60414+
ALD = L.getNode();
60415+
}
60416+
60417+
if (!ALD)
60418+
return SDValue();
60419+
if (!isa<AtomicSDNode>(ALD))
60420+
return SDValue();
60421+
60422+
// Replace the VZEXT_LOAD with the AtomicLoad.
60423+
SDLoc dl(N);
60424+
SDValue SV = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
60425+
N->getValueType(0).changeTypeToInteger(),
60426+
SDValue(ALD, 0));
60427+
SDValue BC = DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), SV);
60428+
BC = DCI.CombineTo(N, BC, SDValue(ALD, 1));
60429+
return BC;
60430+
}
60431+
6038460432
SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
6038560433
DAGCombinerInfo &DCI) const {
6038660434
SelectionDAG &DAG = DCI.DAG;
@@ -60577,6 +60625,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
6057760625
case ISD::INTRINSIC_VOID: return combineINTRINSIC_VOID(N, DAG, DCI);
6057860626
case ISD::FP_TO_SINT_SAT:
6057960627
case ISD::FP_TO_UINT_SAT: return combineFP_TO_xINT_SAT(N, DAG, Subtarget);
60628+
case X86ISD::VZEXT_LOAD: return combineVZEXT_LOAD(N, DAG, DCI);
6058060629
// clang-format on
6058160630
}
6058260631

0 commit comments

Comments
 (0)