Skip to content

Commit da4fe6c

Browse files
committed
[SelectionDAG] Widen <2 x T> vector types for atomic load
Vector types of 2 elements must be widened. This change does this for vector types of atomic load in SelectionDAG so that it can translate aligned vectors of >1 size.
1 parent e9ba6ed commit da4fe6c

File tree

3 files changed

+361
-23
lines changed

3 files changed

+361
-23
lines changed

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1070,6 +1070,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
10701070
SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
10711071
SDValue WidenVecRes_INSERT_SUBVECTOR(SDNode *N);
10721072
SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
1073+
SDValue WidenVecRes_ATOMIC_LOAD(AtomicSDNode *N);
10731074
SDValue WidenVecRes_LOAD(SDNode* N);
10741075
SDValue WidenVecRes_VP_LOAD(VPLoadSDNode *N);
10751076
SDValue WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N);

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 74 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4699,6 +4699,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
46994699
break;
47004700
case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
47014701
case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4702+
case ISD::ATOMIC_LOAD:
4703+
Res = WidenVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
4704+
break;
47024705
case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
47034706
case ISD::STEP_VECTOR:
47044707
case ISD::SPLAT_VECTOR:
@@ -6080,6 +6083,74 @@ SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
60806083
N->getOperand(1), N->getOperand(2));
60816084
}
60826085

6086+
/// Either return the same load or provide appropriate casts
6087+
/// from the load and return that.
6088+
static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6089+
TypeSize LdWidth, TypeSize FirstVTWidth,
6090+
SDLoc dl, SelectionDAG &DAG) {
6091+
assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
6092+
TypeSize WidenWidth = WidenVT.getSizeInBits();
6093+
if (!FirstVT.isVector()) {
6094+
unsigned NumElts =
6095+
WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6096+
EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), FirstVT, NumElts);
6097+
SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
6098+
return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
6099+
}
6100+
assert(FirstVT == WidenVT);
6101+
return LdOp;
6102+
}
6103+
6104+
static std::optional<EVT> findMemType(SelectionDAG &DAG,
6105+
const TargetLowering &TLI, unsigned Width,
6106+
EVT WidenVT, unsigned Align,
6107+
unsigned WidenEx);
6108+
6109+
SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6110+
EVT WidenVT =
6111+
TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
6112+
EVT LdVT = LD->getMemoryVT();
6113+
SDLoc dl(LD);
6114+
assert(LdVT.isVector() && WidenVT.isVector() && "Expected vectors");
6115+
assert(LdVT.isScalableVector() == WidenVT.isScalableVector() &&
6116+
"Must be scalable");
6117+
assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType() &&
6118+
"Expected equivalent element types");
6119+
6120+
// Load information
6121+
SDValue Chain = LD->getChain();
6122+
SDValue BasePtr = LD->getBasePtr();
6123+
MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
6124+
AAMDNodes AAInfo = LD->getAAInfo();
6125+
6126+
TypeSize LdWidth = LdVT.getSizeInBits();
6127+
TypeSize WidenWidth = WidenVT.getSizeInBits();
6128+
TypeSize WidthDiff = WidenWidth - LdWidth;
6129+
6130+
// Find the vector type that can load from.
6131+
std::optional<EVT> FirstVT =
6132+
findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/0,
6133+
WidthDiff.getKnownMinValue());
6134+
6135+
if (!FirstVT)
6136+
return SDValue();
6137+
6138+
SmallVector<EVT, 8> MemVTs;
6139+
TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6140+
6141+
SDValue LdOp = DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, *FirstVT, *FirstVT,
6142+
Chain, BasePtr, LD->getMemOperand());
6143+
6144+
// Load the element with one instruction.
6145+
SDValue Result = coerceLoadedValue(LdOp, *FirstVT, WidenVT, LdWidth,
6146+
FirstVTWidth, dl, DAG);
6147+
6148+
// Modified the chain - switch anything that used the old chain to use
6149+
// the new one.
6150+
ReplaceValueWith(SDValue(LD, 1), LdOp.getValue(1));
6151+
return Result;
6152+
}
6153+
60836154
SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
60846155
LoadSDNode *LD = cast<LoadSDNode>(N);
60856156
ISD::LoadExtType ExtType = LD->getExtensionType();
@@ -7988,29 +8059,9 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
79888059
LdChain.push_back(LdOp.getValue(1));
79898060

79908061
// Check if we can load the element with one instruction.
7991-
if (MemVTs.empty()) {
7992-
assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
7993-
if (!FirstVT->isVector()) {
7994-
unsigned NumElts =
7995-
WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7996-
EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), *FirstVT, NumElts);
7997-
SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
7998-
return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
7999-
}
8000-
if (FirstVT == WidenVT)
8001-
return LdOp;
8002-
8003-
// TODO: We don't currently have any tests that exercise this code path.
8004-
assert(WidenWidth.getFixedValue() % FirstVTWidth.getFixedValue() == 0);
8005-
unsigned NumConcat =
8006-
WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
8007-
SmallVector<SDValue, 16> ConcatOps(NumConcat);
8008-
SDValue UndefVal = DAG.getUNDEF(*FirstVT);
8009-
ConcatOps[0] = LdOp;
8010-
for (unsigned i = 1; i != NumConcat; ++i)
8011-
ConcatOps[i] = UndefVal;
8012-
return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
8013-
}
8062+
if (MemVTs.empty())
8063+
return coerceLoadedValue(LdOp, *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8064+
DAG);
80148065

80158066
// Load vector by using multiple loads from largest vector to scalar.
80168067
SmallVector<SDValue, 16> LdOps;

0 commit comments

Comments
 (0)