Predicated load/store: bug fixing, tests

Fixed several bugs and added tests for
predicated load/store implementation

Author: YuriPlyakhin

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -19700,8 +19700,12 @@ void EmitPass::emitLSCVectorLoad_uniform(
         dSize = 4;
         vSize = vSize * 2;
         ldDest = m_currShader->GetNewAlias(ldDest, ISA_TYPE_UD, 0, 0);
-        if (mergeVal)
-            newMergeVal = m_currShader->GetNewAlias(mergeVal, ISA_TYPE_UD, 0, 0);
+        if (mergeVal) {
+            if (mergeVal->IsImmediate())
+                newMergeVal = m_currShader->ImmToVariable(mergeVal->GetImmediateValue(), ISA_TYPE_UD);
+            else
+                newMergeVal = m_currShader->GetNewAlias(mergeVal, ISA_TYPE_UD, 0, 0);
+        }
     }
 
     bool destUniform = Dest->IsUniform();

IGC/Compiler/CISACodeGen/VectorPreProcess.cpp

@@ -489,7 +489,7 @@ namespace
             SmallVector<std::pair<Type*, uint32_t>, 8>& SplitInfo);
         // If predicated loads are split, we also need to split merge values
         void createSplitMergeValues(
-            BasicBlock* BB,
+            Instruction* Inst,
             Value* OrigMergeVal,
             const SmallVector<std::pair<Type*, uint32_t>, 8>& SplitInfo,
             ValVector& NewMergeVals) const;
@@ -695,7 +695,7 @@ void VectorPreProcess::createSplitVectorTypes(
     }
 }
 
-void VectorPreProcess::createSplitMergeValues(BasicBlock* BB,
+void VectorPreProcess::createSplitMergeValues(Instruction* Inst,
                                               Value* OrigMergeVal,
                                               const SmallVector<std::pair<Type*, uint32_t>, 8>& SplitInfo,
                                               ValVector& NewMergeVals) const
@@ -731,10 +731,7 @@ void VectorPreProcess::createSplitMergeValues(BasicBlock* BB,
         return;
     }
 
-    // Init IRBuilder with Basic block, if OrigMergeVal is not an instruction.
-    std::unique_ptr<IRBuilder<>> Builder = isa<Instruction>(OrigMergeVal) ?
-                                           std::make_unique<IRBuilder<>>(cast<Instruction>(OrigMergeVal)) :
-                                           std::make_unique<IRBuilder<>>(BB);
+    IRBuilder<> Builder(Inst);
 
     // Case when we split vector merge value into subvectors. Element type is the same.
     // Just one big vector is being split into subvectors.
@@ -756,14 +753,14 @@ void VectorPreProcess::createSplitMergeValues(BasicBlock* BB,
                 {
                     for (uint32_t j = 0, e = int_cast<uint32_t>(VTy->getNumElements()); j < e; ++j)
                     {
-                        Value *Elt = (idx < OrigVTyNEl) ? Builder->CreateExtractElement(OrigMergeVal, Builder->getInt32(idx++)) :
+                        Value *Elt = (idx < OrigVTyNEl) ? Builder.CreateExtractElement(OrigMergeVal, Builder.getInt32(idx++)) :
                                                           Constant::getNullValue(VTy->getElementType());
-                        NewMergeVal = Builder->CreateInsertElement(NewMergeVal, Elt, Builder->getInt32(j));
+                        NewMergeVal = Builder.CreateInsertElement(NewMergeVal, Elt, Builder.getInt32(j));
                     }
                 }
                 else
                 {
-                    NewMergeVal = Builder->CreateExtractValue(OrigMergeVal, idx++);
+                    NewMergeVal = Builder.CreateExtractElement(OrigMergeVal, Builder.getInt32(idx++));
                 }
                 NewMergeVals.push_back(NewMergeVal);
             }
@@ -775,7 +772,7 @@ void VectorPreProcess::createSplitMergeValues(BasicBlock* BB,
     // Case when we change scalar value into vector with smaller element type.
     IGC_ASSERT_MESSAGE(SplitInfo.size() == 1, "Unexpected split info!");
     IGC_ASSERT_MESSAGE(SplitInfo[0].second == 1, "Unexpected split info!");
-    Value *NewMergeVal = Builder->CreateBitCast(OrigMergeVal, SplitInfo[0].first);
+    Value *NewMergeVal = Builder.CreateBitCast(OrigMergeVal, SplitInfo[0].first);
     NewMergeVals.push_back(NewMergeVal);
 }
 
@@ -1116,7 +1113,7 @@ bool VectorPreProcess::splitLoad(
 
     ValVector splitMergeValues;
     if (isPredLd)
-        createSplitMergeValues(LI->getParent(), cast<PredicatedLoadIntrinsic>(LI)->getMergeValue(), splitInfo, splitMergeValues);
+        createSplitMergeValues(LI, cast<PredicatedLoadIntrinsic>(LI)->getMergeValue(), splitInfo, splitMergeValues);
 
     Value* Addr = ALI.getPointerOperand();
     auto Align = ALI.getAlignment();
@@ -1306,7 +1303,7 @@ bool VectorPreProcess::splitVector3LoadStore(Instruction* Inst)
 
                 ValVector splitMergeValues;
                 if (isPredLoad)
-                    createSplitMergeValues(Inst->getParent(), cast<PredicatedLoadIntrinsic>(Inst)->getMergeValue(),
+                    createSplitMergeValues(Inst, cast<PredicatedLoadIntrinsic>(Inst)->getMergeValue(),
                                            { {newVTy, 1} }, splitMergeValues);
 
                 Value* V = ALI->Create(newVTy, isPredLoad ? splitMergeValues[0] : nullptr);
@@ -1323,7 +1320,7 @@ bool VectorPreProcess::splitVector3LoadStore(Instruction* Inst)
 
                 ValVector splitMergeValues;
                 if (isPredLoad)
-                    createSplitMergeValues(Inst->getParent(), cast<PredicatedLoadIntrinsic>(Inst)->getMergeValue(),
+                    createSplitMergeValues(Inst, cast<PredicatedLoadIntrinsic>(Inst)->getMergeValue(),
                                            { {newVTy, 1}, {eTy, 1} }, splitMergeValues);
 
                 Value* V2 = ALI->Create(newVTy, isPredLoad ? splitMergeValues[0] : nullptr);
@@ -1778,7 +1775,7 @@ Instruction* VectorPreProcess::simplifyLoadStore(Instruction* Inst)
             bool isPredLoad = isa<PredicatedLoadIntrinsic>(Inst);
             ValVector splitMergeValues;
             if (isPredLoad)
-                createSplitMergeValues(Inst->getParent(), cast<PredicatedLoadIntrinsic>(Inst)->getMergeValue(),
+                createSplitMergeValues(Inst, cast<PredicatedLoadIntrinsic>(Inst)->getMergeValue(),
                                        { {NewVecTy, 1} }, splitMergeValues);
 
             NewLI = ALI.Create(NewVecTy, isPredLoad ? splitMergeValues[0] : nullptr);
@@ -1997,7 +1994,7 @@ bool VectorPreProcess::processScalarLoadStore(Function& F)
             bool isPredLd = isa<PredicatedLoadIntrinsic>(inst);
             ValVector splitMergeValues;
             if (isPredLd)
-                createSplitMergeValues(inst->getParent(), cast<PredicatedLoadIntrinsic>(inst)->getMergeValue(),
+                createSplitMergeValues(inst, cast<PredicatedLoadIntrinsic>(inst)->getMergeValue(),
                                        { {newVecTy, 1} }, splitMergeValues);
             Value *MergeVal = isPredLd ? splitMergeValues[0] : nullptr;
 

IGC/Compiler/CISACodeGen/VectorProcess.cpp

@@ -156,7 +156,8 @@ namespace
     private:
         bool reLayoutLoadStore(Instruction* Inst);
         bool optimizeBitCast(BitCastInst* BC);
-        Value* ProcessMergeValue(BasicBlock *BB, Value* V, Type* NewTy) const;
+        Value* ProcessMergeValue(Instruction *Inst, Value* V, Type* NewTy,
+                                 Type* NewIntETy, Type* NewIntTy) const;
 
     private:
         const DataLayout* m_DL;
@@ -372,6 +373,11 @@ bool VectorProcess::reLayoutLoadStore(Instruction* Inst)
         newPtr = Builder.CreateBitCast(Ptr, newPtrTy, "vptrcast");
     }
 
+    // These types are needed when we are dealing with pointers
+    // and using ptrtoint and inttoptr.
+    Type* int_eTy = Type::getIntNTy(*m_C, eTyBits);
+    Type* new_intTy = VTy ? FixedVectorType::get(int_eTy, nelts) : int_eTy;
+
     if (LI || (II && II->getIntrinsicID() == GenISAIntrinsic::GenISA_PredicatedLoad))
     {
         Instruction* oldLoad = LI ? cast<Instruction>(LI) : cast<Instruction>(II);
@@ -394,7 +400,7 @@ bool VectorProcess::reLayoutLoadStore(Instruction* Inst)
                 GenISAIntrinsic::GenISA_PredicatedLoad,
                 types);
             load = Builder.CreateCall4(F, newPtr, II->getOperand(1), II->getOperand(2),
-                                       ProcessMergeValue(Inst->getParent(), II->getOperand(3), newVTy));
+                                       ProcessMergeValue(Inst, II->getOperand(3), newVTy, int_eTy, new_intTy));
         }
         load->copyMetadata(*oldLoad);
 
@@ -407,8 +413,6 @@ bool VectorProcess::reLayoutLoadStore(Instruction* Inst)
             //        the original vector type with ptr element type replaced
             //        with int-element type.
             // second, IntToPtr cast to the original vector type.
-            Type* int_eTy = Type::getIntNTy(*m_C, eTyBits);
-            Type* new_intTy = VTy ? FixedVectorType::get(int_eTy, nelts) : int_eTy;
             V = Builder.CreateBitCast(V, new_intTy);
             if (VTy)
             {
@@ -723,28 +727,53 @@ bool VectorProcess::runOnFunction(Function& F)
     return changed;
 }
 
-Value* VectorProcess::ProcessMergeValue(BasicBlock *BB, Value* V, Type* NewTy) const
+Value* VectorProcess::ProcessMergeValue(Instruction *Inst, Value* V, Type* NewTy, Type* NewIntEType, Type* NewIntTy) const
 {
     // if V is a zero initializer, undef or poison value, we just need to create
     // corresponding value of NewTy.
     if (isa<ConstantAggregateZero>(V)) {
-        if(IGCLLVM::FixedVectorType *VTy = dyn_cast<IGCLLVM::FixedVectorType>(NewTy))
-            return ConstantAggregateZero::get(VTy);
+        if(IGCLLVM::FixedVectorType *NewVTy = dyn_cast<IGCLLVM::FixedVectorType>(NewTy))
+            return ConstantAggregateZero::get(NewVTy);
         else
             return Constant::getNullValue(NewTy);
     }
 
+    if (isa<PoisonValue>(V))
+        return PoisonValue::get(NewTy);
+
     if (isa<UndefValue>(V))
         return UndefValue::get(NewTy);
 
-    if (isa<PoisonValue>(V))
-        return PoisonValue::get(NewTy);
+    IRBuilder<> Builder(Inst);
+
+    Type *Ty = V->getType();
+    IGCLLVM::FixedVectorType* const VTy = dyn_cast<IGCLLVM::FixedVectorType>(Ty);
+    uint32_t nelts = VTy ? int_cast<uint32_t>(VTy->getNumElements()) : 1;
+    Type* eTy = VTy ? VTy->getElementType() : Ty;
 
-    std::unique_ptr<IRBuilder<>> Builder = isa<Instruction>(V) ?
-                                            std::make_unique<IRBuilder<>>(cast<Instruction>(V)) :
-                                            std::make_unique<IRBuilder<>>(BB);
+    if (eTy->isPointerTy())
+    {
+        // cannot bitcast ptr to int; First, PtrToInt cast
+        // then bitcast int (scalar or vector) to the new type.
+        if (VTy)
+        {
+            // need a vector ptrtoint, scalarize:
+            auto* oldV = V;
+            V = UndefValue::get(NewIntTy);
+            for (unsigned i = 0; i < nelts; ++i)
+            {
+                auto* EE = Builder.CreateExtractElement(oldV, i);
+                auto* PTI = Builder.CreatePtrToInt(EE, NewIntEType);
+                V = Builder.CreateInsertElement(V, PTI, i);
+            }
+        }
+        else
+        {
+            V = Builder.CreatePtrToInt(V, NewIntTy);
+        }
+    }
 
-    return Builder->CreateBitCast(V, NewTy);
+    return Builder.CreateBitCast(V, NewTy);
 }
 
 //

IGC/Compiler/Optimizer/PromoteToPredicatedMemoryAccess.cpp

@@ -47,12 +47,20 @@ namespace IGC {
 
 bool PromoteToPredicatedMemoryAccess::runOnFunction(Function &F) {
   CodeGenContext* pCtx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
+
+  bool isStatlessToBindlessEnabled = (pCtx->type == ShaderType::OPENCL_SHADER &&
+                                      static_cast<OpenCLProgramContext*>(pCtx)->m_InternalOptions.PromoteStatelessToBindless);
   if (!pCtx->platform.hasLSC() ||
       !pCtx->platform.LSCEnabled() ||
-      (pCtx->type == ShaderType::OPENCL_SHADER &&
-               static_cast<OpenCLProgramContext*>(pCtx)->m_InternalOptions.PromoteStatelessToBindless) ||
-      pCtx->useStatelessToStateful())
+      isStatlessToBindlessEnabled ||
+      pCtx->useStatelessToStateful()) {
+    LLVM_DEBUG(dbgs() << "Skip promotion to predicated memory operations because one of conditions is false:\n"
+                      << " - Platform has LSC: " << pCtx->platform.hasLSC() << "\n"
+                      << " - LSC is enabled: " << pCtx->platform.LSCEnabled() << "\n"
+                      << " - PromoteStatelessToBindless is disabled: " << !isStatlessToBindlessEnabled << "\n"
+                      << " - useStatelessToStateful is disabled: " << !pCtx->useStatelessToStateful() << "\n");
     return false;
+  }
 
   SmallVector<std::pair<BranchInst *, bool>, 8> WorkList;
 
@@ -103,10 +111,6 @@ void PromoteToPredicatedMemoryAccess::fixPhiNode(PHINode &Phi, BasicBlock &Prede
 bool PromoteToPredicatedMemoryAccess::trySingleBlockIfConv(Value &Cond, BasicBlock &BranchBB,
                                                 BasicBlock &ConvBB, BasicBlock &SuccBB,
                                                 bool Inverse) {
-  // Reject the candidate if the condition is not an integer compare instruction.
-  if (!isa<ICmpInst>(Cond))
-    return false;
-
   if (!ConvBB.hasNPredecessors(1))
     return false;
 

IGC/Compiler/Optimizer/PromoteToPredicatedMemoryAccess.hpp

@@ -27,8 +27,7 @@ namespace IGC
     //     %17 = call <4 x float> @llvm.genx.GenISA.PredicatedLoad.v4f32.p1v4f32.v4f32(<4 x float> addrspace(1)* %bitc0, i64 16, i1 %pred, <4 x float> %mergeValue)
     // if found in specific pattern and then performs if-conversion.
     //
-    // The pass looks for conditional branches that can be if-converted. The
-    // condition must be an integer compare instruction. The only "hammock" form
+    // The pass looks for conditional branches that can be if-converted. The only "hammock" form
     // of the control flow is supported, i.e. the true block has a single
     // predecessor and the false block has two predecessors. The true block must
     // have a single successor that is the false block.

IGC/Compiler/tests/EmitVISAPass/predicated-load-subdw.ll

@@ -0,0 +1,56 @@
+;=========================== begin_copyright_notice ============================
+;
+; Copyright (C) 2025 Intel Corporation
+;
+; SPDX-License-Identifier: MIT
+;
+;============================ end_copyright_notice =============================
+; REQUIRES: llvm-14-plus, regkeys
+;
+; RUN: igc_opt --opaque-pointers -platformbmg -igc-emit-visa %s -regkey DumpVISAASMToConsole | FileCheck %s
+; ------------------------------------------------
+; EmitVISAPass
+; ------------------------------------------------
+
+; Verifies that predicated loads are emitted correctly for subDW loads
+
+define spir_kernel void @test(ptr addrspace(1) align 1 %in, i32 %predicate) {
+entry:
+  ; calculated predicate
+  %p = icmp slt i32 0, %predicate
+
+  ; CHECK: .decl [[G_ALIAS0:.*]] v_type=G type=b num_elts=4 align=wordx32 alias=<[[GATHER0:.*]], 0>
+  ; CHECK: .decl [[G_ALIAS1:.*]] v_type=G type=b num_elts=4 align=wordx32 alias=<[[GATHER1:.*]], 0>
+
+  ; copy merge value. do predicated load, copy result
+  ; CHECK: mov (M1_NM, 1) [[G_ALIAS0]](0,0)<0> 0x0:b
+  ; CHECK: (P1) lsc_load.ugm (M1_NM, 1)  [[GATHER0]]:d8c32  flat[
+  ; CHECK: mov (M1_NM, 1) res0(0,0)<1> [[G_ALIAS0]](0,0)<0;1,0>
+  %res0 = call i8 @llvm.genx.GenISA.PredicatedLoad.i8.p1.i8(ptr addrspace(1) %in, i64 1, i1 %p, i8 0)
+
+  ; do predicated load, then do predicated copy of result to merge value which is used as dest.
+  ; CHECK: (P1) lsc_load.ugm (M1_NM, 1)  [[GATHER1]]:d8c32  flat[
+  ; CHECK: (P1) mov (M1_NM, 1) mVi8(0,0)<1> [[G_ALIAS1]](0,0)<0;1,0>
+  %mergeV = add i32 %predicate, 5
+  %mVi8 = trunc i32 %mergeV to i8
+  %res1 = call i8 @llvm.genx.GenISA.PredicatedLoad.i8.p1.i8(ptr addrspace(1) %in, i64 1, i1 %p, i8 %mVi8)
+
+  ret void
+}
+
+declare i8 @llvm.genx.GenISA.PredicatedLoad.i8.p1.i8(ptr addrspace(1), i64, i1, i8)
+
+!IGCMetadata = !{!0}
+!igc.functions = !{!3}
+
+!0 = !{!"ModuleMD", !132}
+!3 = !{ptr @test, !4}
+!4 = !{!5}
+!5 = !{!"function_type", i32 0}
+!132 = !{!"FuncMD", !133, !134}
+!133 = !{!"FuncMDMap[0]", ptr @test}
+!134 = !{!"FuncMDValue[0]", !167}
+!167 = !{!"resAllocMD", !171}
+!171 = !{!"argAllocMDList", !172}
+!172 = !{!"argAllocMDListVec[0]", !173}
+!173 = !{!"type", i32 0}