IntDivRemIncrementReduction Optimization Pass

An unrolled loop with udiv/urem pairs gets multiple consecutive
udiv/urem with the dividend incrementing by 1. Instead of replacing each
with the precompiled udiv/urem, utilize previous results to simplify
computation. This also works for nested udiv/urem where the dividend is
the quotient from a previous udiv/urem.

Author: bowenxue-intel

IGC/Compiler/CISACodeGen/ShaderCodeGen.cpp

@@ -99,6 +99,7 @@ SPDX-License-Identifier: MIT
 #include "Compiler/Optimizer/MCSOptimization.hpp"
 #include "Compiler/Optimizer/GatingSimilarSamples.hpp"
 #include "Compiler/Optimizer/IntDivConstantReduction.hpp"
+#include "Compiler/Optimizer/IntDivRemIncrementReduction.hpp"
 #include "Compiler/Optimizer/IntDivRemCombine.hpp"
 #include "Compiler/Optimizer/SynchronizationObjectCoalescing.hpp"
 #include "Compiler/Optimizer/BarrierControlFlowOptimization.hpp"
@@ -1868,6 +1869,8 @@ void OptimizeIR(CodeGenContext* const pContext)
             mpm.add( createWaveAllJointReduction() );
         }
 
+        mpm.add(llvm::createEarlyCSEPass());
+
         if (IGC_IS_FLAG_ENABLED(EnableIntDivRemCombine)) {
             // simplify rem if the quotient is availble
             //
@@ -1887,6 +1890,10 @@ void OptimizeIR(CodeGenContext* const pContext)
             // more efficient sequences of multiplies, shifts, and adds
             mpm.add(createIntDivConstantReductionPass());
         }
+
+        if (IGC_IS_FLAG_ENABLED(EnableIntDivRemIncrementReduction)) {
+          mpm.add(createIntDivRemIncrementReductionPass());
+        }
         GFX_ONLY_PASS { mpm.add(createMergeMemFromBranchOptPass()); }
 
         if (IGC_IS_FLAG_DISABLED(DisableLoadSinking) &&

IGC/Compiler/CustomSafeOptPass.cpp

@@ -1066,18 +1066,31 @@ void CustomSafeOptPass::visitBfi(llvm::CallInst* inst)
     ConstantInt* offsetV = dyn_cast<ConstantInt>(inst->getOperand(1));
     if (widthV && offsetV)
     {
+        ConstantInt* baseV = dyn_cast<ConstantInt>(inst->getOperand(3));
         // transformation is beneficial if src3 is constant or if the offset is zero
-        if (isa<ConstantInt>(inst->getOperand(3)) || offsetV->isZero())
+        if (baseV || offsetV->isZero())
         {
             unsigned int width = static_cast<unsigned int>(widthV->getZExtValue());
             unsigned int offset = static_cast<unsigned int>(offsetV->getZExtValue());
             unsigned int bitMask = ((1 << width) - 1) << offset;
             IRBuilder<> builder(inst);
             // dst = ((src2 << offset) & bitmask) | (src3 & ~bitmask)
-            Value* firstTerm = builder.CreateShl(inst->getOperand(2), offsetV);
+            Value* firstTerm = nullptr;
+            Value* dst = nullptr;
+            if( offset != 0 ) {
+              firstTerm = builder.CreateShl(inst->getOperand(2), offsetV);
+            } else {
+              firstTerm = inst->getOperand(2);
+            }
             firstTerm = builder.CreateAnd(firstTerm, builder.getInt32(bitMask));
-            Value* secondTerm = builder.CreateAnd(inst->getOperand(3), builder.getInt32(~bitMask));
-            Value* dst = builder.CreateOr(firstTerm, secondTerm);
+
+            if (baseV && baseV->isZero()) {
+              dst = firstTerm;
+            } else {
+              auto* secondTerm = builder.CreateAnd(inst->getOperand(3), builder.getInt32(~bitMask));
+              dst = builder.CreateOr(firstTerm, secondTerm);
+            }
+
             inst->replaceAllUsesWith(dst);
             inst->eraseFromParent();
         }

IGC/Compiler/InitializePasses.h

@@ -245,6 +245,7 @@ void initializeFCmpPaternMatchPass(llvm::PassRegistry&);
 void initializeCodeAssumptionPass(llvm::PassRegistry&);
 void initializeIGCInstructionCombiningPassPass(llvm::PassRegistry&);
 void initializeIntDivConstantReductionPass(llvm::PassRegistry&);
+void initializeIntDivRemIncrementReductionPass(llvm::PassRegistry&);
 void initializeIntDivRemCombinePass(llvm::PassRegistry&);
 void initializeGenRotatePass(llvm::PassRegistry&);
 void initializeSynchronizationObjectCoalescingPass(llvm::PassRegistry&);

IGC/Compiler/Optimizer/CMakeLists.txt

@@ -36,6 +36,7 @@ set(IGC_BUILD__SRC__Optimizer
     "${CMAKE_CURRENT_SOURCE_DIR}/BarrierControlFlowOptimization.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/WaveShuffleIndexSinking.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/WaveAllJointReduction.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/IntDivRemIncrementReduction.cpp"
   )
 
 set(IGC_BUILD__SRC__Compiler_Optimizer
@@ -63,8 +64,9 @@ set(IGC_BUILD__HDR__Optimizer
     "${CMAKE_CURRENT_SOURCE_DIR}/ValueTracker.h"
     "${CMAKE_CURRENT_SOURCE_DIR}/RuntimeValueVectorExtractPass.h"
     "${CMAKE_CURRENT_SOURCE_DIR}/BarrierControlFlowOptimization.hpp"
-    "${CMAKE_CURRENT_SOURCE_DIR}/WaveShuffleIndexSinking.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/WaveShuffleIndexSinking.hpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/WaveAllJointReduction.hpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/IntDivRemIncrementReduction.hpp"
   )
 
 set(IGC_BUILD__HDR__Optimizer