More than 64 part4

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 //    The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
     assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
     return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+//    mask               - the register mask
+//
+// Return Value:
+//    The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+    assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+    /* Convert the mask to a register number */
+
+    regNumber regNum = (regNumber)BitScanForward(mask);
+
+    return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 //          register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
     /* Convert the mask to a register number */
 
-    regNumber regNum = (regNumber)BitScanForward(mask);
+    regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
-    mask ^= genRegMask(regNum);
+    mask ^= regNum;
 
     return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+//          register number and also toggle the bit in the `mask`.
 // Arguments:
 //    mask               - the register mask
 //
 // Return Value:
-//    The number of the first register contained in the mask.
+//    The number of the first register contained in the mask and updates the `mask` to toggle
+//    the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
-    assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+    assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
     /* Convert the mask to a register number */
 
-    regNumber regNum = (regNumber)BitScanForward(mask);
+    regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+    mask ^= genRegMask(regNum);
 
     return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
     RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-    regMaskTP killedRegs = killRefPosition->registerAssignment;
+    SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
     while (killedRegs != RBM_NONE)
     {
         regNumber  killedReg        = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
     // For each physical register that can hold a GC type,
     // if it is occupied by an interval of a GC type, spill that interval.
-    regMaskTP candidateRegs = killRefPosition->registerAssignment;
+    SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
     INDEBUG(bool killedRegs = false);
     while (candidateRegs != RBM_NONE)
     {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
     {
         // If we're rotating the register locations at block boundaries, try to use
         // the next higher register number of the appropriate register type.
-        regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
-        regNumber firstReg      = REG_NA;
-        regNumber newReg        = REG_NA;
+        SingleTypeRegSet candidateRegs =
+            allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+        regNumber firstReg = REG_NA;
+        regNumber newReg   = REG_NA;
         while (candidateRegs != RBM_NONE)
         {
             regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
                 // However, we will assert that, at resolution time, no registers contain GC refs.
                 {
                     DBEXEC(VERBOSE, printf("           "));
-                    regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+                    SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
                     while (candidateRegs != RBM_NONE)
                     {
                         regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
         availableRegCount = REG_INT_COUNT;
     }
 
-    if (availableRegCount < (sizeof(regMaskTP) * 8))
+    static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+    if (availableRegCount < (sizeof(regMaskSmall) * 8))
+    {
+        // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+        actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+    }
+    else if (availableRegCount < (sizeof(regMaskTP) * 8))
     {
-        // Mask out the bits that are between 64 ~ availableRegCount
-        actualRegistersMask = (1ULL << availableRegCount) - 1;
+        // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+        // Subtract one extra for stack.
+        unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+        actualRegistersMask  = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
     }
     else
     {
-        actualRegistersMask = ~RBM_NONE;
+        actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
     }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
     bool rsRegsModified(regMaskTP mask) const
     {
         assert(rsModifiedRegsMaskInitialized);
-        return (rsModifiedRegsMask & mask) != 0;
+        return (rsModifiedRegsMask & mask).IsNonEmpty();
     }
 
     void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall    SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
     regMaskSmall low;
+    regMaskSmall high;
+
 public:
-    constexpr regMaskTP(regMaskSmall regMask)
+
+    regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+        : low(lowMask)
+        , high(highMask)
+    {
+    }
+
+    regMaskTP(regMaskSmall regMask)
         : low(regMask)
     {
     }
@@ -278,12 +289,12 @@ struct regMaskTP
         return low;
     }
 
-    bool IsEmpty()
+    bool IsEmpty() const
     {
         return low == RBM_NONE;
     }
 
-    bool IsNonEmpty()
+    bool IsNonEmpty() const
     {
         return !IsEmpty();
     }
@@ -296,50 +307,52 @@ struct regMaskTP
     void RemoveRegNumFromMask(regNumber reg);
 
     bool IsRegNumInMask(regNumber reg);
+
+    void operator|=(const regMaskTP& second)
+    {
+        low |= second.getLow();
+    }
+
+    void operator^=(const regMaskTP& second)
+    {
+        low ^= second.getLow();
+    }
+
+    void operator^=(const regNumber reg)
+    {
+        low ^= genRegMask(reg);
+    }
+
+    void operator&=(const regMaskTP& second)
+    {
+        low &= second.getLow();
+    }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
     regMaskTP result(first.getLow() ^ second.getLow());
     return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
     regMaskTP result(first.getLow() & second.getLow());
     return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
     regMaskTP result(first.getLow() | second.getLow());
     return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
-    first = first | second;
-    return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
-    first = first ^ second;
-    return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
-    first = first & second;
-    return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
     return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
     return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
     regMaskTP result(~first.getLow());
     return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
     return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
     return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */