Align inner loops

src/coreclr/ToolBox/superpmi/superpmi/icorjitinfo.cpp

@@ -1609,7 +1609,21 @@ void MyICJI::allocMem(ULONG              hotCodeSize,   /* IN */
     jitInstance->mc->cr->AddCall("allocMem");
 
     // TODO-Cleanup: Could hot block size be ever 0?
-    *hotCodeBlock = jitInstance->mc->cr->allocateMemory(hotCodeSize);
+    size_t codeAlignment      = sizeof(void*);
+    size_t hotCodeAlignedSize = static_cast<size_t>(hotCodeSize);
+
+    if ((flag & CORJIT_ALLOCMEM_FLG_32BYTE_ALIGN) != 0)
+    {
+         codeAlignment = 32;
+    }
+    else if ((flag & CORJIT_ALLOCMEM_FLG_16BYTE_ALIGN) != 0)
+    {
+         codeAlignment = 16;
+    }
+    hotCodeAlignedSize = ALIGN_UP_SPMI(hotCodeAlignedSize, codeAlignment);
+    hotCodeAlignedSize = hotCodeAlignedSize + (codeAlignment - sizeof(void*));
+    *hotCodeBlock      = jitInstance->mc->cr->allocateMemory(hotCodeAlignedSize);
+    *hotCodeBlock      = ALIGN_UP_SPMI(*hotCodeBlock, codeAlignment);
 
     if (coldCodeSize > 0)
         *coldCodeBlock = jitInstance->mc->cr->allocateMemory(coldCodeSize);

src/coreclr/inc/clrconfigvalues.h

@@ -302,7 +302,6 @@ RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_UseIBCFile, W("UseIBCFile"), 0, "", CLRConf
 ///
 /// JIT
 ///
-RETAIL_CONFIG_DWORD_INFO_DIRECT_ACCESS(UNSUPPORTED_JitAlignLoops, W("JitAlignLoops"), "Aligns loop targets to 8 byte boundaries")
 CONFIG_DWORD_INFO_EX(INTERNAL_JitBreakEmit, W("JitBreakEmit"), (DWORD)-1, "", CLRConfig::EEConfig_default)
 CONFIG_DWORD_INFO_DIRECT_ACCESS(INTERNAL_JitDebuggable, W("JitDebuggable"), "")
 #if !defined(DEBUG) && !defined(_DEBUG)

src/coreclr/inc/corjitflags.h

@@ -79,45 +79,45 @@ class CORJIT_FLAGS
         CORJIT_FLAG_BBINSTR                 = 29, // Collect basic block profile information
         CORJIT_FLAG_BBOPT                   = 30, // Optimize method based on profile information
         CORJIT_FLAG_FRAMED                  = 31, // All methods have an EBP frame
-        CORJIT_FLAG_ALIGN_LOOPS             = 32, // add NOPs before loops to align them at 16 byte boundaries
+        CORJIT_FLAG_UNUSED12                = 32,
         CORJIT_FLAG_PUBLISH_SECRET_PARAM    = 33, // JIT must place stub secret param into local 0.  (used by IL stubs)
-        CORJIT_FLAG_UNUSED12                = 34,
+        CORJIT_FLAG_UNUSED13                = 34,
         CORJIT_FLAG_SAMPLING_JIT_BACKGROUND = 35, // JIT is being invoked as a result of stack sampling for hot methods in the background
         CORJIT_FLAG_USE_PINVOKE_HELPERS     = 36, // The JIT should use the PINVOKE_{BEGIN,END} helpers instead of emitting inline transitions
         CORJIT_FLAG_REVERSE_PINVOKE         = 37, // The JIT should insert REVERSE_PINVOKE_{ENTER,EXIT} helpers into method prolog/epilog
-        CORJIT_FLAG_UNUSED13                = 38,
+        CORJIT_FLAG_UNUSED14                = 38,
         CORJIT_FLAG_TIER0                   = 39, // This is the initial tier for tiered compilation which should generate code as quickly as possible
         CORJIT_FLAG_TIER1                   = 40, // This is the final tier (for now) for tiered compilation which should generate high quality code
 
 #if defined(TARGET_ARM)
         CORJIT_FLAG_RELATIVE_CODE_RELOCS    = 41, // JIT should generate PC-relative address computations instead of EE relocation records
 #else // !defined(TARGET_ARM)
-        CORJIT_FLAG_UNUSED14                = 41,
+        CORJIT_FLAG_UNUSED15                = 41,
 #endif // !defined(TARGET_ARM)
 
         CORJIT_FLAG_NO_INLINING             = 42, // JIT should not inline any called method into this method
 
-        CORJIT_FLAG_UNUSED15                = 43,
-        CORJIT_FLAG_UNUSED16                = 44,
-        CORJIT_FLAG_UNUSED17                = 45,
-        CORJIT_FLAG_UNUSED18                = 46,
-        CORJIT_FLAG_UNUSED19                = 47,
-        CORJIT_FLAG_UNUSED20                = 48,
-        CORJIT_FLAG_UNUSED21                = 49,
-        CORJIT_FLAG_UNUSED22                = 50,
-        CORJIT_FLAG_UNUSED23                = 51,
-        CORJIT_FLAG_UNUSED24                = 52,
-        CORJIT_FLAG_UNUSED25                = 53,
-        CORJIT_FLAG_UNUSED26                = 54,
-        CORJIT_FLAG_UNUSED27                = 55,
-        CORJIT_FLAG_UNUSED28                = 56,
-        CORJIT_FLAG_UNUSED29                = 57,
-        CORJIT_FLAG_UNUSED30                = 58,
-        CORJIT_FLAG_UNUSED31                = 59,
-        CORJIT_FLAG_UNUSED32                = 60,
-        CORJIT_FLAG_UNUSED33                = 61,
-        CORJIT_FLAG_UNUSED34                = 62,
-        CORJIT_FLAG_UNUSED35                = 63
+        CORJIT_FLAG_UNUSED16                = 43,
+        CORJIT_FLAG_UNUSED17                = 44,
+        CORJIT_FLAG_UNUSED18                = 45,
+        CORJIT_FLAG_UNUSED19                = 46,
+        CORJIT_FLAG_UNUSED20                = 47,
+        CORJIT_FLAG_UNUSED21                = 48,
+        CORJIT_FLAG_UNUSED22                = 49,
+        CORJIT_FLAG_UNUSED23                = 50,
+        CORJIT_FLAG_UNUSED24                = 51,
+        CORJIT_FLAG_UNUSED25                = 52,
+        CORJIT_FLAG_UNUSED26                = 53,
+        CORJIT_FLAG_UNUSED27                = 54,
+        CORJIT_FLAG_UNUSED28                = 55,
+        CORJIT_FLAG_UNUSED29                = 56,
+        CORJIT_FLAG_UNUSED30                = 57,
+        CORJIT_FLAG_UNUSED31                = 58,
+        CORJIT_FLAG_UNUSED32                = 59,
+        CORJIT_FLAG_UNUSED33                = 60,
+        CORJIT_FLAG_UNUSED34                = 61,
+        CORJIT_FLAG_UNUSED35                = 62,
+        CORJIT_FLAG_UNUSED36                = 63
     };
 
     CORJIT_FLAGS()

src/coreclr/inc/jiteeversionguid.h

@@ -31,11 +31,11 @@
 //
 //////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-constexpr GUID JITEEVersionIdentifier = { /* 8e32c24d-62fe-4d78-ae73-eedddb928ee2 */
-    0x8e32c24d,
-    0x62fe,
-    0x4d78,
-    {0xae, 0x73, 0xee, 0xdd, 0xdb, 0x92, 0x8e, 0xe2}
+constexpr GUID JITEEVersionIdentifier = { /* de81f48e-7701-45f2-a91b-1914f88dfd11 */
+    0xde81f48e,
+    0x7701,
+    0x45f2,
+    {0xa9, 0x1b, 0x19, 0x14, 0xf8, 0x8d, 0xfd, 0x11}
 };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

src/coreclr/jit/block.cpp

@@ -505,6 +505,10 @@ void BasicBlock::dspFlags()
     {
         printf("cfe ");
     }
+    if (bbFlags & BBF_LOOP_ALIGN)
+    {
+        printf("align ");
+    }
 }
 
 /*****************************************************************************

src/coreclr/jit/block.h

@@ -448,6 +448,7 @@ struct BasicBlock : private LIR::Range
 
 #define BBF_PATCHPOINT                     MAKE_BBFLAG(36) // Block is a patchpoint
 #define BBF_HAS_CLASS_PROFILE              MAKE_BBFLAG(37) // BB contains a call needing a class profile
+#define BBF_LOOP_ALIGN                     MAKE_BBFLAG(39) // Block is lexically the first block in a loop we intend to align.
 
 // clang-format on
 
@@ -463,6 +464,10 @@ struct BasicBlock : private LIR::Range
     {
         return ((bbFlags & BBF_LOOP_HEAD) != 0);
     }
+    bool isLoopAlign() const
+    {
+        return ((bbFlags & BBF_LOOP_ALIGN) != 0);
+    }
 
 // Flags to update when two blocks are compacted
 

src/coreclr/jit/codegencommon.cpp

@@ -2258,6 +2258,12 @@ void CodeGen::genGenerateMachineCode()
 
     GetEmitter()->emitJumpDistBind();
 
+#if FEATURE_LOOP_ALIGN
+    /* Perform alignment adjustments */
+
+    GetEmitter()->emitLoopAlignAdjustments();
+#endif
+
     /* The code is now complete and final; it should not change after this. */
 }
 
@@ -2345,10 +2351,12 @@ void CodeGen::genEmitMachineCode()
 #ifdef DEBUG
     if (compiler->opts.disAsm || verbose)
     {
-        printf("\n; Total bytes of code %d, prolog size %d, PerfScore %.2f, instruction count %d (MethodHash=%08x) for "
+        printf("\n; Total bytes of code %d, prolog size %d, PerfScore %.2f, instruction count %d, allocated bytes for "
+               "code %d (MethodHash=%08x) for "
                "method %s\n",
-               codeSize, prologSize, compiler->info.compPerfScore, instrCount, compiler->info.compMethodHash(),
-               compiler->info.compFullName);
+               codeSize, prologSize, compiler->info.compPerfScore, instrCount,
+               GetEmitter()->emitTotalHotCodeSize + GetEmitter()->emitTotalColdCodeSize,
+               compiler->info.compMethodHash(), compiler->info.compFullName);
         printf("; ============================================================\n\n");
         printf(""); // in our logic this causes a flush
     }

src/coreclr/jit/codegenlinear.cpp

@@ -311,13 +311,6 @@ void CodeGen::genCodeForBBlist()
 
         genUpdateCurrentFunclet(block);
 
-#ifdef TARGET_XARCH
-        if (ShouldAlignLoops() && block->bbFlags & BBF_LOOP_HEAD)
-        {
-            GetEmitter()->emitLoopAlign();
-        }
-#endif
-
         genLogLabel(block);
 
         // Tell everyone which basic block we're working on
@@ -356,6 +349,14 @@ void CodeGen::genCodeForBBlist()
             needLabel = true;
         }
 
+#if FEATURE_LOOP_ALIGN
+        if (GetEmitter()->emitEndsWithAlignInstr())
+        {
+            // we had better be planning on starting a new IG
+            assert(needLabel);
+        }
+#endif
+
         if (needLabel)
         {
             // Mark a label and update the current set of live GC refs
@@ -667,10 +668,6 @@ void CodeGen::genCodeForBBlist()
 
         switch (block->bbJumpKind)
         {
-            case BBJ_ALWAYS:
-                inst_JMP(EJ_jmp, block->bbJumpDest);
-                break;
-
             case BBJ_RETURN:
                 genExitCode(block);
                 break;
@@ -741,15 +738,55 @@ void CodeGen::genCodeForBBlist()
 #endif // !FEATURE_EH_FUNCLETS
 
             case BBJ_NONE:
-            case BBJ_COND:
             case BBJ_SWITCH:
                 break;
 
+            case BBJ_ALWAYS:
+                inst_JMP(EJ_jmp, block->bbJumpDest);
+                FALLTHROUGH;
+
+            case BBJ_COND:
+
+#if FEATURE_LOOP_ALIGN
+                // This is the last place where we operate on blocks and after this, we operate
+                // on IG. Hence, if we know that the destination of "block" is the first block
+                // of a loop and needs alignment (it has BBF_LOOP_ALIGN), then "block" represents
+                // end of the loop. Propagate that information on the IG through "igLoopBackEdge".
+                //
+                // During emitter, this information will be used to calculate the loop size.
+                // Depending on the loop size, decision of whether to align a loop or not will be taken.
+
+                if (block->bbJumpDest->isLoopAlign())
+                {
+                    GetEmitter()->emitSetLoopBackEdge(block->bbJumpDest);
+                }
+#endif
+                break;
+
             default:
                 noway_assert(!"Unexpected bbJumpKind");
                 break;
         }
 
+#if FEATURE_LOOP_ALIGN
+
+        // If next block is the first block of a loop (identified by BBF_LOOP_ALIGN),
+        // then need to add align instruction in current "block". Also mark the
+        // corresponding IG with IGF_LOOP_ALIGN to know that there will be align
+        // instructions at the end of that IG.
+        //
+        // For non-adaptive alignment, add alignment instruction of size depending on the
+        // compJitAlignLoopBoundary.
+        // For adaptive alignment, alignment instruction will always be of 15 bytes.
+
+        if ((block->bbNext != nullptr) && (block->bbNext->isLoopAlign()))
+        {
+            assert(ShouldAlignLoops());
+
+            GetEmitter()->emitLoopAlignment();
+        }
+#endif
+
 #if defined(DEBUG) && defined(USING_VARIABLE_LIVE_RANGE)
         if (compiler->verbose)
         {

src/coreclr/jit/compiler.cpp

@@ -2308,7 +2308,7 @@ void Compiler::compSetProcessor()
     opts.compUseCMOV = jitFlags.IsSet(JitFlags::JIT_FLAG_USE_CMOV);
 #ifdef DEBUG
     if (opts.compUseCMOV)
-        opts.compUseCMOV = !compStressCompile(STRESS_USE_CMOV, 50);
+        opts.compUseCMOV                = !compStressCompile(STRESS_USE_CMOV, 50);
 #endif // DEBUG
 
 #endif // TARGET_X86
@@ -2615,6 +2615,29 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
     opts.compDbgInfo = jitFlags->IsSet(JitFlags::JIT_FLAG_DEBUG_INFO);
     opts.compDbgEnC  = jitFlags->IsSet(JitFlags::JIT_FLAG_DEBUG_EnC);
 
+#ifdef DEBUG
+    opts.compJitAlignLoopAdaptive       = JitConfig.JitAlignLoopAdaptive() == 1;
+    opts.compJitAlignLoopBoundary       = (unsigned short)JitConfig.JitAlignLoopBoundary();
+    opts.compJitAlignLoopMinBlockWeight = (unsigned short)JitConfig.JitAlignLoopMinBlockWeight();
+
+    opts.compJitAlignLoopForJcc      = JitConfig.JitAlignLoopForJcc() == 1;
+    opts.compJitAlignLoopMaxCodeSize = (unsigned short)JitConfig.JitAlignLoopMaxCodeSize();
+#else
+    opts.compJitAlignLoopAdaptive       = true;
+    opts.compJitAlignLoopBoundary       = DEFAULT_ALIGN_LOOP_BOUNDARY;
+    opts.compJitAlignLoopMinBlockWeight = DEFAULT_ALIGN_LOOP_MIN_BLOCK_WEIGHT;
+#endif
+    if (opts.compJitAlignLoopAdaptive)
+    {
+        opts.compJitAlignPaddingLimit = (opts.compJitAlignLoopBoundary >> 1) - 1;
+    }
+    else
+    {
+        opts.compJitAlignPaddingLimit = opts.compJitAlignLoopBoundary - 1;
+    }
+
+    assert(isPow2(opts.compJitAlignLoopBoundary));
+
 #if REGEN_SHORTCUTS || REGEN_CALLPAT
     // We never want to have debugging enabled when regenerating GC encoding patterns
     opts.compDbgCode = false;
@@ -3913,19 +3936,17 @@ void Compiler::compSetOptimizationLevel()
             codeGen->setFrameRequired(true);
 #endif
 
-        if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_RELOC))
+        if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT))
         {
-            codeGen->SetAlignLoops(false); // loop alignment not supported for prejitted code
-
-            // The zapper doesn't set JitFlags::JIT_FLAG_ALIGN_LOOPS, and there is
-            // no reason for it to set it as the JIT doesn't currently support loop alignment
-            // for prejitted images. (The JIT doesn't know the final address of the code, hence
+            // The JIT doesn't currently support loop alignment for prejitted images.
+            // (The JIT doesn't know the final address of the code, hence
             // it can't align code based on unknown addresses.)
-            assert(!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_ALIGN_LOOPS));
+
+            codeGen->SetAlignLoops(false); // loop alignment not supported for prejitted code
         }
         else
         {
-            codeGen->SetAlignLoops(opts.jitFlags->IsSet(JitFlags::JIT_FLAG_ALIGN_LOOPS));
+            codeGen->SetAlignLoops(JitConfig.JitAlignLoops() == 1);
         }
     }
 

src/coreclr/jit/compiler.h

@@ -6367,6 +6367,8 @@ class Compiler
 
     void optFindNaturalLoops();
 
+    void optIdentifyLoopsForAlignment();
+
     // Ensures that all the loops in the loop nest rooted at "loopInd" (an index into the loop table) are 'canonical' --
     // each loop has a unique "top."  Returns "true" iff the flowgraph has been modified.
     bool optCanonicalizeLoopNest(unsigned char loopInd);
@@ -9036,6 +9038,43 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
         bool dspGCtbls;       // Display the GC tables
 #endif
 
+// Default numbers used to perform loop alignment. All the numbers are choosen
+// based on experimenting with various benchmarks.
+
+// Default minimum loop block weight required to enable loop alignment.
+#define DEFAULT_ALIGN_LOOP_MIN_BLOCK_WEIGHT 4
+
+// By default a loop will be aligned at 32B address boundary to get better
+// performance as per architecture manuals.
+#define DEFAULT_ALIGN_LOOP_BOUNDARY 0x20
+
+// For non-adaptive loop alignment, by default, only align a loop whose size is
+// at most 3 times the alignment block size. If the loop is bigger than that, it is most
+// likely complicated enough that loop alignment will not impact performance.
+#define DEFAULT_MAX_LOOPSIZE_FOR_ALIGN DEFAULT_ALIGN_LOOP_BOUNDARY * 3
+
+#ifdef DEBUG
+        // Loop alignment variables
+
+        // If set, for non-adaptive alignment, ensure loop jmps are not on or cross alignment boundary.
+        bool compJitAlignLoopForJcc;
+#endif
+        // For non-adaptive alignment, minimum loop size (in bytes) for which alignment will be done.
+        unsigned short compJitAlignLoopMaxCodeSize;
+
+        // Minimum weight needed for the first block of a loop to make it a candidate for alignment.
+        unsigned short compJitAlignLoopMinBlockWeight;
+
+        // For non-adaptive alignment, address boundary (power of 2) at which loop alignment should
+        // be done. By default, 32B.
+        unsigned short compJitAlignLoopBoundary;
+
+        // Padding limit to align a loop.
+        unsigned short compJitAlignPaddingLimit;
+
+        // If set, perform adaptive loop alignment that limits number of padding based on loop size.
+        bool compJitAlignLoopAdaptive;
+
 #ifdef LATE_DISASM
         bool doLateDisasm; // Run the late disassembler
 #endif                     // LATE_DISASM