Fix explicit offset of ByRefLike fields. (#111584)

* Fix explicit offset of ByRefLike fields.

A ByRefLike type was previously forced to be
pointer aligned so ref fields were implicitly aligned.
Using Explicit layout one can create valid aligned
ByRefLike types with no object pointers. This change
tkaes into account the base offset of the ByRefLike type and
includes that for determining offsets of fields in the
ByRefLike type.

---------

Co-authored-by: Michal Strehovský <[email protected]>

Author: AaronRobinsonMSFT

src/coreclr/tools/Common/Compiler/Int128FieldLayoutAlgorithm.cs

@@ -61,6 +61,12 @@ public override bool ComputeContainsGCPointers(DefType type)
             return false;
         }
 
+        public override bool ComputeContainsByRefs(DefType type)
+        {
+            Debug.Assert(!_fallbackAlgorithm.ComputeContainsByRefs(type));
+            return false;
+        }
+
         public override bool ComputeIsUnsafeValueType(DefType type)
         {
             Debug.Assert(!_fallbackAlgorithm.ComputeIsUnsafeValueType(type));

src/coreclr/tools/Common/Compiler/VectorFieldLayoutAlgorithm.cs

@@ -133,6 +133,12 @@ public override bool ComputeContainsGCPointers(DefType type)
             return false;
         }
 
+        public override bool ComputeContainsByRefs(DefType type)
+        {
+            Debug.Assert(!_fallbackAlgorithm.ComputeContainsByRefs(type));
+            return false;
+        }
+
         public override bool ComputeIsUnsafeValueType(DefType type)
         {
             Debug.Assert(!_fallbackAlgorithm.ComputeIsUnsafeValueType(type));

src/coreclr/tools/Common/JitInterface/SwiftPhysicalLowering.cs

@@ -101,7 +101,7 @@ protected override LoweredType GetIntervalDataForType(int offset, TypeDesc field
                 return LoweredType.Opaque;
             }
 
-            protected override bool NeedsRecursiveLayout(int offset, TypeDesc fieldType) => fieldType.IsValueType && !fieldType.IsPrimitiveNumeric;
+            protected override bool NeedsRecursiveLayout(TypeDesc fieldType) => fieldType.IsValueType && !fieldType.IsPrimitiveNumeric;
 
             private List<FieldLayoutInterval> CreateConsolidatedIntervals()
             {

src/coreclr/tools/Common/TypeSystem/Common/DefType.FieldLayout.cs

@@ -78,6 +78,16 @@ private static class FieldLayoutFlags
             /// True if the type transitively has a Vector<T> in it or is Vector<T>
             /// </summary>
             public const int IsVectorTOrHasVectorTFields = 0x1000;
+
+            /// <summary>
+            /// True if ContainsByRefs has been computed
+            /// </summary>
+            public const int ComputedContainsByRefs = 0x2000;
+
+            /// <summary>
+            /// True if the type contains byrefs
+            /// </summary>
+            public const int ContainsByRefs = 0x4000;
         }
 
         private sealed class StaticBlockInfo
@@ -113,6 +123,21 @@ public bool ContainsGCPointers
             }
         }
 
+        /// <summary>
+        /// Does a type transitively have any fields which are byrefs
+        /// </summary>
+        public bool ContainsByRefs
+        {
+            get
+            {
+                if (!_fieldLayoutFlags.HasFlags(FieldLayoutFlags.ComputedContainsByRefs))
+                {
+                    ComputeTypeContainsByRefs();
+                }
+                return _fieldLayoutFlags.HasFlags(FieldLayoutFlags.ContainsByRefs);
+            }
+        }
+
         /// <summary>
         /// Does a type transitively have any fields which are marked with System.Runtime.CompilerServices.UnsafeValueTypeAttribute
         /// </summary>
@@ -545,6 +570,20 @@ public void ComputeTypeContainsGCPointers()
             _fieldLayoutFlags.AddFlags(flagsToAdd);
         }
 
+        public void ComputeTypeContainsByRefs()
+        {
+            if (_fieldLayoutFlags.HasFlags(FieldLayoutFlags.ComputedContainsByRefs))
+                return;
+
+            int flagsToAdd = FieldLayoutFlags.ComputedContainsByRefs;
+            if (this.Context.GetLayoutAlgorithmForType(this).ComputeContainsByRefs(this))
+            {
+                flagsToAdd |= FieldLayoutFlags.ContainsByRefs;
+            }
+
+            _fieldLayoutFlags.AddFlags(flagsToAdd);
+        }
+
         public void ComputeIsUnsafeValueType()
         {
             if (_fieldLayoutFlags.HasFlags(FieldLayoutFlags.ComputedIsUnsafeValueType))

src/coreclr/tools/Common/TypeSystem/Common/ExplicitLayoutValidator.cs

@@ -60,12 +60,12 @@ protected override FieldLayoutTag GetIntervalDataForType(int offset, TypeDesc fi
                 else if (fieldType.IsValueType)
                 {
                     MetadataType mdType = (MetadataType)fieldType;
-                    if (!mdType.ContainsGCPointers && !mdType.IsByRefLike)
+                    if (!mdType.ContainsGCPointers && !mdType.ContainsByRefs)
                     {
                         // Plain value type, mark the entire range as NonORef
                         return FieldLayoutTag.NonORef;
                     }
-                    Debug.Fail("We should recurse on value types with GC pointers or ByRefLike types");
+                    Debug.Fail("We should recurse on value types with GC pointers or byrefs");
                     return FieldLayoutTag.Empty;
                 }
                 else
@@ -74,20 +74,16 @@ protected override FieldLayoutTag GetIntervalDataForType(int offset, TypeDesc fi
                 }
             }
 
-            protected override bool NeedsRecursiveLayout(int offset, TypeDesc fieldType)
+            protected override bool NeedsRecursiveLayout(TypeDesc fieldType)
             {
-                if (!fieldType.IsValueType || !((MetadataType)fieldType).ContainsGCPointers && !fieldType.IsByRefLike)
+                if (!fieldType.IsValueType)
                 {
                     return false;
                 }
 
-                if (offset % PointerSize != 0)
-                {
-                    // Misaligned struct with GC pointers or ByRef
-                    ThrowFieldLayoutError(offset);
-                }
-
-                return true;
+                // Valuetypes with GC references or byrefs need to be checked for overlaps and alignment
+                MetadataType mdType = (MetadataType)fieldType;
+                return mdType.ContainsGCPointers || mdType.ContainsByRefs;
             }
 
             private void ThrowFieldLayoutError(int offset)

src/coreclr/tools/Common/TypeSystem/Common/FieldLayoutAlgorithm.cs

@@ -31,6 +31,11 @@ public abstract class FieldLayoutAlgorithm
         /// </summary>
         public abstract bool ComputeContainsGCPointers(DefType type);
 
+        /// <summary>
+        /// Compute whether the fields of the specified type contains a byref.
+        /// </summary>
+        public abstract bool ComputeContainsByRefs(DefType type);
+
         /// <summary>
         /// Compute whether the specified type is a value type that transitively has UnsafeValueTypeAttribute
         /// </summary>

src/coreclr/tools/Common/TypeSystem/Common/FieldLayoutIntervalCalculator.cs

@@ -55,7 +55,7 @@ public FieldLayoutIntervalCalculator(int pointerSize)
             PointerSize = pointerSize;
         }
 
-        protected abstract bool NeedsRecursiveLayout(int offset, TypeDesc fieldType);
+        protected abstract bool NeedsRecursiveLayout(TypeDesc fieldType);
 
         protected abstract TIntervalTag GetIntervalDataForType(int offset, TypeDesc fieldType);
 
@@ -84,7 +84,7 @@ private int GetFieldSize(TypeDesc fieldType)
 
         public void AddToFieldLayout(int offset, TypeDesc fieldType, bool addTrailingEmptyInterval)
         {
-            if (NeedsRecursiveLayout(offset, fieldType))
+            if (NeedsRecursiveLayout(fieldType))
             {
                 if (fieldType is MetadataType { IsInlineArray: true } mdType)
                 {
@@ -126,7 +126,7 @@ public void AddToFieldLayout(int offset, TypeDesc fieldType, bool addTrailingEmp
 
         private void AddToFieldLayout(List<FieldLayoutInterval> fieldLayout, int offset, TypeDesc fieldType)
         {
-            if (NeedsRecursiveLayout(offset, fieldType))
+            if (NeedsRecursiveLayout(fieldType))
             {
                 if (fieldType is MetadataType { IsInlineArray: true } mdType)
                 {