Add internal Encoding.TryGetBytes (#84609)

stephentoub · web-flow · commit 19ff97834663 · 2023-04-11T06:44:16.000-04:00
diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIEncoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIEncoding.cs
@@ -322,8 +322,31 @@ public override unsafe int GetBytes(ReadOnlySpan<char> chars, Span<byte> bytes)
             }
         }
 
+        // TODO https://github.com/dotnet/runtime/issues/84425: Make this public.
+        /// <summary>Encodes into a span of bytes a set of characters from the specified read-only span if the destination is large enough.</summary>
+        /// <param name="chars">The span containing the set of characters to encode.</param>
+        /// <param name="bytes">The byte span to hold the encoded bytes.</param>
+        /// <param name="bytesWritten">Upon successful completion of the operation, the number of bytes encoded into <paramref name="bytes"/>.</param>
+        /// <returns><see langword="true"/> if all of the characters were encoded into the destination; <see langword="false"/> if the destination was too small to contain all the encoded bytes.</returns>
+        internal override unsafe bool TryGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int bytesWritten)
+        {
+            fixed (char* charsPtr = &MemoryMarshal.GetReference(chars))
+            fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes))
+            {
+                int written = GetBytesCommon(charsPtr, chars.Length, bytesPtr, bytes.Length, throwForDestinationOverflow: false);
+                if (written >= 0)
+                {
+                    bytesWritten = written;
+                    return true;
+                }
+
+                bytesWritten = 0;
+                return false;
+            }
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount)
+        private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount, bool throwForDestinationOverflow = true)
         {
             // Common helper method for all non-EncoderNLS entry points to GetBytes.
             // A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32.
@@ -347,7 +370,7 @@ private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int
             {
                 // Simple narrowing conversion couldn't operate on entire buffer - invoke fallback.
 
-                return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten);
+                return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten, throwForDestinationOverflow);
             }
         }
 
@@ -360,7 +383,7 @@ private protected sealed override unsafe int GetBytesFast(char* pChars, int char
             return bytesWritten;
         }
 
-        private protected sealed override unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS? encoder)
+        private protected sealed override unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS? encoder, bool throwForDestinationOverflow = true)
         {
             // We special-case EncoderReplacementFallback if it's telling us to write a single ASCII char,
             // since we believe this to be relatively common and we can handle it more efficiently than
@@ -406,7 +429,7 @@ private protected sealed override unsafe int GetBytesWithFallback(ReadOnlySpan<c
             }
             else
             {
-                return base.GetBytesWithFallback(chars, originalCharsLength, bytes, originalBytesLength, encoder);
+                return base.GetBytesWithFallback(chars, originalCharsLength, bytes, originalBytesLength, encoder, throwForDestinationOverflow);
             }
         }
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.Internal.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.Internal.cs
@@ -485,7 +485,7 @@ private protected virtual unsafe int GetBytesFast(char* pChars, int charsLength,
         /// If the destination buffer is not large enough to hold the entirety of the transcoded data.
         /// </exception>
         [MethodImpl(MethodImplOptions.NoInlining)]
-        private protected unsafe int GetBytesWithFallback(char* pOriginalChars, int originalCharCount, byte* pOriginalBytes, int originalByteCount, int charsConsumedSoFar, int bytesWrittenSoFar)
+        private protected unsafe int GetBytesWithFallback(char* pOriginalChars, int originalCharCount, byte* pOriginalBytes, int originalByteCount, int charsConsumedSoFar, int bytesWrittenSoFar, bool throwForDestinationOverflow = true)
         {
             // This is a stub method that's marked "no-inlining" so that it we don't stack-spill spans
             // into our immediate caller. Doing so increases the method prolog in what's supposed to
@@ -499,7 +499,8 @@ private protected unsafe int GetBytesWithFallback(char* pOriginalChars, int orig
                 originalCharsLength: originalCharCount,
                 bytes: new Span<byte>(pOriginalBytes, originalByteCount).Slice(bytesWrittenSoFar),
                 originalBytesLength: originalByteCount,
-                encoder: null);
+                encoder: null,
+                throwForDestinationOverflow);
         }
 
         /// <summary>
@@ -508,7 +509,7 @@ private protected unsafe int GetBytesWithFallback(char* pOriginalChars, int orig
         /// and <paramref name="bytesWrittenSoFar"/> signal where in the provided buffers the fallback loop
         /// should begin operating. The behavior of this method is to drain any leftover data in the
         /// <see cref="EncoderNLS"/> instance, then to invoke the <see cref="GetBytesFast"/> virtual method
-        /// after data has been drained, then to call <see cref="GetBytesWithFallback(ReadOnlySpan{char}, int, Span{byte}, int, EncoderNLS)"/>.
+        /// after data has been drained, then to call <see cref="GetBytesWithFallback(ReadOnlySpan{char}, int, Span{byte}, int, EncoderNLS, bool)"/>.
         /// </summary>
         /// <returns>
         /// The total number of bytes written to <paramref name="pOriginalBytes"/>, including <paramref name="bytesWrittenSoFar"/>.
@@ -582,7 +583,7 @@ private unsafe int GetBytesWithFallback(char* pOriginalChars, int originalCharCo
         /// implementation, deferring to the base implementation if needed. This method calls <see cref="ThrowBytesOverflow"/>
         /// if necessary.
         /// </remarks>
-        private protected virtual unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS? encoder)
+        private protected virtual unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS? encoder, bool throwForDestinationOverflow = true)
         {
             Debug.Assert(!chars.IsEmpty, "Caller shouldn't invoke this method with an empty input buffer.");
             Debug.Assert(originalCharsLength >= 0, "Caller provided invalid parameter.");
@@ -678,8 +679,15 @@ private protected virtual unsafe int GetBytesWithFallback(ReadOnlySpan<char> cha
                     // The line below will also throw if the encoder couldn't make any progress at all
                     // because the output buffer wasn't large enough to contain the result of even
                     // a single scalar conversion or fallback.
-
-                    ThrowBytesOverflow(encoder, nothingEncoded: bytes.Length == originalBytesLength);
+                    if (throwForDestinationOverflow)
+                    {
+                        ThrowBytesOverflow(encoder, nothingEncoded: bytes.Length == originalBytesLength);
+                    }
+                    else
+                    {
+                        Debug.Assert(encoder is null);
+                        return -1;
+                    }
                 }
 
                 // If an EncoderNLS instance is active, update its "total consumed character count" value.
diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs
@@ -733,6 +733,25 @@ public virtual unsafe int GetBytes(ReadOnlySpan<char> chars, Span<byte> bytes)
             }
         }
 
+        // TODO https://github.com/dotnet/runtime/issues/84425: Make this public.
+        /// <summary>Encodes into a span of bytes a set of characters from the specified read-only span if the destination is large enough.</summary>
+        /// <param name="chars">The span containing the set of characters to encode.</param>
+        /// <param name="bytes">The byte span to hold the encoded bytes.</param>
+        /// <param name="bytesWritten">Upon successful completion of the operation, the number of bytes encoded into <paramref name="bytes"/>.</param>
+        /// <returns><see langword="true"/> if all of the characters were encoded into the destination; <see langword="false"/> if the destination was too small to contain all the encoded bytes.</returns>
+        internal virtual bool TryGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int bytesWritten)
+        {
+            int required = GetByteCount(chars);
+            if (required <= bytes.Length)
+            {
+                bytesWritten = GetBytes(chars, bytes);
+                return true;
+            }
+
+            bytesWritten = 0;
+            return false;
+        }
+
         // Returns the number of characters produced by decoding the given byte
         // array.
         //
diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Latin1Encoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Latin1Encoding.cs
@@ -234,6 +234,29 @@ public override unsafe int GetBytes(ReadOnlySpan<char> chars, Span<byte> bytes)
             }
         }
 
+        // TODO https://github.com/dotnet/runtime/issues/84425: Make this public.
+        /// <summary>Encodes into a span of bytes a set of characters from the specified read-only span if the destination is large enough.</summary>
+        /// <param name="chars">The span containing the set of characters to encode.</param>
+        /// <param name="bytes">The byte span to hold the encoded bytes.</param>
+        /// <param name="bytesWritten">Upon successful completion of the operation, the number of bytes encoded into <paramref name="bytes"/>.</param>
+        /// <returns><see langword="true"/> if all of the characters were encoded into the destination; <see langword="false"/> if the destination was too small to contain all the encoded bytes.</returns>
+        internal override unsafe bool TryGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int bytesWritten)
+        {
+            fixed (char* charsPtr = &MemoryMarshal.GetReference(chars))
+            fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes))
+            {
+                int written = GetBytesCommon(charsPtr, chars.Length, bytesPtr, bytes.Length, throwForDestinationOverflow: false);
+                if (written >= 0)
+                {
+                    bytesWritten = written;
+                    return true;
+                }
+
+                bytesWritten = 0;
+                return false;
+            }
+        }
+
         public override unsafe int GetBytes(string s, int charIndex, int charCount, byte[] bytes, int byteIndex)
         {
             if (s is null || bytes is null)
@@ -269,7 +292,7 @@ public override unsafe int GetBytes(string s, int charIndex, int charCount, byte
 
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount)
+        private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount, bool throwForDestinationOverflow = true)
         {
             // Common helper method for all non-EncoderNLS entry points to GetBytes.
             // A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32.
@@ -293,7 +316,7 @@ private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int
             {
                 // Simple narrowing conversion couldn't operate on entire buffer - invoke fallback.
 
-                return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten);
+                return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten, throwForDestinationOverflow);
             }
         }
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/UTF8Encoding.Sealed.cs b/src/libraries/System.Private.CoreLib/src/System/Text/UTF8Encoding.Sealed.cs
@@ -146,6 +146,13 @@ private unsafe string GetStringForSmallInput(byte[] bytes)
 
                 return new string(new ReadOnlySpan<char>(ref *pDestination, charsWritten)); // this overload of ROS ctor doesn't validate length
             }
+
+            // TODO https://github.com/dotnet/runtime/issues/84425: Make this public.
+            // TODO: Make this [Intrinsic] and handle JIT-time UTF8 encoding of literal `chars`.
+            internal override unsafe bool TryGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int bytesWritten)
+            {
+                return base.TryGetBytes(chars, bytes, out bytesWritten);
+            }
         }
     }
 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/UTF8Encoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/UTF8Encoding.cs
@@ -369,8 +369,31 @@ public override unsafe int GetBytes(ReadOnlySpan<char> chars, Span<byte> bytes)
             }
         }
 
+        // TODO https://github.com/dotnet/runtime/issues/84425: Make this public.
+        /// <summary>Encodes into a span of bytes a set of characters from the specified read-only span if the destination is large enough.</summary>
+        /// <param name="chars">The span containing the set of characters to encode.</param>
+        /// <param name="bytes">The byte span to hold the encoded bytes.</param>
+        /// <param name="bytesWritten">Upon successful completion of the operation, the number of bytes encoded into <paramref name="bytes"/>.</param>
+        /// <returns><see langword="true"/> if all of the characters were encoded into the destination; <see langword="false"/> if the destination was too small to contain all the encoded bytes.</returns>
+        internal override unsafe bool TryGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int bytesWritten)
+        {
+            fixed (char* charsPtr = &MemoryMarshal.GetReference(chars))
+            fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes))
+            {
+                int written = GetBytesCommon(charsPtr, chars.Length, bytesPtr, bytes.Length, throwForDestinationOverflow: false);
+                if (written >= 0)
+                {
+                    bytesWritten = written;
+                    return true;
+                }
+
+                bytesWritten = 0;
+                return false;
+            }
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount)
+        private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount, bool throwForDestinationOverflow = true)
         {
             // Common helper method for all non-EncoderNLS entry points to GetBytes.
             // A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32.
@@ -394,7 +417,7 @@ private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int
             {
                 // Simple narrowing conversion couldn't operate on entire buffer - invoke fallback.
 
-                return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten);
+                return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten, throwForDestinationOverflow);
             }
         }
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs
@@ -441,7 +441,7 @@ public bool AppendFormatted<T>(T value, int alignment, string? format)
             /// <param name="value">The span to write.</param>
             public bool AppendFormatted(scoped ReadOnlySpan<char> value)
             {
-                if (FromUtf16(value, _destination.Slice(_pos), out _, out int bytesWritten) == OperationStatus.Done)
+                if (Encoding.UTF8.TryGetBytes(value, _destination.Slice(_pos), out int bytesWritten))
                 {
                     _pos += bytesWritten;
                     return true;

Original file line number	Diff line number	Diff line change
`@@ -322,8 +322,31 @@ public override unsafe int GetBytes(ReadOnlySpan<char> chars, Span<byte> bytes)`
`322`	`322`	`}`
`323`	`323`	`}`
`324`	`324`
	`325`	`+ // TODO https://github.com/dotnet/runtime/issues/84425: Make this public.`
	`326`	`+ /// <summary>Encodes into a span of bytes a set of characters from the specified read-only span if the destination is large enough.</summary>`
	`327`	`+ /// <param name="chars">The span containing the set of characters to encode.</param>`
	`328`	`+ /// <param name="bytes">The byte span to hold the encoded bytes.</param>`
	`329`	`+ /// <param name="bytesWritten">Upon successful completion of the operation, the number of bytes encoded into <paramref name="bytes"/>.</param>`
	`330`	`+ /// <returns><see langword="true"/> if all of the characters were encoded into the destination; <see langword="false"/> if the destination was too small to contain all the encoded bytes.</returns>`
	`331`	`+ internal override unsafe bool TryGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int bytesWritten)`
	`332`	`+ {`
	`333`	`+ fixed (char* charsPtr = &MemoryMarshal.GetReference(chars))`
	`334`	`+ fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes))`
	`335`	`+ {`
	`336`	`+ int written = GetBytesCommon(charsPtr, chars.Length, bytesPtr, bytes.Length, throwForDestinationOverflow: false);`
	`337`	`+ if (written >= 0)`
	`338`	`+ {`
	`339`	`+ bytesWritten = written;`
	`340`	`+ return true;`
	`341`	`+ }`
	`342`	`+`
	`343`	`+ bytesWritten = 0;`
	`344`	`+ return false;`
	`345`	`+ }`
	`346`	`+ }`
	`347`	`+`
`325`	`348`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`326`		`- private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount)`
	`349`	`+ private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount, bool throwForDestinationOverflow = true)`
`327`	`350`	`{`
`328`	`351`	`// Common helper method for all non-EncoderNLS entry points to GetBytes.`
`329`	`352`	`// A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32.`
`@@ -347,7 +370,7 @@ private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int`
`347`	`370`	`{`
`348`	`371`	`// Simple narrowing conversion couldn't operate on entire buffer - invoke fallback.`
`349`	`372`
`350`		`- return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten);`
	`373`	`+ return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten, throwForDestinationOverflow);`
`351`	`374`	`}`
`352`	`375`	`}`
`353`	`376`
`@@ -360,7 +383,7 @@ private protected sealed override unsafe int GetBytesFast(char* pChars, int char`
`360`	`383`	`return bytesWritten;`
`361`	`384`	`}`
`362`	`385`
`363`		`- private protected sealed override unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS? encoder)`
	`386`	`+ private protected sealed override unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS? encoder, bool throwForDestinationOverflow = true)`
`364`	`387`	`{`
`365`	`388`	`// We special-case EncoderReplacementFallback if it's telling us to write a single ASCII char,`
`366`	`389`	`// since we believe this to be relatively common and we can handle it more efficiently than`
`@@ -406,7 +429,7 @@ private protected sealed override unsafe int GetBytesWithFallback(ReadOnlySpan<c`
`406`	`429`	`}`
`407`	`430`	`else`
`408`	`431`	`{`
`409`		`- return base.GetBytesWithFallback(chars, originalCharsLength, bytes, originalBytesLength, encoder);`
	`432`	`+ return base.GetBytesWithFallback(chars, originalCharsLength, bytes, originalBytesLength, encoder, throwForDestinationOverflow);`
`410`	`433`	`}`
`411`	`434`	`}`
`412`	`435`
Original file line number	Diff line number	Diff line change
`@@ -146,6 +146,13 @@ private unsafe string GetStringForSmallInput(byte[] bytes)`
`146`	`146`
`147`	`147`	`return new string(new ReadOnlySpan<char>(ref *pDestination, charsWritten)); // this overload of ROS ctor doesn't validate length`
`148`	`148`	`}`
	`149`	`+`
	`150`	`+ // TODO https://github.com/dotnet/runtime/issues/84425: Make this public.`
	`151`	+ // TODO: Make this [Intrinsic] and handle JIT-time UTF8 encoding of literal `chars`.
	`152`	`+ internal override unsafe bool TryGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int bytesWritten)`
	`153`	`+ {`
	`154`	`+ return base.TryGetBytes(chars, bytes, out bytesWritten);`
	`155`	`+ }`
`149`	`156`	`}`
`150`	`157`	`}`
`151`	`158`	`}`
Original file line number	Diff line number	Diff line change
`@@ -441,7 +441,7 @@ public bool AppendFormatted<T>(T value, int alignment, string? format)`
`441`	`441`	`/// <param name="value">The span to write.</param>`
`442`	`442`	`public bool AppendFormatted(scoped ReadOnlySpan<char> value)`
`443`	`443`	`{`
`444`		`- if (FromUtf16(value, _destination.Slice(_pos), out _, out int bytesWritten) == OperationStatus.Done)`
	`444`	`+ if (Encoding.UTF8.TryGetBytes(value, _destination.Slice(_pos), out int bytesWritten))`
`445`	`445`	`{`
`446`	`446`	`_pos += bytesWritten;`
`447`	`447`	`return true;`