Skip to content

Convert.Try{From/To}HexString #86556

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 28 commits into from
Oct 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions src/libraries/Common/src/System/HexConverter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -227,22 +227,22 @@ public static char ToCharLower(int value)
return (char)value;
}

public static bool TryDecodeFromUtf16(ReadOnlySpan<char> chars, Span<byte> bytes)
public static bool TryDecodeFromUtf16(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsProcessed)
{
#if SYSTEM_PRIVATE_CORELIB
if (BitConverter.IsLittleEndian && (Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) &&
chars.Length >= Vector128<ushort>.Count * 2)
{
return TryDecodeFromUtf16_Vector128(chars, bytes);
return TryDecodeFromUtf16_Vector128(chars, bytes, out charsProcessed);
}
#endif
return TryDecodeFromUtf16(chars, bytes, out _);
return TryDecodeFromUtf16_Scalar(chars, bytes, out charsProcessed);
}

#if SYSTEM_PRIVATE_CORELIB
[CompExactlyDependsOn(typeof(AdvSimd.Arm64))]
[CompExactlyDependsOn(typeof(Ssse3))]
public static bool TryDecodeFromUtf16_Vector128(ReadOnlySpan<char> chars, Span<byte> bytes)
public static bool TryDecodeFromUtf16_Vector128(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsProcessed)
{
Debug.Assert(Ssse3.IsSupported || AdvSimd.Arm64.IsSupported);
Debug.Assert(chars.Length <= bytes.Length * 2);
Expand Down Expand Up @@ -309,6 +309,7 @@ public static bool TryDecodeFromUtf16_Vector128(ReadOnlySpan<char> chars, Span<b
offset += (nuint)Vector128<ushort>.Count * 2;
if (offset == (nuint)chars.Length)
{
charsProcessed = chars.Length;
return true;
}
// Overlap with the current chunk for trailing elements
Expand All @@ -320,11 +321,13 @@ public static bool TryDecodeFromUtf16_Vector128(ReadOnlySpan<char> chars, Span<b
while (true);

// Fall back to the scalar routine in case of invalid input.
return TryDecodeFromUtf16(chars.Slice((int)offset), bytes.Slice((int)(offset / 2)), out _);
bool fallbackResult = TryDecodeFromUtf16_Scalar(chars.Slice((int)offset), bytes.Slice((int)(offset / 2)), out int fallbackProcessed);
charsProcessed = (int)offset + fallbackProcessed;
return fallbackResult;
}
#endif

public static bool TryDecodeFromUtf16(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsProcessed)
private static bool TryDecodeFromUtf16_Scalar(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsProcessed)
{
Debug.Assert(chars.Length % 2 == 0, "Un-even number of characters provided");
Debug.Assert(chars.Length / 2 == bytes.Length, "Target buffer not right-sized for provided characters");
Expand Down
98 changes: 97 additions & 1 deletion src/libraries/System.Private.CoreLib/src/System/Convert.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2949,12 +2949,82 @@ public static byte[] FromHexString(ReadOnlySpan<char> chars)

byte[] result = GC.AllocateUninitializedArray<byte>(chars.Length >> 1);

if (!HexConverter.TryDecodeFromUtf16(chars, result))
if (!HexConverter.TryDecodeFromUtf16(chars, result, out _))
throw new FormatException(SR.Format_BadHexChar);

return result;
}

/// <summary>
/// Converts the string, which encodes binary data as hex characters, to an equivalent 8-bit unsigned integer span.
/// </summary>
/// <param name="source">The string to convert.</param>
/// <param name="destination">
/// The span in which to write the converted 8-bit unsigned integers. When this method returns value different than <see cref="OperationStatus.Done"/>,
/// either the span remains unmodified or contains an incomplete conversion of <paramref name="source"/>,
/// up to the last valid character.
/// </param>
/// <param name="bytesWritten">When this method returns, contains the number of bytes that were written to <paramref name="destination"/>.</param>
/// <param name="charsConsumed">When this method returns, contains the number of characters that were consumed from <paramref name="source"/>.</param>
/// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
/// <exception cref="ArgumentNullException">Passed string <paramref name="source"/> is null.</exception>
public static OperationStatus FromHexString(string source, Span<byte> destination, out int charsConsumed, out int bytesWritten)
{
ArgumentNullException.ThrowIfNull(source);

return FromHexString(source.AsSpan(), destination, out charsConsumed, out bytesWritten);
}

/// <summary>
/// Converts the span of chars, which encodes binary data as hex characters, to an equivalent 8-bit unsigned integer span.
/// </summary>
/// <param name="source">The span to convert.</param>
/// <param name="destination">
/// The span in which to write the converted 8-bit unsigned integers. When this method returns value different than <see cref="OperationStatus.Done"/>,
/// either the span remains unmodified or contains an incomplete conversion of <paramref name="source"/>,
/// up to the last valid character.
/// </param>
/// <param name="bytesWritten">When this method returns, contains the number of bytes that were written to <paramref name="destination"/>.</param>
/// <param name="charsConsumed">When this method returns, contains the number of characters that were consumed from <paramref name="source"/>.</param>
/// <returns>An <see cref="OperationStatus"/> describing the result of the operation.</returns>
public static OperationStatus FromHexString(ReadOnlySpan<char> source, Span<byte> destination, out int charsConsumed, out int bytesWritten)
{
(int quotient, int remainder) = Math.DivRem(source.Length, 2);

if (quotient == 0)
{
charsConsumed = 0;
bytesWritten = 0;

return remainder == 1 ? OperationStatus.NeedMoreData : OperationStatus.Done;
}

var result = OperationStatus.Done;

if (destination.Length < quotient)
{
source = source.Slice(0, destination.Length * 2);
quotient = destination.Length;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

subjective nit: using a variable called quotient to store both "quotient" and "bytesWritten" is confusing to me, we could simply assign bytesWritten to destination.Length here

result = OperationStatus.DestinationTooSmall;
}
else if (remainder == 1)
{
source = source.Slice(0, source.Length - 1);
destination = destination.Slice(0, destination.Length - 1);
result = OperationStatus.NeedMoreData;
}

if (!HexConverter.TryDecodeFromUtf16(source, destination, out charsConsumed))
{
bytesWritten = charsConsumed / 2;
return OperationStatus.InvalidData;
}

bytesWritten = quotient;
charsConsumed = source.Length;
return result;
}

/// <summary>
/// Converts an array of 8-bit unsigned integers to its equivalent string representation that is encoded with uppercase hex characters.
/// </summary>
Expand Down Expand Up @@ -3006,5 +3076,31 @@ public static string ToHexString(ReadOnlySpan<byte> bytes)

return HexConverter.ToString(bytes, HexConverter.Casing.Upper);
}


/// <summary>
/// Converts a span of 8-bit unsigned integers to its equivalent span representation that is encoded with uppercase hex characters.
/// </summary>
/// <param name="source">A span of 8-bit unsigned integers.</param>
/// <param name="destination">The span representation in hex of the elements in <paramref name="source"/>.</param>
/// <param name="charsWritten">When this method returns, contains the number of chars that were written in <paramref name="destination"/>.</param>
/// <returns>true if the conversion was successful; otherwise, false.</returns>
public static bool TryToHexString(ReadOnlySpan<byte> source, Span<char> destination, out int charsWritten)
{
if (source.Length == 0)
{
charsWritten = 0;
return true;
}
else if (source.Length > int.MaxValue / 2 || destination.Length > source.Length * 2)
{
charsWritten = 0;
return false;
}

HexConverter.EncodeToUtf16(source, destination);
charsWritten = source.Length * 2;
return true;
}
} // class Convert
} // namespace
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Text;
using System.Buffers;
using System.Text;
using Xunit;

namespace System.Tests
Expand Down Expand Up @@ -34,93 +35,124 @@ public static void CompleteValueRange()

private static void TestSequence(byte[] expected, string actual)
{
Assert.Equal(expected, Convert.FromHexString(actual));
byte[] fromResult = Convert.FromHexString(actual);
Assert.Equal(expected, fromResult);

Span<byte> tryResult = new byte[actual.Length / 2];
Assert.Equal(OperationStatus.Done, Convert.FromHexString(actual, tryResult, out int consumed, out int written));
Assert.Equal(fromResult.Length, written);
Assert.Equal(actual.Length, consumed);
AssertExtensions.SequenceEqual(expected, tryResult);
}

[Fact]
public static void InvalidInputString_Null()
{
AssertExtensions.Throws<ArgumentNullException>("s", () => Convert.FromHexString(null));
AssertExtensions.Throws<ArgumentNullException>("source", () => Convert.FromHexString(null, default, out _, out _));
}

[Fact]
public static void InvalidInputString_HalfByte()
[Theory]
[InlineData("01-02-FD-FE-FF")]
[InlineData("00 01 02FD FE FF")]
[InlineData("000102FDFEFF ")]
[InlineData(" 000102FDFEFF")]
[InlineData("\u200B 000102FDFEFF")]
[InlineData("0\u0308")]
[InlineData("0x")]
[InlineData("x0")]
public static void InvalidInputString_FormatException_Or_FalseResult(string invalidInput)
{
Assert.Throws<FormatException>(() => Convert.FromHexString("ABC"));
}
Assert.Throws<FormatException>(() => Convert.FromHexString(invalidInput));

[Fact]
public static void InvalidInputString_BadFirstCharacter()
{
Assert.Throws<FormatException>(() => Convert.FromHexString("x0"));
Span<byte> buffer = stackalloc byte[invalidInput.Length / 2];
Assert.Equal(OperationStatus.InvalidData, Convert.FromHexString(invalidInput.AsSpan(), buffer, out _, out _));
}

[Fact]
public static void InvalidInputString_BadSecondCharacter()
public static void ZeroLength()
{
Assert.Throws<FormatException>(() => Convert.FromHexString("0x"));
}
Assert.Same(Array.Empty<byte>(), Convert.FromHexString(string.Empty));

[Fact]
public static void InvalidInputString_NonAsciiCharacter()
{
Assert.Throws<FormatException>(() => Convert.FromHexString("0\u0308"));
}
OperationStatus convertResult = Convert.FromHexString(string.Empty, Span<byte>.Empty, out int consumed, out int written);

[Fact]
public static void InvalidInputString_ZeroWidthSpace()
{
Assert.Throws<FormatException>(() => Convert.FromHexString("\u200B 000102FDFEFF"));
Assert.Equal(OperationStatus.Done, convertResult);
Assert.Equal(0, written);
Assert.Equal(0, consumed);
}

[Fact]
public static void InvalidInputString_LeadingWhiteSpace()
public static void ToHexFromHexRoundtrip()
{
Assert.Throws<FormatException>(() => Convert.FromHexString(" 000102FDFEFF"));
}
const int loopCount = 50;
Span<char> buffer = stackalloc char[loopCount * 2];
for (int i = 1; i < loopCount; i++)
{
byte[] data = Security.Cryptography.RandomNumberGenerator.GetBytes(i);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We usually use a seeded Random so the tests are reproducible.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with Dan, but since the existing tests in this type were already using Security.Cryptography.RandomNumberGenerator it's acceptable to keep it.

string hex = Convert.ToHexString(data);

[Fact]
public static void InvalidInputString_TrailingWhiteSpace()
{
Assert.Throws<FormatException>(() => Convert.FromHexString("000102FDFEFF "));
}
Span<char> currentBuffer = buffer.Slice(0, i * 2);
bool tryHex = Convert.TryToHexString(data, currentBuffer, out int written);
Assert.True(tryHex);
AssertExtensions.SequenceEqual(hex.AsSpan(), currentBuffer);
Assert.Equal(hex.Length, written);

[Fact]
public static void InvalidInputString_WhiteSpace()
{
Assert.Throws<FormatException>(() => Convert.FromHexString("00 01 02FD FE FF"));
}
TestSequence(data, hex);
TestSequence(data, hex.ToLowerInvariant());
TestSequence(data, hex.ToUpperInvariant());

[Fact]
public static void InvalidInputString_Dash()
{
Assert.Throws<FormatException>(() => Convert.FromHexString("01-02-FD-FE-FF"));
string mixedCase1 = hex.Substring(0, hex.Length / 2).ToUpperInvariant() +
hex.Substring(hex.Length / 2).ToLowerInvariant();
string mixedCase2 = hex.Substring(0, hex.Length / 2).ToLowerInvariant() +
hex.Substring(hex.Length / 2).ToUpperInvariant();

TestSequence(data, mixedCase1);
TestSequence(data, mixedCase2);

Assert.Throws<FormatException>(() => Convert.FromHexString(hex + " "));
Assert.Throws<FormatException>(() => Convert.FromHexString("\uAAAA" + hex));
}
}

[Fact]
public static void ZeroLength()
public static void TooShortDestination()
{
Assert.Same(Array.Empty<byte>(), Convert.FromHexString(string.Empty));
const int destinationSize = 10;
Span<byte> destination = stackalloc byte[destinationSize];
byte[] data = Security.Cryptography.RandomNumberGenerator.GetBytes(destinationSize * 2 + 1);
string hex = Convert.ToHexString(data);

OperationStatus result = Convert.FromHexString(hex, destination, out int charsConsumed, out int bytesWritten);

Assert.Equal(OperationStatus.DestinationTooSmall, result);
Assert.Equal(destinationSize * 2, charsConsumed);
Assert.Equal(destinationSize, bytesWritten);
}

[Fact]
public static void ToHexFromHexRoundtrip()
public static void NeedMoreData_OrFormatException()
{
for (int i = 1; i < 50; i++)
{
byte[] data = System.Security.Cryptography.RandomNumberGenerator.GetBytes(i);
string hex = Convert.ToHexString(data);
Assert.Equal(data, Convert.FromHexString(hex.ToLowerInvariant()));
Assert.Equal(data, Convert.FromHexString(hex.ToUpperInvariant()));
string mixedCase1 = hex.Substring(0, hex.Length / 2).ToUpperInvariant() +
hex.Substring(hex.Length / 2).ToLowerInvariant();
string mixedCase2 = hex.Substring(0, hex.Length / 2).ToLowerInvariant() +
hex.Substring(hex.Length / 2).ToUpperInvariant();
Assert.Equal(data, Convert.FromHexString(mixedCase1));
Assert.Equal(data, Convert.FromHexString(mixedCase2));
Assert.Throws<FormatException>(() => Convert.FromHexString(hex + " "));
Assert.Throws<FormatException>(() => Convert.FromHexString("\uAAAA" + hex));
}
const int destinationSize = 10;
byte[] data = Security.Cryptography.RandomNumberGenerator.GetBytes(destinationSize);
Span<byte> destination = stackalloc byte[destinationSize];
var hex = Convert.ToHexString(data);

var spanHex = hex.AsSpan(0, 1);
var singeResult = Convert.FromHexString(spanHex, destination, out int consumed, out int written);

Assert.Throws<FormatException>(() => Convert.FromHexString(hex.Substring(0, 1)));
Assert.Equal(OperationStatus.NeedMoreData, singeResult);
Assert.Equal(0, consumed);
Assert.Equal(0, written);

spanHex = hex.AsSpan(0, hex.Length - 1);

var oneOffResult = Convert.FromHexString(spanHex, destination, out consumed, out written);

Assert.Throws<FormatException>(() => Convert.FromHexString(hex.Substring(0, hex.Length - 1)));
Assert.Equal(OperationStatus.NeedMoreData, oneOffResult);
Assert.Equal(spanHex.Length - 1, consumed);
Assert.Equal((spanHex.Length - 1) / 2, written);
}
}
}
3 changes: 3 additions & 0 deletions src/libraries/System.Runtime/ref/System.Runtime.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1124,7 +1124,9 @@ public static partial class Convert
public static byte[] FromBase64CharArray(char[] inArray, int offset, int length) { throw null; }
public static byte[] FromBase64String(string s) { throw null; }
public static byte[] FromHexString(System.ReadOnlySpan<char> chars) { throw null; }
public static System.Buffers.OperationStatus FromHexString(System.ReadOnlySpan<char> source, Span<byte> destination, out int charsConsumed, out int bytesWritten) { throw null; }
public static byte[] FromHexString(string s) { throw null; }
public static System.Buffers.OperationStatus FromHexString(string source, Span<byte> destination, out int charsConsumed, out int bytesWritten) { throw null; }
public static System.TypeCode GetTypeCode(object? value) { throw null; }
public static bool IsDBNull([System.Diagnostics.CodeAnalysis.NotNullWhenAttribute(true)] object? value) { throw null; }
public static int ToBase64CharArray(byte[] inArray, int offsetIn, int length, char[] outArray, int offsetOut) { throw null; }
Expand Down Expand Up @@ -1270,6 +1272,7 @@ public static partial class Convert
public static string ToHexString(byte[] inArray) { throw null; }
public static string ToHexString(byte[] inArray, int offset, int length) { throw null; }
public static string ToHexString(System.ReadOnlySpan<byte> bytes) { throw null; }
public static bool TryToHexString(System.ReadOnlySpan<byte> source, System.Span<char> destination, out int charsWritten) { throw null; }
public static short ToInt16(bool value) { throw null; }
public static short ToInt16(byte value) { throw null; }
public static short ToInt16(char value) { throw null; }
Expand Down