Skip to content

Commit 1edad6a

Browse files
ManickaPManickaP
authored
HTTP stress test improvements (#42313)
Fixes: stress client double read of content fixed fixed stress client hangs at start and stop leveraged HttpVersionPolicy increased pipeline timeout since we doubled the runs fixed base docker images to avoid missing IO.Pipelines Kestrel exception. Re-hauled tracing: added server file logging added log file rotation Minor renames. Contributes to: #42211 and #42198
1 parent 98fc7ed commit 1edad6a

14 files changed

+194
-168
lines changed

‎eng/docker/libraries-sdk-aspnetcore.linux.Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Builds and copies library artifacts into target dotnet sdk image
22
ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df28-20191023143754
3-
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/sdk:5.0-buster-slim
3+
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:5.0-buster-slim
44

55
FROM $BUILD_BASE_IMAGE as corefxbuild
66

‎eng/docker/libraries-sdk-aspnetcore.windows.Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# escape=`
22
# Simple Dockerfile which copies library build artifacts into target dotnet sdk image
3-
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/sdk:5.0-nanoserver-1809
3+
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:5.0-nanoserver-1809
44
FROM $SDK_BASE_IMAGE as target
55

66
ARG TESTHOST_LOCATION=".\\artifacts\\bin\\testhost"

‎eng/docker/libraries-sdk.linux.Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Builds and copies library artifacts into target dotnet sdk image
22
ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df28-20191023143754
3-
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/sdk:5.0-buster-slim
3+
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:5.0-buster-slim
44

55
FROM $BUILD_BASE_IMAGE as corefxbuild
66

‎eng/docker/libraries-sdk.windows.Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# escape=`
22
# Simple Dockerfile which copies clr and library build artifacts into target dotnet sdk image
3-
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/sdk:5.0-nanoserver-1809
3+
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:5.0-nanoserver-1809
44
FROM $SDK_BASE_IMAGE as target
55

66
ARG TESTHOST_LOCATION=".\\artifacts\\bin\\testhost"

‎eng/pipelines/libraries/stress/http.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ variables:
2525
jobs:
2626
- job: linux
2727
displayName: Docker Linux
28-
timeoutInMinutes: 120
28+
timeoutInMinutes: 150
2929
pool:
3030
name: NetCorePublic-Pool
3131
queue: BuildPool.Ubuntu.1604.Amd64.Open
@@ -65,7 +65,7 @@ jobs:
6565
6666
- job: windows
6767
displayName: Docker NanoServer
68-
timeoutInMinutes: 120
68+
timeoutInMinutes: 150
6969
pool:
7070
name: NetCorePublic-Pool
7171
queue: BuildPool.Server.Amd64.VS2019.Open

‎src/libraries/System.Net.Http/tests/StressTests/HttpStress/ClientOperations.cs

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -216,19 +216,19 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
216216
ctx.PopulateWithRandomHeaders(req.Headers);
217217
ulong expectedChecksum = CRC.CalculateHeaderCrc(req.Headers.Select(x => (x.Key, x.Value)));
218218

219-
using HttpResponseMessage res = await ctx.SendAsync(req);
219+
using HttpResponseMessage m = await ctx.SendAsync(req);
220220

221-
ValidateStatusCode(res);
221+
ValidateStatusCode(m);
222222

223-
await res.Content.ReadAsStringAsync();
223+
await m.Content.ReadAsStringAsync();
224224

225-
bool isValidChecksum = ValidateServerChecksum(res.Headers, expectedChecksum);
225+
bool isValidChecksum = ValidateServerChecksum(m.Headers, expectedChecksum);
226226
string failureDetails = isValidChecksum ? "server checksum matches client checksum" : "server checksum mismatch";
227227

228228
// Validate that request headers are being echoed
229229
foreach (KeyValuePair<string, IEnumerable<string>> reqHeader in req.Headers)
230230
{
231-
if (!res.Headers.TryGetValues(reqHeader.Key, out IEnumerable<string>? values))
231+
if (!m.Headers.TryGetValues(reqHeader.Key, out IEnumerable<string>? values))
232232
{
233233
throw new Exception($"Expected response header name {reqHeader.Key} missing. {failureDetails}");
234234
}
@@ -239,11 +239,11 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
239239
}
240240

241241
// Validate trailing headers are being echoed
242-
if (res.TrailingHeaders.Count() > 0)
242+
if (m.TrailingHeaders.Count() > 0)
243243
{
244244
foreach (KeyValuePair<string, IEnumerable<string>> reqHeader in req.Headers)
245245
{
246-
if (!res.TrailingHeaders.TryGetValues(reqHeader.Key + "-trailer", out IEnumerable<string>? values))
246+
if (!m.TrailingHeaders.TryGetValues(reqHeader.Key + "-trailer", out IEnumerable<string>? values))
247247
{
248248
throw new Exception($"Expected trailing header name {reqHeader.Key}-trailer missing. {failureDetails}");
249249
}
@@ -330,6 +330,7 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
330330
using HttpResponseMessage m = await ctx.SendAsync(req);
331331

332332
ValidateStatusCode(m);
333+
333334
string checksumMessage = ValidateServerChecksum(m.Headers, checksum) ? "server checksum matches client checksum" : "server checksum mismatch";
334335
ValidateContent(content, await m.Content.ReadAsStringAsync(), checksumMessage);
335336
}),
@@ -344,6 +345,7 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
344345
using HttpResponseMessage m = await ctx.SendAsync(req);
345346

346347
ValidateStatusCode(m);
348+
347349
string checksumMessage = ValidateServerChecksum(m.Headers, checksum) ? "server checksum matches client checksum" : "server checksum mismatch";
348350
ValidateContent(formData.expected, await m.Content.ReadAsStringAsync(), checksumMessage);
349351
}),
@@ -361,7 +363,7 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
361363
string response = await m.Content.ReadAsStringAsync();
362364

363365
string checksumMessage = ValidateServerChecksum(m.TrailingHeaders, checksum, required: false) ? "server checksum matches client checksum" : "server checksum mismatch";
364-
ValidateContent(content, await m.Content.ReadAsStringAsync(), checksumMessage);
366+
ValidateContent(content, response, checksumMessage);
365367
}),
366368

367369
("POST Duplex Slow",
@@ -414,6 +416,7 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
414416
using HttpResponseMessage m = await ctx.SendAsync(req, HttpCompletionOption.ResponseHeadersRead);
415417

416418
ValidateStatusCode(m);
419+
417420
string checksumMessage = ValidateServerChecksum(m.Headers, checksum) ? "server checksum matches client checksum" : "server checksum mismatch";
418421
ValidateContent(content, await m.Content.ReadAsStringAsync(), checksumMessage);
419422
}),
@@ -431,8 +434,8 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
431434
{
432435
throw new Exception($"Expected {expectedLength}, got {m.Content.Headers.ContentLength}");
433436
}
434-
string r = await m.Content.ReadAsStringAsync();
435-
if (r.Length > 0) throw new Exception($"Got unexpected response: {r}");
437+
string response = await m.Content.ReadAsStringAsync();
438+
if (response.Length > 0) throw new Exception($"Got unexpected response: {response}");
436439
}),
437440

438441
("PUT",
@@ -445,8 +448,8 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
445448

446449
ValidateStatusCode(m);
447450

448-
string r = await m.Content.ReadAsStringAsync();
449-
if (r != "") throw new Exception($"Got unexpected response: {r}");
451+
string response = await m.Content.ReadAsStringAsync();
452+
if (response != "") throw new Exception($"Got unexpected response: {response}");
450453
}),
451454

452455
("PUT Slow",
@@ -459,8 +462,8 @@ public static (string name, Func<RequestContext, Task> operation)[] Operations =
459462

460463
ValidateStatusCode(m);
461464

462-
string r = await m.Content.ReadAsStringAsync();
463-
if (r != "") throw new Exception($"Got unexpected response: {r}");
465+
string response = await m.Content.ReadAsStringAsync();
466+
if (response != "") throw new Exception($"Got unexpected response: {response}");
464467
}),
465468

466469
("GET Slow",

‎src/libraries/System.Net.Http/tests/StressTests/HttpStress/Configuration.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ public class Configuration
4343
public double CancellationProbability { get; set; }
4444

4545
public bool UseHttpSys { get; set; }
46-
public string? LogPath { get; set; }
4746
public bool LogAspNet { get; set; }
47+
public bool Trace { get; set; }
4848
public int? ServerMaxConcurrentStreams { get; set; }
4949
public int? ServerMaxFrameSize { get; set; }
5050
public int? ServerInitialConnectionWindowSize { get; set; }

‎src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/sdk:5.0-buster-slim
1+
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:5.0-buster-slim
22
FROM $SDK_BASE_IMAGE
33

44
RUN echo "DOTNET_SDK_VERSION="$DOTNET_SDK_VERSION

‎src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpEventListener.cs

Lines changed: 61 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,37 @@
33

44
using System;
55
using System.Diagnostics.Tracing;
6+
using System.Threading.Channels;
67
using System.Text;
78
using System.IO;
9+
using System.Linq;
10+
using System.Threading;
11+
using System.Threading.Tasks;
812

913
namespace HttpStress
1014
{
1115
public sealed class LogHttpEventListener : EventListener
1216
{
13-
private readonly StreamWriter _log;
17+
private int _lastLogNumber = 0;
18+
private StreamWriter _log;
19+
private Channel<string> _messagesChannel = Channel.CreateUnbounded<string>();
20+
private Task _processMessages;
21+
private CancellationTokenSource _stopProcessing;
1422

15-
public LogHttpEventListener(string logPath)
23+
public LogHttpEventListener()
1624
{
17-
_log = new StreamWriter(logPath, true) { AutoFlush = true };
25+
foreach (var filename in Directory.GetFiles(".", "client*.log"))
26+
{
27+
try
28+
{
29+
File.Delete(filename);
30+
} catch {}
31+
}
32+
_log = new StreamWriter("client.log", false) { AutoFlush = true };
33+
34+
_messagesChannel = Channel.CreateUnbounded<string>();
35+
_processMessages = ProcessMessagesAsync();
36+
_stopProcessing = new CancellationTokenSource();
1837
}
1938

2039
protected override void OnEventSourceCreated(EventSource eventSource)
@@ -25,63 +44,63 @@ protected override void OnEventSourceCreated(EventSource eventSource)
2544
}
2645
}
2746

28-
protected override void OnEventWritten(EventWrittenEventArgs eventData)
47+
private async Task ProcessMessagesAsync()
2948
{
30-
lock (_log)
49+
await Task.Yield();
50+
51+
try
3152
{
32-
var sb = new StringBuilder().Append($"{eventData.TimeStamp:HH:mm:ss.fffffff}[{eventData.EventName}] ");
33-
for (int i = 0; i < eventData.Payload?.Count; i++)
53+
int i = 0;
54+
await foreach (string message in _messagesChannel.Reader.ReadAllAsync(_stopProcessing.Token))
3455
{
35-
if (i > 0)
56+
if ((++i % 10_000) == 0)
3657
{
37-
sb.Append(", ");
58+
RotateFiles();
3859
}
39-
sb.Append(eventData.PayloadNames?[i]).Append(": ").Append(eventData.Payload[i]);
60+
61+
_log.WriteLine(message);
4062
}
41-
_log.WriteLine(sb.ToString());
4263
}
43-
}
64+
catch (OperationCanceledException)
65+
{
66+
return;
67+
}
4468

45-
public override void Dispose()
46-
{
47-
_log.Dispose();
48-
base.Dispose();
69+
void RotateFiles()
70+
{
71+
// Rotate the log if it reaches 50 MB size.
72+
if (_log.BaseStream.Length > (50 << 20))
73+
{
74+
_log.Close();
75+
_log = new StreamWriter($"client_{++_lastLogNumber:000}.log", false) { AutoFlush = true };
76+
}
77+
}
4978
}
50-
}
51-
52-
public sealed class ConsoleHttpEventListener : EventListener
53-
{
54-
public ConsoleHttpEventListener()
55-
{ }
5679

57-
protected override void OnEventSourceCreated(EventSource eventSource)
80+
protected override async void OnEventWritten(EventWrittenEventArgs eventData)
5881
{
59-
if (eventSource.Name == "Private.InternalDiagnostics.System.Net.Http")
82+
var sb = new StringBuilder().Append($"{eventData.TimeStamp:HH:mm:ss.fffffff}[{eventData.EventName}] ");
83+
for (int i = 0; i < eventData.Payload?.Count; i++)
6084
{
61-
EnableEvents(eventSource, EventLevel.LogAlways);
85+
if (i > 0)
86+
{
87+
sb.Append(", ");
88+
}
89+
sb.Append(eventData.PayloadNames?[i]).Append(": ").Append(eventData.Payload[i]);
6290
}
91+
await _messagesChannel.Writer.WriteAsync(sb.ToString());
6392
}
6493

65-
protected override void OnEventWritten(EventWrittenEventArgs eventData)
94+
public override void Dispose()
6695
{
67-
lock (Console.Out)
96+
base.Dispose();
97+
98+
if (!_processMessages.Wait(TimeSpan.FromSeconds(30)))
6899
{
69-
Console.ForegroundColor = ConsoleColor.DarkYellow;
70-
Console.Write($"{eventData.TimeStamp:HH:mm:ss.fffffff}[{eventData.EventName}] ");
71-
Console.ResetColor();
72-
for (int i = 0; i < eventData.Payload?.Count; i++)
73-
{
74-
if (i > 0)
75-
{
76-
Console.Write(", ");
77-
}
78-
Console.ForegroundColor = ConsoleColor.DarkGray;
79-
Console.Write(eventData.PayloadNames?[i] + ": ");
80-
Console.ResetColor();
81-
Console.Write(eventData.Payload[i]);
82-
}
83-
Console.WriteLine();
100+
_stopProcessing.Cancel();
101+
_processMessages.Wait();
84102
}
103+
_log.Dispose();
85104
}
86105
}
87106
}

‎src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88
</PropertyGroup>
99

1010
<ItemGroup>
11+
<PackageReference Include="Serilog.AspNetCore" Version="3.4.0" />
12+
<PackageReference Include="Serilog.Extensions.Logging.File" Version="2.0.0" />
13+
<PackageReference Include="Serilog.Sinks.File" Version="4.1.0" />
1114
<PackageReference Include="System.CommandLine.Experimental" Version="0.3.0-alpha.19577.1" />
1215
<PackageReference Include="System.Net.Http.WinHttpHandler" Version="4.5.4" />
1316
</ItemGroup>

‎src/libraries/System.Net.Http/tests/StressTests/HttpStress/Program.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
/// </summary>
1919
public static class Program
2020
{
21-
2221
public enum ExitCode { Success = 0, StressError = 1, CliError = 2 };
2322

2423
public static async Task<int> Main(string[] args)
@@ -46,7 +45,7 @@ private static bool TryParseCli(string[] args, [NotNullWhen(true)] out Configura
4645
cmd.AddOption(new Option("-connectionLifetime", "Max connection lifetime length (milliseconds).") { Argument = new Argument<int?>("connectionLifetime", null) });
4746
cmd.AddOption(new Option("-ops", "Indices of the operations to use") { Argument = new Argument<int[]?>("space-delimited indices", null) });
4847
cmd.AddOption(new Option("-xops", "Indices of the operations to exclude") { Argument = new Argument<int[]?>("space-delimited indices", null) });
49-
cmd.AddOption(new Option("-trace", "Enable System.Net.Http.InternalDiagnostics tracing.") { Argument = new Argument<string>("\"console\" or path") });
48+
cmd.AddOption(new Option("-trace", "Enable System.Net.Http.InternalDiagnostics (client) and/or ASP.NET dignostics (server) tracing.") { Argument = new Argument<bool>("enable", false) });
5049
cmd.AddOption(new Option("-aspnetlog", "Enable ASP.NET warning and error logging.") { Argument = new Argument<bool>("enable", false) });
5150
cmd.AddOption(new Option("-listOps", "List available options.") { Argument = new Argument<bool>("enable", false) });
5251
cmd.AddOption(new Option("-seed", "Seed for generating pseudo-random parameters for a given -n argument.") { Argument = new Argument<int?>("seed", null) });
@@ -99,7 +98,7 @@ private static bool TryParseCli(string[] args, [NotNullWhen(true)] out Configura
9998

10099
UseHttpSys = cmdline.ValueForOption<bool>("-httpSys"),
101100
LogAspNet = cmdline.ValueForOption<bool>("-aspnetlog"),
102-
LogPath = cmdline.HasOption("-trace") ? cmdline.ValueForOption<string>("-trace") : null,
101+
Trace = cmdline.ValueForOption<bool>("-trace"),
103102
ServerMaxConcurrentStreams = cmdline.ValueForOption<int?>("-serverMaxConcurrentStreams"),
104103
ServerMaxFrameSize = cmdline.ValueForOption<int?>("-serverMaxFrameSize"),
105104
ServerInitialConnectionWindowSize = cmdline.ValueForOption<int?>("-serverInitialConnectionWindowSize"),
@@ -158,11 +157,12 @@ private static async Task<ExitCode> Run(Configuration config)
158157
Console.WriteLine(" System.Net.Http: " + GetAssemblyInfo(typeof(System.Net.Http.HttpClient).Assembly));
159158
Console.WriteLine(" Server: " + (config.UseHttpSys ? "http.sys" : "Kestrel"));
160159
Console.WriteLine(" Server URL: " + config.ServerUri);
161-
Console.WriteLine(" Tracing: " + (config.LogPath == null ? (object)false : config.LogPath.Length == 0 ? (object)true : config.LogPath));
160+
Console.WriteLine(" Client Tracing: " + (config.Trace && config.RunMode.HasFlag(RunMode.client) ? "ON (client.log)" : "OFF"));
161+
Console.WriteLine(" Server Tracing: " + (config.Trace && config.RunMode.HasFlag(RunMode.server) ? "ON (server.log)" : "OFF"));
162162
Console.WriteLine(" ASP.NET Log: " + config.LogAspNet);
163163
Console.WriteLine(" Concurrency: " + config.ConcurrentRequests);
164164
Console.WriteLine(" Content Length: " + config.MaxContentLength);
165-
Console.WriteLine(" HTTP2 Version: " + config.HttpVersion);
165+
Console.WriteLine(" HTTP Version: " + config.HttpVersion);
166166
Console.WriteLine(" Lifetime: " + (config.ConnectionLifetime.HasValue ? $"{config.ConnectionLifetime.Value.TotalMilliseconds}ms" : "(infinite)"));
167167
Console.WriteLine(" Operations: " + string.Join(", ", usedClientOperations.Select(o => o.name)));
168168
Console.WriteLine(" Random Seed: " + config.RandomSeed);

0 commit comments

Comments
 (0)