Fix race condition in GrpcMonitor's GrpcChannel management (pulumi#304)

While creating integration tests I faced a race condition in GrpcMonitor
when the channel retrieved from the static _monitorChannels was null
although the key had already been set.
Replacing the complex implementation with custom locking by the standard
pattern ConcurrentDictionary<K, Lazy<V>> has resolved this.

Author: algompluecker

.changes/unreleased/Bug Fixes-304.yaml

@@ -0,0 +1,6 @@
+component: sdk
+kind: Bug Fixes
+body: Fix race condition in GrpcMonitor's GrpcChannel management
+time: 2024-07-21T08:17:42.5134291+02:00
+custom:
+    PR: "304"

sdk/Pulumi/Deployment/GrpcMonitor.cs

@@ -13,43 +13,32 @@ namespace Pulumi
     internal class GrpcMonitor : IMonitor
     {
         private readonly ResourceMonitor.ResourceMonitorClient _client;
+
         // Using a static dictionary to keep track of and re-use gRPC channels
         // According to the docs (https://docs.microsoft.com/en-us/aspnet/core/grpc/performance?view=aspnetcore-6.0#reuse-grpc-channels), creating GrpcChannels is expensive so we keep track of a bunch of them here
-        private static readonly ConcurrentDictionary<string, GrpcChannel> _monitorChannels = new ConcurrentDictionary<string, GrpcChannel>();
-        private static readonly object _channelsLock = new object();
+        private static readonly ConcurrentDictionary<string, Lazy<GrpcChannel>> _monitorChannels = new();
+
         public GrpcMonitor(string monitorAddress)
         {
-            // maxRpcMessageSize raises the gRPC Max Message size from `4194304` (4mb) to `419430400` (400mb)
+            var monitorChannel = _monitorChannels.GetOrAdd(monitorAddress, LazyCreateChannel);
+            this._client = new ResourceMonitor.ResourceMonitorClient(monitorChannel.Value);
+        }
+
+        private static Lazy<GrpcChannel> LazyCreateChannel(string monitorAddress)
+        {
+            return new Lazy<GrpcChannel>(() => CreateChannel(monitorAddress));
+        }
+
+        private static GrpcChannel CreateChannel(string monitorAddress)
+        {
             const int maxRpcMessageSize = 400 * 1024 * 1024;
-            if (_monitorChannels.TryGetValue(monitorAddress, out var monitorChannel))
+            var channel = GrpcChannel.ForAddress(new Uri($"http://{monitorAddress}"), new GrpcChannelOptions
             {
-                // A channel already exists for this address
-                this._client = new ResourceMonitor.ResourceMonitorClient(monitorChannel);
-            }
-            else
-            {
-                lock (_channelsLock)
-                {
-                    if (_monitorChannels.TryGetValue(monitorAddress, out var existingChannel))
-                    {
-                        // A channel already exists for this address
-                        this._client = new ResourceMonitor.ResourceMonitorClient(monitorChannel);
-                    }
-                    else
-                    {
-                        // Inititialize the monitor channel once for this monitor address
-                        var channel = GrpcChannel.ForAddress(new Uri($"http://{monitorAddress}"), new GrpcChannelOptions
-                        {
-                            MaxReceiveMessageSize = maxRpcMessageSize,
-                            MaxSendMessageSize = maxRpcMessageSize,
-                            Credentials = ChannelCredentials.Insecure
-                        });
-
-                        _monitorChannels[monitorAddress] = channel;
-                        this._client = new ResourceMonitor.ResourceMonitorClient(channel);
-                    }
-                }
-            }
+                MaxReceiveMessageSize = maxRpcMessageSize,
+                MaxSendMessageSize = maxRpcMessageSize,
+                Credentials = ChannelCredentials.Insecure
+            });
+            return channel;
         }
 
         public async Task<SupportsFeatureResponse> SupportsFeatureAsync(SupportsFeatureRequest request)