Skip to content

Commit 62edf08

Browse files
ArkatufusAaronontheweb
Arkatufus
and
Aaronontheweb
authored
Implement Sharding.Delivery bypass feature (#7106)
* Implement Sharding.Delivery bypass feature * Update API approval list * Update ShardingConsumerControllerImpl.cs Changed `.Tell` to `.Forward` --------- Co-authored-by: Aaron Stannard <[email protected]>
1 parent 87bd24a commit 62edf08

File tree

8 files changed

+471
-4
lines changed

8 files changed

+471
-4
lines changed

‎src/contrib/cluster/Akka.Cluster.Sharding.Tests.MultiNode/Delivery/ClusterShardingDeliveryGracefulShutdownSpec.cs

+292
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
//-----------------------------------------------------------------------
2+
// <copyright file="ClusterShardingGracefulShutdownOldestSpec.cs" company="Akka.NET Project">
3+
// Copyright (C) 2009-2023 Lightbend Inc. <http://www.lightbend.com>
4+
// Copyright (C) 2013-2023 .NET Foundation <https://github.com/akkadotnet/akka.net>
5+
// </copyright>
6+
//-----------------------------------------------------------------------
7+
8+
using System;
9+
using System.Collections.Immutable;
10+
using System.Linq;
11+
using Akka.Actor;
12+
using Akka.Cluster.Sharding.Delivery;
13+
using Akka.Configuration;
14+
using Akka.MultiNode.TestAdapter;
15+
using Akka.Remote.TestKit;
16+
using Akka.Util;
17+
using FluentAssertions;
18+
19+
namespace Akka.Cluster.Sharding.Tests.MultiNode.Delivery
20+
{
21+
public class ClusterShardingDeliveryGracefulShutdownSpecConfig : MultiNodeClusterShardingConfig
22+
{
23+
public RoleName First { get; }
24+
public RoleName Second { get; }
25+
26+
public ClusterShardingDeliveryGracefulShutdownSpecConfig(StateStoreMode mode)
27+
: base(mode: mode, loglevel: "DEBUG", additionalConfig: @"
28+
# don't leak ddata state across runs
29+
akka.cluster.sharding.distributed-data.durable.keys = []
30+
akka.reliable-delivery.sharding.consumer-controller.allow-bypass = true
31+
")
32+
{
33+
First = Role("first");
34+
Second = Role("second");
35+
}
36+
}
37+
38+
public class PersistentClusterShardingDeliveryGracefulShutdownSpecConfig : ClusterShardingDeliveryGracefulShutdownSpecConfig
39+
{
40+
public PersistentClusterShardingDeliveryGracefulShutdownSpecConfig()
41+
: base(StateStoreMode.Persistence)
42+
{
43+
}
44+
}
45+
46+
public class DDataClusterShardingDeliveryGracefulShutdownSpecConfig : ClusterShardingDeliveryGracefulShutdownSpecConfig
47+
{
48+
public DDataClusterShardingDeliveryGracefulShutdownSpecConfig()
49+
: base(StateStoreMode.DData)
50+
{
51+
}
52+
}
53+
54+
public class PersistentClusterShardingDeliveryGracefulShutdownSpec : ClusterShardingDeliveryGracefulShutdownSpec
55+
{
56+
public PersistentClusterShardingDeliveryGracefulShutdownSpec()
57+
: base(new PersistentClusterShardingDeliveryGracefulShutdownSpecConfig(), typeof(PersistentClusterShardingDeliveryGracefulShutdownSpec))
58+
{
59+
}
60+
}
61+
62+
public class DDataClusterShardingDeliveryGracefulShutdownSpec : ClusterShardingDeliveryGracefulShutdownSpec
63+
{
64+
public DDataClusterShardingDeliveryGracefulShutdownSpec()
65+
: base(new DDataClusterShardingDeliveryGracefulShutdownSpecConfig(), typeof(DDataClusterShardingDeliveryGracefulShutdownSpec))
66+
{
67+
}
68+
}
69+
70+
public abstract class ClusterShardingDeliveryGracefulShutdownSpec : MultiNodeClusterShardingSpec<ClusterShardingDeliveryGracefulShutdownSpecConfig>
71+
{
72+
#region setup
73+
74+
public class TerminationOrderActor : ActorBase
75+
{
76+
public class RegionTerminated
77+
{
78+
public static RegionTerminated Instance = new();
79+
80+
private RegionTerminated()
81+
{
82+
}
83+
}
84+
85+
public class CoordinatorTerminated
86+
{
87+
public static CoordinatorTerminated Instance = new();
88+
89+
private CoordinatorTerminated()
90+
{
91+
}
92+
}
93+
94+
public static Props Props(IActorRef probe, IActorRef coordinator, IActorRef region)
95+
{
96+
return Actor.Props.Create(() => new TerminationOrderActor(probe, coordinator, region));
97+
}
98+
99+
private readonly IActorRef _probe;
100+
private readonly IActorRef _coordinator;
101+
private readonly IActorRef _region;
102+
103+
public TerminationOrderActor(IActorRef probe, IActorRef coordinator, IActorRef region)
104+
{
105+
_probe = probe;
106+
_coordinator = coordinator;
107+
_region = region;
108+
109+
Context.Watch(coordinator);
110+
Context.Watch(region);
111+
}
112+
113+
protected override bool Receive(object message)
114+
{
115+
switch (message)
116+
{
117+
case Terminated t when t.ActorRef.Equals(_coordinator):
118+
_probe.Tell(CoordinatorTerminated.Instance);
119+
return true;
120+
121+
case Terminated t when t.ActorRef.Equals(_region):
122+
_probe.Tell(RegionTerminated.Instance);
123+
return true;
124+
}
125+
return false;
126+
}
127+
}
128+
129+
private sealed class MessageExtractor: IMessageExtractor
130+
{
131+
public string EntityId(object message)
132+
=> message switch
133+
{
134+
SlowStopConsumerEntity.Job j => j.Payload.ToString(),
135+
_ => null
136+
};
137+
138+
public object EntityMessage(object message)
139+
=> message;
140+
141+
public string ShardId(object message)
142+
=> message switch
143+
{
144+
SlowStopConsumerEntity.Job j => j.Payload.ToString(),
145+
_ => null
146+
};
147+
148+
public string ShardId(string entityId, object messageHint = null)
149+
=> entityId;
150+
}
151+
152+
private const string TypeName = "SlowStopEntity";
153+
private IActorRef _producer;
154+
private IActorRef _producerController;
155+
156+
protected ClusterShardingDeliveryGracefulShutdownSpec(ClusterShardingDeliveryGracefulShutdownSpecConfig config, Type type)
157+
: base(config, type)
158+
{
159+
}
160+
161+
private IActorRef CreateProducer(string producerId)
162+
{
163+
_producerController =
164+
Sys.ActorOf(
165+
ShardingProducerController.Create<SlowStopConsumerEntity.Job>(
166+
producerId: producerId,
167+
shardRegion: ClusterSharding.Get(Sys).ShardRegion(TypeName),
168+
durableQueue: Option<Props>.None,
169+
settings: ShardingProducerController.Settings.Create(Sys)),
170+
"shardingProducerController");
171+
_producer = Sys.ActorOf(Props.Create(() => new TestShardingProducer(_producerController, TestActor)),
172+
"producer");
173+
return _producer;
174+
}
175+
176+
private IActorRef StartSharding()
177+
{
178+
return ClusterSharding.Get(Sys).Start(
179+
typeName: TypeName,
180+
entityPropsFactory: e => ShardingConsumerController.Create<SlowStopConsumerEntity.Job>(
181+
c => Props.Create(() => new SlowStopConsumerEntity(e, c)),
182+
ShardingConsumerController.Settings.Create(Sys)),
183+
settings: Settings.Value.WithRole(null),
184+
messageExtractor: new MessageExtractor(),
185+
allocationStrategy: ShardAllocationStrategy.LeastShardAllocationStrategy(absoluteLimit: 2, relativeLimit: 1.0),
186+
handOffStopMessage: SlowStopConsumerEntity.Stop.Instance);
187+
}
188+
189+
#endregion
190+
191+
[MultiNodeFact]
192+
public void ClusterShardingGracefulShutdownSpecs()
193+
{
194+
Cluster_sharding_must_join_cluster();
195+
Cluster_sharding_must_start_some_shards_in_both_regions();
196+
Cluster_sharding_must_gracefully_shutdown_the_oldest_region();
197+
}
198+
199+
private void Cluster_sharding_must_join_cluster()
200+
{
201+
StartPersistenceIfNeeded(startOn: Config.First, Config.First, Config.Second);
202+
203+
Join(Config.First, Config.First);
204+
Join(Config.Second, Config.First);
205+
206+
// make sure all nodes are up
207+
AwaitAssert(() =>
208+
{
209+
Cluster.Get(Sys).SendCurrentClusterState(TestActor);
210+
ExpectMsg<ClusterEvent.CurrentClusterState>().Members.Count.Should().Be(2);
211+
});
212+
213+
RunOn(() =>
214+
{
215+
StartSharding();
216+
}, Config.First);
217+
218+
RunOn(() =>
219+
{
220+
StartSharding();
221+
}, Config.Second);
222+
223+
EnterBarrier("sharding started");
224+
}
225+
226+
private void Cluster_sharding_must_start_some_shards_in_both_regions()
227+
{
228+
RunOn(() =>
229+
{
230+
var producer = CreateProducer("p-1");
231+
Within(TimeSpan.FromSeconds(30), () =>
232+
{
233+
var regionAddresses = Enumerable.Range(1, 20).Select(n =>
234+
{
235+
producer.Tell(n, TestActor);
236+
ExpectMsg(n, TimeSpan.FromSeconds(1));
237+
return LastSender.Path.Address;
238+
}).ToImmutableHashSet();
239+
240+
regionAddresses.Count.Should().Be(2);
241+
});
242+
}, Config.First);
243+
244+
EnterBarrier("after-2");
245+
}
246+
247+
private void Cluster_sharding_must_gracefully_shutdown_the_oldest_region()
248+
{
249+
Within(TimeSpan.FromSeconds(30), () =>
250+
{
251+
RunOn(() =>
252+
{
253+
IActorRef coordinator = null;
254+
AwaitAssert(() =>
255+
{
256+
coordinator = Sys
257+
.ActorSelection($"/system/sharding/{TypeName}Coordinator/singleton/coordinator")
258+
.ResolveOne(RemainingOrDefault).Result;
259+
});
260+
var terminationProbe = CreateTestProbe();
261+
var region = ClusterSharding.Get(Sys).ShardRegion(TypeName);
262+
Sys.ActorOf(TerminationOrderActor.Props(terminationProbe.Ref, coordinator, region));
263+
264+
// trigger graceful shutdown
265+
Cluster.Leave(GetAddress(Config.First));
266+
267+
// region first
268+
terminationProbe.ExpectMsg<TerminationOrderActor.RegionTerminated>();
269+
terminationProbe.ExpectMsg<TerminationOrderActor.CoordinatorTerminated>();
270+
}, Config.First);
271+
272+
EnterBarrier("terminated");
273+
274+
RunOn(() =>
275+
{
276+
var producer = CreateProducer("p-2");
277+
AwaitAssert(() =>
278+
{
279+
var responses = Enumerable.Range(1, 20).Select(n =>
280+
{
281+
producer.Tell(n, TestActor);
282+
return ExpectMsg(n, TimeSpan.FromSeconds(1));
283+
}).ToImmutableHashSet();
284+
285+
responses.Count.Should().Be(20);
286+
});
287+
}, Config.Second);
288+
EnterBarrier("done-o");
289+
});
290+
}
291+
}
292+
}

‎src/contrib/cluster/Akka.Cluster.Sharding.Tests.MultiNode/Delivery/SlowStopConsumerEntity.cs

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// -----------------------------------------------------------------------
2+
// <copyright file="TestConsumer.cs" company="Akka.NET Project">
3+
// Copyright (C) 2009-2023 Lightbend Inc. <http://www.lightbend.com>
4+
// Copyright (C) 2013-2023 .NET Foundation <https://github.com/akkadotnet/akka.net>
5+
// </copyright>
6+
// -----------------------------------------------------------------------
7+
#nullable enable
8+
using System;
9+
using Akka.Actor;
10+
using Akka.Delivery;
11+
using Akka.Event;
12+
13+
namespace Akka.Cluster.Sharding.Tests.MultiNode.Delivery;
14+
15+
/// <summary>
16+
/// INTERNAL API
17+
/// </summary>
18+
public sealed class SlowStopConsumerEntity : ReceiveActor, IWithTimers
19+
{
20+
private readonly IActorRef _consumerController;
21+
22+
public SlowStopConsumerEntity(string persistenceId, IActorRef consumerController)
23+
{
24+
_consumerController = consumerController;
25+
26+
Receive<ConsumerController.Delivery<Job>>(delivery =>
27+
{
28+
var job = delivery.Message;
29+
job.Probe.Tell(job.Payload);
30+
delivery.ConfirmTo.Tell(ConsumerController.Confirmed.Instance);
31+
});
32+
33+
Receive<Stop>(_ =>
34+
{
35+
Timers.StartSingleTimer(ActualStop.Instance, ActualStop.Instance, TimeSpan.FromMilliseconds(50));
36+
});
37+
38+
Receive<ActualStop>(_ => Context.Stop(Self));
39+
}
40+
41+
protected override void PreStart()
42+
{
43+
_consumerController.Tell(new ConsumerController.Start<Job>(Self));
44+
}
45+
46+
public sealed class Stop: ConsumerController.IConsumerCommand<Job>
47+
{
48+
public static readonly Stop Instance = new();
49+
private Stop() { }
50+
}
51+
52+
public sealed class ActualStop
53+
{
54+
public static readonly ActualStop Instance = new();
55+
private ActualStop() { }
56+
}
57+
58+
public sealed record Job(int Payload, IActorRef Probe);
59+
60+
public ITimerScheduler Timers { get; set; } = null!;
61+
}

0 commit comments

Comments
 (0)