Merge remote-tracking branch 'upstream/dev' into tm_spam_reduction

.github/workflows/ci-pipeline.yml

@@ -2322,9 +2322,9 @@ jobs:
 
   update-nix:
     name: Update Nix SHA
-    needs: deploy-tgs
+    needs: changelog-regen
     runs-on: ubuntu-latest
-    if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success'
+    if: (!(cancelled() || failure())) && needs.changelog-regen.result == 'success'
     steps:
       - name: Install Native Packages # Name checked in rerunFlakyTests.js
         run: |

build/Dockerfile

@@ -79,6 +79,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh
 
 VOLUME ["/config_data", "/tgs_logs", "/app/lib"]
 
-HEALTHCHECK CMD --curl --fail http://localhost:5000/health || exit
+HEALTHCHECK --start-interval=60s CMD curl --fail http://localhost:5000/health || exit
 
 ENTRYPOINT ["./tgs.sh"]

build/TestCommon.props

@@ -18,9 +18,9 @@
     <!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
     <PackageReference Include="Moq" Version="4.20.72" />
     <!-- Usage: MSTest execution -->
-    <PackageReference Include="MSTest.TestAdapter" Version="3.8.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.8.2" />
     <!-- Usage: MSTest asserts etc... -->
-    <PackageReference Include="MSTest.TestFramework" Version="3.8.0" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.8.2" />
   </ItemGroup>
 
 </Project>

build/Version.props

@@ -3,7 +3,7 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.14.0</TgsCoreVersion>
+    <TgsCoreVersion>6.14.1</TgsCoreVersion>
     <TgsConfigVersion>5.5.0</TgsConfigVersion>
     <TgsRestVersion>10.12.1</TgsRestVersion>
     <TgsGraphQLVersion>0.5.0</TgsGraphQLVersion>

src/Tgstation.Server.Api/Tgstation.Server.Api.csproj

@@ -28,7 +28,7 @@
     <!-- Usage: HTTP constants reference -->
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.3.0" />
     <!-- Usage: Decoding the 'nbf' property of JWTs -->
-    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.5.0" />
+    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.6.0" />
     <!-- Usage: Data model annotating -->
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
   </ItemGroup>

src/Tgstation.Server.Host/Components/Chat/ChatManager.cs

@@ -377,6 +377,7 @@ public void QueueWatchdogMessage(string message)
 		/// <inheritdoc />
 		public Func<string?, string, Action<bool>> QueueDeploymentMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,
@@ -407,6 +408,7 @@ public void QueueWatchdogMessage(string message)
 						{
 							var callback = await provider.SendUpdateMessage(
 								revisionInformation,
+								previousRevisionInformation,
 								engineVersion,
 								estimatedCompletionTime,
 								gitHubOwner,

src/Tgstation.Server.Host/Components/Chat/IChatManager.cs

@@ -61,6 +61,7 @@ public interface IChatManager : IComponentService, IAsyncDisposable
 		/// Send the message for a deployment to configured deployment channels.
 		/// </summary>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/> of the deployment.</param>
+		/// <param name="previousRevisionInformation">The optional <see cref="RevisionInformation"/> of the previous deployment.</param>
 		/// <param name="engineVersion">The <see cref="Api.Models.EngineVersion"/> of the deployment.</param>
 		/// <param name="estimatedCompletionTime">The optional <see cref="DateTimeOffset"/> the deployment is expected to be completed at.</param>
 		/// <param name="gitHubOwner">The repository GitHub owner, if any.</param>
@@ -69,6 +70,7 @@ public interface IChatManager : IComponentService, IAsyncDisposable
 		/// <returns>A <see cref="Func{T1, T2, TResult}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. Returns an <see cref="Action"/> to call to mark the deployment as active/inactive. Parameter: If the deployment is being activated or inactivated.</returns>
 		Func<string?, string, Action<bool>> QueueDeploymentMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			Api.Models.EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,

src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs

@@ -303,6 +303,7 @@ await ValueTaskExtensions.WhenAll(
 		/// <inheritdoc />
 		public override async ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,
@@ -316,7 +317,7 @@ await ValueTaskExtensions.WhenAll(
 
 			localCommitPushed |= revisionInformation.CommitSha == revisionInformation.OriginCommitSha;
 
-			var fields = BuildUpdateEmbedFields(revisionInformation, engineVersion, gitHubOwner, gitHubRepo, localCommitPushed);
+			var fields = BuildUpdateEmbedFields(revisionInformation, previousRevisionInformation, engineVersion, gitHubOwner, gitHubRepo, localCommitPushed);
 			Optional<IEmbedAuthor> author = new EmbedAuthor(assemblyInformationProvider.VersionPrefix)
 			{
 				Url = "https://github.com/tgstation/tgstation-server",
@@ -900,13 +901,15 @@ async ValueTask<IEnumerable<IChannel>> GetGuildChannels(IPartialGuild guild)
 		/// Create a <see cref="List{T}"/> of <see cref="IEmbedField"/>s for a discord update embed.
 		/// </summary>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/> of the deployment.</param>
+		/// <param name="previousRevisionInformation">The optional <see cref="RevisionInformation"/> of the previous deployment.</param>
 		/// <param name="engineVersion">The <see cref="EngineVersion"/> of the deployment.</param>
 		/// <param name="gitHubOwner">The repository GitHub owner, if any.</param>
 		/// <param name="gitHubRepo">The repository GitHub name, if any.</param>
 		/// <param name="localCommitPushed"><see langword="true"/> if the local deployment commit was pushed to the remote repository.</param>
 		/// <returns>A new <see cref="List{T}"/> of <see cref="IEmbedField"/>s to use.</returns>
 		List<IEmbedField> BuildUpdateEmbedFields(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			string? gitHubOwner,
 			string? gitHubRepo,
@@ -936,6 +939,31 @@ List<IEmbedField> BuildUpdateEmbedFields(
 			if (gitHubOwner == null || gitHubRepo == null)
 				return fields;
 
+			var previousTestMerges = (IEnumerable<RevInfoTestMerge>?)previousRevisionInformation?.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+			var currentTestMerges = (IEnumerable<RevInfoTestMerge>?)revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+
+			// determine what TMs were changed and how
+			var addedTestMerges = currentTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => !previousTestMerges
+					.Any(y => y.TestMerge.Number == x.Number))
+				.ToList();
+			var removedTestMerges = previousTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => !currentTestMerges
+					.Any(y => y.TestMerge.Number == x.Number))
+				.ToList();
+			var updatedTestMerges = currentTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => previousTestMerges
+					.Any(y => y.TestMerge.Number == x.Number && y.TestMerge.TargetCommitSha != x.TargetCommitSha))
+				.ToList();
+			var unchangedTestMerges = currentTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => previousTestMerges
+					.Any(y => y.TestMerge.Number == x.Number && y.TestMerge.TargetCommitSha == x.TargetCommitSha))
+				.ToList();
+
 			fields.Add(
 				new EmbedField(
 					"Local Commit",
@@ -952,13 +980,33 @@ List<IEmbedField> BuildUpdateEmbedFields(
 						: revisionOriginSha[..7],
 					true));
 
-			fields.AddRange((revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>())
-				.Select(x => x.TestMerge)
+			fields.AddRange(addedTestMerges
+				.Select(x => new EmbedField(
+					$"#{x.Number} (Added)",
+					$"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.TargetCommitSha![..7]}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.TargetCommitSha}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}",
+					false)));
+
+			fields.AddRange(updatedTestMerges
+				.Select(x => new EmbedField(
+					$"#{x.Number} (Updated)",
+					$"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.TargetCommitSha![..7]}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.TargetCommitSha}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}",
+					false)));
+
+			fields.AddRange(unchangedTestMerges
 				.Select(x => new EmbedField(
 					$"#{x.Number}",
 					$"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.TargetCommitSha![..7]}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.TargetCommitSha}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}",
 					false)));
 
+			if (removedTestMerges.Count != 0)
+				fields.Add(
+				new EmbedField(
+					"Removed:",
+					String.Join(
+						Environment.NewLine,
+						removedTestMerges
+							.Select(x => $"- #{x.Number} [{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_"))));
+
 			return fields;
 		}
 

src/Tgstation.Server.Host/Components/Chat/Providers/IProvider.cs

@@ -84,6 +84,7 @@ interface IProvider : IAsyncDisposable
 		/// Send the message for a deployment.
 		/// </summary>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/> of the deployment.</param>
+		/// <param name="previousRevisionInformation">The <see cref="RevisionInformation"/> of the previous deployment if any.</param>
 		/// <param name="engineVersion">The <see cref="Api.Models.EngineVersion"/> of the deployment.</param>
 		/// <param name="estimatedCompletionTime">The optional <see cref="DateTimeOffset"/> the deployment is expected to be completed at.</param>
 		/// <param name="gitHubOwner">The repository GitHub owner, if any.</param>
@@ -94,6 +95,7 @@ interface IProvider : IAsyncDisposable
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a <see cref="Func{T1, T2, TResult}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. Returns another callback which should be called to mark the deployment as active.</returns>
 		ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			Api.Models.EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,

src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs

@@ -17,6 +17,7 @@
 using Tgstation.Server.Host.Extensions;
 using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
+using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.System;
 using Tgstation.Server.Host.Utils;
 
@@ -217,6 +218,7 @@ await Task.Factory.StartNew(
 		/// <inheritdoc />
 		public override async ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,
@@ -228,6 +230,9 @@ await Task.Factory.StartNew(
 			ArgumentNullException.ThrowIfNull(revisionInformation);
 			ArgumentNullException.ThrowIfNull(engineVersion);
 
+			var previousTestMerges = (IEnumerable<RevInfoTestMerge>?)previousRevisionInformation?.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+			var currentTestMerges = (IEnumerable<RevInfoTestMerge>?)revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+
 			var commitInsert = revisionInformation.CommitSha![..7];
 			string remoteCommitInsert;
 			if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha)
@@ -238,21 +243,35 @@ await Task.Factory.StartNew(
 			else
 				remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha![..7]);
 
-			var testmergeInsert = (revisionInformation.ActiveTestMerges?.Count ?? 0) == 0
+			var testmergeInsert = !currentTestMerges.Any()
 				? String.Empty
 				: String.Format(
 					CultureInfo.InvariantCulture,
 					" (Test Merges: {0})",
 					String.Join(
 						", ",
-						revisionInformation
-							.ActiveTestMerges!
+						currentTestMerges
 							.Select(x => x.TestMerge)
 							.Select(x =>
 							{
-								var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.TargetCommitSha![..7]);
-								if (x.Comment != null)
-									result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment);
+								var status = string.Empty;
+								if (!previousTestMerges.Any(y => y.TestMerge.Number == x.Number))
+									status = "Added";
+								else if (previousTestMerges.Any(y => y.TestMerge.Number == x.Number && y.TestMerge.TargetCommitSha != x.TargetCommitSha))
+									status = "Updated";
+
+								var result = $"#{x.Number} at {x.TargetCommitSha![..7]}";
+
+								if (!string.IsNullOrEmpty(x.Comment))
+								{
+									if (!string.IsNullOrEmpty(status))
+										result += $" ({status} - {x.Comment})";
+									else
+										result += $" ({x.Comment})";
+								}
+								else if (!string.IsNullOrEmpty(status))
+									result += $" ({status})";
+
 								return result;
 							})));
 

src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs

@@ -199,6 +199,7 @@ public Task SetReconnectInterval(uint reconnectInterval, bool connectNow)
 		/// <inheritdoc />
 		public abstract ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			RevisionInformation revisionInformation,
+			RevisionInformation? previousRevisionInformation,
 			Api.Models.EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -336,13 +336,17 @@ await databaseContextFactory.UseContext(
 						}
 					});
 
-				var likelyPushedTestMergeCommit =
-					repositorySettings!.PushTestMergeCommits!.Value
-					&& repositorySettings.AccessToken != null
-					&& repositorySettings.AccessUser != null;
+				Models.CompileJob? oldCompileJob;
 				using (repo)
+				{
+					var likelyPushedTestMergeCommit =
+						repositorySettings!.PushTestMergeCommits!.Value
+						&& repositorySettings.AccessToken != null
+						&& repositorySettings.AccessUser != null;
+					oldCompileJob = await compileJobConsumer.LatestCompileJob();
 					compileJob = await Compile(
 						job,
+						oldCompileJob,
 						revInfo!,
 						dreamMakerSettings!,
 						ddSettings!,
@@ -352,8 +356,8 @@ await databaseContextFactory.UseContext(
 						averageSpan,
 						likelyPushedTestMergeCommit,
 						cancellationToken);
+				}
 
-				var activeCompileJob = await compileJobConsumer.LatestCompileJob();
 				try
 				{
 					await databaseContextFactory.UseContext(
@@ -402,7 +406,7 @@ await databaseContextFactory.UseContext(
 
 				var commentsTask = remoteDeploymentManager!.PostDeploymentComments(
 					compileJob,
-					activeCompileJob?.RevisionInformation,
+					oldCompileJob?.RevisionInformation,
 					repositorySettings,
 					repoOwner,
 					repoName,
@@ -479,6 +483,7 @@ await databaseContextFactory.UseContext(
 		/// Run the compile implementation.
 		/// </summary>
 		/// <param name="job">The currently running <see cref="Job"/>.</param>
+		/// <param name="oldCompileJob">The optional <see cref="CompileJob"/> of the previous deployment.</param>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/>.</param>
 		/// <param name="dreamMakerSettings">The <see cref="Api.Models.Internal.DreamMakerSettings"/>.</param>
 		/// <param name="launchParameters">The <see cref="DreamDaemonLaunchParameters"/>.</param>
@@ -491,6 +496,7 @@ await databaseContextFactory.UseContext(
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the completed <see cref="CompileJob"/>.</returns>
 		async ValueTask<Models.CompileJob> Compile(
 			Models.Job job,
+			Models.CompileJob? oldCompileJob,
 			Models.RevisionInformation revisionInformation,
 			Api.Models.Internal.DreamMakerSettings dreamMakerSettings,
 			DreamDaemonLaunchParameters launchParameters,
@@ -512,6 +518,7 @@ await databaseContextFactory.UseContext(
 				using var engineLock = await engineManager.UseExecutables(null, null, cancellationToken);
 				currentChatCallback = chatManager.QueueDeploymentMessage(
 					revisionInformation,
+					oldCompileJob?.RevisionInformation,
 					engineLock.Version,
 					DateTimeOffset.UtcNow + estimatedDuration,
 					repository.RemoteRepositoryOwner,

src/Tgstation.Server.Host/Components/Deployment/Remote/BaseRemoteDeploymentManager.cs

@@ -71,36 +71,28 @@ public async ValueTask PostDeploymentComments(
 			if (repositorySettings.AccessToken == null)
 				return;
 
-			var deployedRevisionInformation = compileJob.RevisionInformation;
-			if ((previousRevisionInformation != null && previousRevisionInformation.CommitSha == deployedRevisionInformation.CommitSha)
+			var revisionInformation = compileJob.RevisionInformation;
+			if ((previousRevisionInformation != null && previousRevisionInformation.CommitSha == revisionInformation.CommitSha)
 				|| !repositorySettings.PostTestMergeComment!.Value)
 				return;
 
-			previousRevisionInformation ??= new RevisionInformation();
-			previousRevisionInformation.ActiveTestMerges ??= new List<RevInfoTestMerge>();
+			var previousTestMerges = (IEnumerable<RevInfoTestMerge>?)previousRevisionInformation?.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+			var currentTestMerges = (IEnumerable<RevInfoTestMerge>?)revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
 
-			deployedRevisionInformation.ActiveTestMerges ??= new List<RevInfoTestMerge>();
-
-			// added prs
-			var addedTestMerges = deployedRevisionInformation
-				.ActiveTestMerges
+			// determine what TMs were changed and how
+			var addedTestMerges = currentTestMerges
 				.Select(x => x.TestMerge)
-				.Where(x => !previousRevisionInformation
-					.ActiveTestMerges
+				.Where(x => !previousTestMerges
 					.Any(y => y.TestMerge.Number == x.Number))
 				.ToList();
-			var removedTestMerges = previousRevisionInformation
-				.ActiveTestMerges
+			var removedTestMerges = previousTestMerges
 				.Select(x => x.TestMerge)
-				.Where(x => !deployedRevisionInformation
-					.ActiveTestMerges
+				.Where(x => !currentTestMerges
 					.Any(y => y.TestMerge.Number == x.Number))
 				.ToList();
-			var updatedTestMerges = deployedRevisionInformation
-				.ActiveTestMerges
+			var updatedTestMerges = currentTestMerges
 				.Select(x => x.TestMerge)
-				.Where(x => previousRevisionInformation
-					.ActiveTestMerges
+				.Where(x => previousTestMerges
 					.Any(y => y.TestMerge.Number == x.Number))
 				.ToList();
 

src/Tgstation.Server.Host/Tgstation.Server.Host.csproj

@@ -152,9 +152,9 @@
     <!-- Usage: File logging plugin -->
     <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
     <!-- Usage: OpenAPI spec generator -->
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="7.2.0" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="7.3.1" />
     <!-- Usage: Newtonsoft.Json plugin for OpenAPI spec generator -->
-    <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="7.2.0" />
+    <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="7.3.1" />
     <!-- Usage: Windows authentication plugin allowing searching for users by name -->
     <PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.2" />
     <!-- Usage: Identifying owning user of Windows Process objects -->