Skip to content

Commit baf6877

Browse files
cotticotti
authored
Real redirects support (#1397)
* Output redirects.json artifact * Output redirects_artifact_path during build-all * Add deploy update-redirects command to docs-assembler
1 parent a164826 commit baf6877

File tree

6 files changed

+148
-1
lines changed

6 files changed

+148
-1
lines changed

‎Directory.Packages.props

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
<PackageVersion Include="Amazon.Lambda.S3Events" Version="3.1.0" />
1414
<PackageVersion Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4" />
1515
<PackageVersion Include="Amazon.Lambda.SQSEvents" Version="2.2.0" />
16+
<PackageVersion Include="AWSSDK.CloudFront" Version="4.0.0.10" />
17+
<PackageVersion Include="AWSSDK.CloudFrontKeyValueStore" Version="4.0.0.9" />
1618
<PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
1719
<PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
1820
<PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
@@ -70,4 +72,4 @@
7072
</PackageVersion>
7173
<PackageVersion Include="xunit.v3" Version="2.0.2" />
7274
</ItemGroup>
73-
</Project>
75+
</Project>

‎src/Elastic.Documentation/Serialization/SourceGenerationContext.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
33
// See the LICENSE file in the project root for more information
44

5+
using System.Collections.Frozen;
56
using System.Text.Json.Serialization;
67
using Elastic.Documentation.Links;
78
using Elastic.Documentation.Search;
@@ -18,4 +19,5 @@ namespace Elastic.Documentation.Serialization;
1819
[JsonSerializable(typeof(LinkRegistry))]
1920
[JsonSerializable(typeof(LinkRegistryEntry))]
2021
[JsonSerializable(typeof(DocumentationDocument))]
22+
[JsonSerializable(typeof(Dictionary<string, string>))]
2123
public sealed partial class SourceGenerationContext : JsonSerializerContext;

‎src/tooling/docs-assembler/Building/AssemblerBuilder.cs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
// See the LICENSE file in the project root for more information
44

55
using System.Collections.Frozen;
6+
using System.Text.Json;
67
using Documentation.Assembler.Exporters;
78
using Documentation.Assembler.Navigation;
89
using Elastic.Documentation.Legacy;
910
using Elastic.Documentation.Links;
11+
using Elastic.Documentation.Serialization;
1012
using Elastic.Markdown;
1113
using Elastic.Markdown.Exporters;
1214
using Elastic.Markdown.Links.CrossLinks;
@@ -90,6 +92,9 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
9092
_ = await exporter.FinishExportAsync(context.OutputDirectory, ctx);
9193
}
9294

95+
await OutputRedirectsAsync(redirects
96+
.Where(r => !r.Key.TrimEnd('/').Equals(r.Value.TrimEnd('/'), StringComparison.OrdinalIgnoreCase))
97+
.ToDictionary(r => r.Key.TrimEnd('/'), r => r.Value), ctx);
9398

9499
tasks = markdownExporters.Select(async e => await e.StopAsync(ctx));
95100
await Task.WhenAll(tasks);
@@ -149,4 +154,16 @@ private void SetFeatureFlags(AssemblerDocumentationSet set)
149154
set.DocumentationSet.Configuration.Features.Set(configurationFeatureFlag.Key, configurationFeatureFlag.Value);
150155
}
151156
}
157+
158+
private async Task OutputRedirectsAsync(Dictionary<string, string> redirects, Cancel ctx)
159+
{
160+
var uniqueRedirects = redirects
161+
.Where(x => !x.Key.TrimEnd('/').Equals(x.Value.TrimEnd('/')))
162+
.ToDictionary();
163+
var redirectsFile = context.WriteFileSystem.FileInfo.New(Path.Combine(context.OutputDirectory.FullName, "redirects.json"));
164+
_logger.LogInformation("Writing {Count} resolved redirects to {Path}", uniqueRedirects.Count, redirectsFile.FullName);
165+
166+
var redirectsJson = JsonSerializer.Serialize(uniqueRedirects, SourceGenerationContext.Default.DictionaryStringString);
167+
await context.WriteFileSystem.File.WriteAllTextAsync(redirectsFile.FullName, redirectsJson, ctx);
168+
}
152169
}

‎src/tooling/docs-assembler/Cli/DeployCommands.cs

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,28 @@
44

55
using System.Diagnostics.CodeAnalysis;
66
using System.IO.Abstractions;
7+
using System.Text.Json;
78
using Actions.Core.Services;
9+
using Amazon.CloudFront;
10+
using Amazon.CloudFrontKeyValueStore;
11+
using Amazon.CloudFrontKeyValueStore.Model;
812
using Amazon.S3;
913
using Amazon.S3.Transfer;
1014
using ConsoleAppFramework;
1115
using Documentation.Assembler.Deploying;
16+
using Elastic.Documentation.Serialization;
1217
using Elastic.Documentation.Tooling.Diagnostics.Console;
18+
using Elastic.Documentation.Tooling.Filters;
1319
using Microsoft.Extensions.Logging;
1420

1521
namespace Documentation.Assembler.Cli;
1622

23+
internal enum KvsOperation
24+
{
25+
Puts,
26+
Deletes
27+
}
28+
1729
internal sealed class DeployCommands(ILoggerFactory logger, ICoreService githubActionsService)
1830
{
1931
[SuppressMessage("Usage", "CA2254:Template should be a static expression")]
@@ -94,4 +106,112 @@ public async Task<int> Apply(
94106
await collector.StopAsync(ctx);
95107
return collector.Errors;
96108
}
109+
110+
/// <summary>Refreshes the redirects mapping in Cloudfront's KeyValueStore</summary>
111+
/// <param name="environment">The environment to build</param>
112+
/// <param name="redirectsFile">Path to the redirects mapping pre-generated by docs-assembler</param>
113+
/// <param name="ctx"></param>
114+
[Command("update-redirects")]
115+
[ConsoleAppFilter<StopwatchFilter>]
116+
[ConsoleAppFilter<CatchExceptionFilter>]
117+
public async Task<int> UpdateRedirects(
118+
string environment,
119+
string redirectsFile = ".artifacts/assembly/redirects.json",
120+
Cancel ctx = default)
121+
{
122+
AssignOutputLogger();
123+
await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService)
124+
{
125+
NoHints = true
126+
}.StartAsync(ctx);
127+
128+
if (!File.Exists(redirectsFile))
129+
{
130+
collector.EmitError(redirectsFile, "Redirects mapping does not exist.");
131+
await collector.StopAsync(ctx);
132+
return collector.Errors;
133+
}
134+
135+
ConsoleApp.Log("Parsing redirects mapping");
136+
var jsonContent = await File.ReadAllTextAsync(redirectsFile, ctx);
137+
var sourcedRedirects = JsonSerializer.Deserialize(jsonContent, SourceGenerationContext.Default.DictionaryStringString);
138+
139+
if (sourcedRedirects is null)
140+
{
141+
collector.EmitError(redirectsFile, "Redirects mapping is invalid.");
142+
await collector.StopAsync(ctx);
143+
return collector.Errors;
144+
}
145+
146+
var kvsName = $"elastic-docs-v3-{environment}-redirects-kvs";
147+
148+
var cfClient = new AmazonCloudFrontClient();
149+
var kvsClient = new AmazonCloudFrontKeyValueStoreClient();
150+
151+
ConsoleApp.Log("Describing KVS");
152+
var describeResponse = await cfClient.DescribeKeyValueStoreAsync(new Amazon.CloudFront.Model.DescribeKeyValueStoreRequest { Name = kvsName }, ctx);
153+
154+
var kvsArn = describeResponse.KeyValueStore.ARN;
155+
var eTag = describeResponse.ETag;
156+
var existingRedirects = new HashSet<string>();
157+
158+
var listKeysRequest = new ListKeysRequest { KvsARN = kvsArn };
159+
ListKeysResponse listKeysResponse;
160+
161+
do
162+
{
163+
listKeysResponse = await kvsClient.ListKeysAsync(listKeysRequest, ctx);
164+
foreach (var item in listKeysResponse.Items)
165+
_ = existingRedirects.Add(item.Key);
166+
listKeysRequest.NextToken = listKeysResponse.NextToken;
167+
}
168+
while (!string.IsNullOrEmpty(listKeysResponse.NextToken));
169+
170+
var toPut = sourcedRedirects
171+
.Select(kvp => new PutKeyRequestListItem { Key = kvp.Key, Value = kvp.Value });
172+
var toDelete = existingRedirects
173+
.Except(sourcedRedirects.Keys)
174+
.Select(k => new DeleteKeyRequestListItem { Key = k });
175+
176+
ConsoleApp.Log("Updating redirects in KVS");
177+
const int batchSize = 50;
178+
179+
eTag = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toPut, batchSize, KvsOperation.Puts, ctx);
180+
_ = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toDelete, batchSize, KvsOperation.Deletes, ctx);
181+
182+
await collector.StopAsync(ctx);
183+
return collector.Errors;
184+
}
185+
186+
private static async Task<string> ProcessBatchUpdatesAsync(
187+
IAmazonCloudFrontKeyValueStore kvsClient,
188+
string kvsArn,
189+
string eTag,
190+
IEnumerable<object> items,
191+
int batchSize,
192+
KvsOperation operation,
193+
Cancel ctx)
194+
{
195+
var enumerable = items.ToList();
196+
for (var i = 0; i < enumerable.Count; i += batchSize)
197+
{
198+
var batch = enumerable.Skip(i).Take(batchSize);
199+
var updateRequest = new UpdateKeysRequest
200+
{
201+
KvsARN = kvsArn,
202+
IfMatch = eTag
203+
};
204+
205+
if (operation is KvsOperation.Puts)
206+
updateRequest.Puts = batch.Cast<PutKeyRequestListItem>().ToList();
207+
else if (operation is KvsOperation.Deletes)
208+
updateRequest.Deletes = batch.Cast<DeleteKeyRequestListItem>().ToList();
209+
210+
var update = await kvsClient.UpdateKeysAsync(updateRequest, ctx);
211+
eTag = update.ETag;
212+
}
213+
214+
return eTag;
215+
}
216+
97217
}

‎src/tooling/docs-assembler/Cli/RepositoryCommands.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,10 @@ public async Task<int> BuildAll(
136136

137137
await cloner.WriteLinkRegistrySnapshot(checkoutResult.LinkRegistrySnapshot, ctx);
138138

139+
var redirectsPath = Path.Combine(assembleContext.OutputDirectory.FullName, "redirects.json");
140+
if (File.Exists(redirectsPath))
141+
await githubActionsService.SetOutputAsync("redirects-artifact-path", redirectsPath);
142+
139143
var sitemapBuilder = new SitemapBuilder(navigation.NavigationItems, assembleContext.WriteFileSystem, assembleContext.OutputDirectory);
140144
sitemapBuilder.Generate();
141145

‎src/tooling/docs-assembler/docs-assembler.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
</PropertyGroup>
1818

1919
<ItemGroup>
20+
<PackageReference Include="AWSSDK.CloudFront" />
21+
<PackageReference Include="AWSSDK.CloudFrontKeyValueStore" />
2022
<PackageReference Include="AWSSDK.S3"/>
2123
<PackageReference Include="ConsoleAppFramework.Abstractions"/>
2224
<PackageReference Include="ConsoleAppFramework" />

0 commit comments

Comments
 (0)