Track if remote listing completed succesully when validating incremental deploy plan (#1817)

* Track if remote listing completed succesully when validating incremental deploy plan

* Ensure we can incrementally deploy to empty remote s3 bucket

Author: Mpdreamz

src/tooling/docs-assembler/Cli/DeployCommands.cs

@@ -40,6 +40,7 @@ private void AssignOutputLogger()
 	/// <param name="out"> The file to write the plan to</param>
 	/// <param name="deleteThreshold"> The percentage of deletions allowed in the plan as percentage of total files to sync</param>
 	/// <param name="ctx"></param>
+	[Command("plan")]
 	public async Task<int> Plan(
 		string environment,
 		string s3BucketName,
@@ -58,7 +59,7 @@ public async Task<int> Plan(
 		var s3Client = new AmazonS3Client();
 		var planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext);
 		var plan = await planner.Plan(deleteThreshold, ctx);
-		_logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests);
+		_logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted);
 		_logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count);
 		_logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count);
 		_logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count);
@@ -70,7 +71,7 @@ public async Task<int> Plan(
 		if (!validationResult.Valid)
 		{
 			await githubActionsService.SetOutputAsync("plan-valid", "false");
-			collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions");
+			collector.EmitError(@out, $"Plan is invalid, {validationResult}, delete ratio: {validationResult.DeleteRatio}, remote listing completed: {plan.RemoteListingCompleted}");
 			await collector.StopAsync(ctx);
 			return collector.Errors;
 		}
@@ -93,6 +94,7 @@ public async Task<int> Plan(
 	/// <param name="s3BucketName">The S3 bucket name to deploy to</param>
 	/// <param name="planFile">The path to the plan file to apply</param>
 	/// <param name="ctx"></param>
+	[Command("apply")]
 	public async Task<int> Apply(string environment, string s3BucketName, string planFile, Cancel ctx = default)
 	{
 		AssignOutputLogger();
@@ -116,6 +118,7 @@ public async Task<int> Apply(string environment, string s3BucketName, string pla
 		}
 		var planJson = await File.ReadAllTextAsync(planFile, ctx);
 		var plan = SyncPlan.Deserialize(planJson);
+		_logger.LogInformation("Remote listing completed: {RemoteListingCompleted}", plan.RemoteListingCompleted);
 		_logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests);
 		_logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count);
 		_logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count);
@@ -133,7 +136,7 @@ public async Task<int> Apply(string environment, string s3BucketName, string pla
 		var validationResult = validator.Validate(plan);
 		if (!validationResult.Valid)
 		{
-			collector.EmitError(planFile, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions");
+			collector.EmitError(planFile, $"Plan is invalid, {validationResult}, delete ratio: {validationResult.DeleteRatio}, remote listing completed: {plan.RemoteListingCompleted}");
 			await collector.StopAsync(ctx);
 			return collector.Errors;
 		}

src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs

@@ -92,7 +92,7 @@ private bool IsSymlink(string path)
 
 	public async Task<SyncPlan> Plan(float? deleteThreshold, Cancel ctx = default)
 	{
-		var remoteObjects = await ListObjects(ctx);
+		var (readToCompletion, remoteObjects) = await ListObjects(ctx);
 		var localObjects = context.OutputDirectory.GetFiles("*", SearchOption.AllDirectories)
 			.Where(f => !IsSymlink(f.FullName))
 			.ToArray();
@@ -156,6 +156,7 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
 
 		return new SyncPlan
 		{
+			RemoteListingCompleted = readToCompletion,
 			DeleteThresholdDefault = deleteThreshold,
 			TotalRemoteFiles = remoteObjects.Count,
 			TotalSourceFiles = localObjects.Length,
@@ -167,24 +168,56 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
 		};
 	}
 
-	private async Task<Dictionary<string, S3Object>> ListObjects(Cancel ctx = default)
+	private async Task<(bool readToCompletion, Dictionary<string, S3Object> objects)> ListObjects(Cancel ctx = default)
 	{
 		var listBucketRequest = new ListObjectsV2Request
 		{
 			BucketName = bucketName,
-			MaxKeys = 1000,
+			MaxKeys = 1000
 		};
 		var objects = new List<S3Object>();
+		var bucketExists = await S3BucketExists(ctx);
+		if (!bucketExists)
+		{
+			context.Collector.EmitGlobalError("Bucket does not exist, cannot list objects");
+			return (false, objects.ToDictionary(o => o.Key));
+		}
+
+		var readToCompletion = true;
 		ListObjectsV2Response response;
 		do
 		{
 			response = await s3Client.ListObjectsV2Async(listBucketRequest, ctx);
 			if (response is null or { S3Objects: null })
+			{
+				if (response?.IsTruncated == true)
+				{
+					context.Collector.EmitGlobalError("Failed to list objects in S3 to completion");
+					readToCompletion = false;
+				}
 				break;
+			}
 			objects.AddRange(response.S3Objects);
 			listBucketRequest.ContinuationToken = response.NextContinuationToken;
 		} while (response.IsTruncated == true);
 
-		return objects.ToDictionary(o => o.Key);
+		return (readToCompletion, objects.ToDictionary(o => o.Key));
+	}
+
+	private async Task<bool> S3BucketExists(Cancel ctx)
+	{
+		//https://docs.aws.amazon.com/code-library/latest/ug/s3_example_s3_Scenario_DoesBucketExist_section.html
+		try
+		{
+			_ = await s3Client.GetBucketAclAsync(new GetBucketAclRequest
+			{
+				BucketName = bucketName
+			}, ctx);
+			return true;
+		}
+		catch
+		{
+			return false;
+		}
 	}
 }

src/tooling/docs-assembler/Deploying/DocsSync.cs

@@ -55,6 +55,10 @@ public record SyncPlan
 	[JsonPropertyName("deletion_threshold_default")]
 	public required float? DeleteThresholdDefault { get; init; }
 
+	/// The user-specified delete threshold
+	[JsonPropertyName("remote_listing_completed")]
+	public required bool RemoteListingCompleted { get; init; }
+
 	/// The total number of source files that were located in the build output
 	[JsonPropertyName("total_source_files")]
 	public required int TotalSourceFiles { get; init; }

src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs

@@ -16,6 +16,12 @@ public PlanValidationResult Validate(SyncPlan plan)
 			_logger.LogInformation("Using user-specified delete threshold of {Threshold}", plan.DeleteThresholdDefault);
 
 		var deleteThreshold = plan.DeleteThresholdDefault ?? 0.2f;
+		if (!plan.RemoteListingCompleted)
+		{
+			_logger.LogError("Remote files were not read to completion, cannot validate deployment plan");
+			return new(false, 1.0f, deleteThreshold);
+		}
+
 		if (plan.TotalSourceFiles == 0)
 		{
 			_logger.LogError("No files to sync");
@@ -30,7 +36,7 @@ public PlanValidationResult Validate(SyncPlan plan)
 		}
 		// if the total remote files are less than or equal to 100, we enforce a higher ratio of 0.8
 		// this allows newer assembled documentation to be in a higher state of flux
-		if (plan.TotalRemoteFiles <= 100)
+		else if (plan.TotalRemoteFiles <= 100)
 		{
 			_logger.LogInformation("Plan has less than 100 total remote files ensuring delete threshold is at minimum 0.8");
 			deleteThreshold = Math.Max(deleteThreshold, 0.8f);

tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs

@@ -235,6 +235,7 @@ public async Task TestApply()
 		var context = new AssembleContext(config, configurationContext, "dev", collector, fileSystem, fileSystem, null, checkoutDirectory);
 		var plan = new SyncPlan
 		{
+			RemoteListingCompleted = true,
 			DeleteThresholdDefault = null,
 			TotalRemoteFiles = 0,
 			TotalSourceFiles = 5,