Skip to content

Image caching fix for locally built images #1518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 12, 2024
Merged

Image caching fix for locally built images #1518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/BuildCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,7 @@ private async Task BuildImagesAsync()
_imageDigestCache,
_imageNameResolver.Value,
sourceRepoUrl: Options.SourceRepoUrl,
isLocalBaseImageExpected: true,
isDryRun: Options.IsDryRun);

if (cacheResult.State.HasFlag(ImageCacheState.Cached))
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateBuildMatrixCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,7 @@ await Parallel.ForEachAsync(subgraph, async (platformMapping, _) =>
_imageDigestCache,
_imageNameResolver.Value,
Options.SourceRepoUrl,
isLocalBaseImageExpected: false,
Options.IsDryRun);

bool includePlatformInMatrix = !cacheResult.State.HasFlag(ImageCacheState.Cached);
Expand Down
29 changes: 23 additions & 6 deletions src/Microsoft.DotNet.ImageBuilder/src/ImageCacheService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ Task<ImageCacheResult> CheckForCachedImageAsync(
ImageDigestCache imageDigestCache,
ImageNameResolver imageNameResolver,
string? sourceRepoUrl,
bool isLocalBaseImageExpected,
bool isDryRun);
}

Expand Down Expand Up @@ -60,6 +61,7 @@ public async Task<ImageCacheResult> CheckForCachedImageAsync(
ImageDigestCache imageDigestCache,
ImageNameResolver imageNameResolver,
string? sourceRepoUrl,
bool isLocalBaseImageExpected,
bool isDryRun)
{
ImageCacheState cacheState = ImageCacheState.NotCached;
Expand Down Expand Up @@ -96,6 +98,7 @@ public async Task<ImageCacheResult> CheckForCachedImageAsync(
imageDigestCache,
imageNameResolver,
sourceRepoUrl,
isLocalBaseImageExpected,
isDryRun);

if (isCachedImage)
Expand Down Expand Up @@ -127,14 +130,15 @@ private async Task<bool> CheckForCachedImageFromImageInfoAsync(
ImageDigestCache imageDigestCache,
ImageNameResolver imageNameResolver,
string? sourceRepoUrl,
bool isLocalBaseImageExpected,
bool isDryRun)
{
_loggerService.WriteMessage($"Checking for cached image for '{platform.DockerfilePathRelativeToManifest}'");

// If the previously published image was based on an image that is still the latest version AND
// the Dockerfile hasn't changed since it was last published
if (await IsBaseImageDigestUpToDateAsync(
platform, srcPlatformData, imageDigestCache, imageNameResolver, isDryRun) &&
platform, srcPlatformData, imageDigestCache, imageNameResolver, isLocalBaseImageExpected, isDryRun) &&
IsDockerfileUpToDate(platform, srcPlatformData, sourceRepoUrl))
{
return true;
Expand All @@ -151,6 +155,7 @@ private async Task<bool> IsBaseImageDigestUpToDateAsync(
PlatformData srcPlatformData,
ImageDigestCache imageDigestCache,
ImageNameResolver imageNameResolver,
bool isLocalImageExpected,
bool isDryRun)
{
_loggerService.WriteMessage();
Expand All @@ -164,14 +169,26 @@ private async Task<bool> IsBaseImageDigestUpToDateAsync(
string queryImage = imageNameResolver.GetFinalStageImageNameForDigestQuery(platform);

string? currentSha;
try
if (isLocalImageExpected)
{
currentSha = await imageDigestCache.GetManifestDigestShaAsync(queryImage, isDryRun);
currentSha = await imageDigestCache.GetLocalImageDigestAsync(
imageNameResolver.GetFromImageLocalTag(platform.FinalStageFromImage), isDryRun);
if (currentSha is not null)
{
currentSha = DockerHelper.GetDigestSha(currentSha);
}
}
// Handle cases where the image is not found in the registry yet
catch (Exception)
else
{
currentSha = null;
try
{
currentSha = await imageDigestCache.GetManifestDigestShaAsync(queryImage, isDryRun);
}
// Handle cases where the image is not found in the registry yet
catch (Exception)
{
currentSha = null;
}
}

string? imageInfoSha = srcPlatformData.BaseImageDigest is not null ?
Expand Down
26 changes: 12 additions & 14 deletions src/Microsoft.DotNet.ImageBuilder/tests/BuildCommandTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1085,6 +1085,13 @@ public async Task BuildCommand_ThrowsIfImageIsPulled(bool isSkipPullingEnabled)
null, "runtimeDepsCommitSha-1",
null, "runtimeCommitSha-1",
false, false)]
[InlineData(
"All previously published, commit diff for runtime-deps",
"sha256:baseImageSha", "sha256:baseImageSha",
"sha256:runtimeDepsImageSha-1", "sha256:runtimeDepsImageSha-1",
"runtimeDepsCommitSha-1", "runtimeDepsCommitSha-2",
"runtimeCommitSha", "runtimeCommitSha",
false, false)]
public async Task BuildCommand_Caching(
string scenario,
string sourceBaseImageSha,
Expand Down Expand Up @@ -1125,7 +1132,7 @@ public async Task BuildCommand_Caching(
localImageDigestResults:
[
new($"{runtimeDepsRepoQualified}:{tag}", runtimeDepsDigest),
new($"{overridePrefix}{runtimeDepsRepo}:{tag}", runtimeDepsDigest),
new($"{overridePrefix}{runtimeDepsRepo}:{tag}", runtimeDepsDigest, OnCallCount: 2),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the reason for only returning this on the second call now?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The first call should return null to simulate it's locally built and doesn't have a digest yet. Later during the build, after a push, it calls it again and now it should have the digest.

new($"{runtimeRepoQualified}:{tag}", runtimeDigest),
new($"{overridePrefix}{runtimeRepo}:{tag}", runtimeDigest),
new(baseImageTag, runtimeDepsBaseImageDigest),
Expand Down Expand Up @@ -1443,11 +1450,8 @@ public async Task BuildCommand_Caching_SharedDockerfile_MissingSourceImageInfoEn
new($"{runtimeDepsRepo}:{linuxTag}", runtimeDepsLinuxDigest),
new($"{runtimeDepsRepo}:{windowsTag}", runtimeDepsWindowsDigest),
new($"{runtimeDeps2Repo}:{linuxTag}", runtimeDeps2Digest),
],
externalImageDigestResults:
[
new(linuxBaseImageTag, runtimeDepsLinuxBaseImageDigestSha),
new(windowsBaseImageTag, runtimeDepsWindowsBaseImageDigestSha),
new(linuxBaseImageTag, runtimeDepsLinuxBaseImageDigest),
new(windowsBaseImageTag, runtimeDepsWindowsBaseImageDigest),
]);
Mock<IManifestServiceFactory> manifestServiceFactoryMock = CreateManifestServiceFactoryMock(manifestServiceMock);

Expand Down Expand Up @@ -2516,10 +2520,7 @@ public async Task BuildCommand_Caching_TagUpdate()
localImageDigestResults:
[
new($"{runtimeDepsRepo}:{tag}", runtimeDepsDigest),
],
externalImageDigestResults:
[
new(baseImageTag, runtimeDepsLinuxBaseImageDigestSha),
new(baseImageTag, runtimeDepsLinuxBaseImageDigest),
]);

DateTime createdDate = DateTime.Now;
Expand Down Expand Up @@ -2740,10 +2741,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_TagUpdate()
[
new($"{runtimeDepsRepo}:{tag}", runtimeDepsLinuxDigest),
new($"{runtimeDeps2Repo}:{tag}", runtimeDeps2Digest),
],
externalImageDigestResults:
[
new(baseImageTag, runtimeDepsLinuxBaseImageDigestSha),
new(baseImageTag, runtimeDepsLinuxBaseImageDigest),
]);

DateTime createdDate = DateTime.Now;
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.DotNet.ImageBuilder/tests/GenerateBuildMatrixCommandTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ private static void SetCacheResult(Mock<IImageCacheService> imageCacheServiceMoc
It.IsAny<ImageDigestCache>(),
It.IsAny<ImageNameResolver>(),
It.IsAny<string>(),
It.IsAny<bool>(),
It.IsAny<bool>()))
.ReturnsAsync(new ImageCacheResult(cacheState, false, null));
}
Expand Down
10 changes: 5 additions & 5 deletions src/Microsoft.DotNet.ImageBuilder/tests/Helpers/ManifestServiceHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace Microsoft.DotNet.ImageBuilder.Tests.Helpers;

internal static class ManifestServiceHelper
{
public record ImageDigestResults(string Image, string Digest);
public record ImageDigestResults(string Image, string Digest, int OnCallCount = 1);

public record ImageLayersResults(string Image, IEnumerable<string> Layers);

Expand Down Expand Up @@ -53,18 +53,18 @@ public static Mock<IManifestService> CreateManifestServiceMock(
externalImageDigestResults ??= [];
imageLayersResults ??= [];

foreach ((string image, string digest) in localImageDigestResults)
foreach ((string image, string digest, int onCallCount) in localImageDigestResults)
{
manifestServiceMock
.Setup(o => o.GetLocalImageDigestAsync(image, false))
.ReturnsAsync(digest);
.ReturnsAsync(callCount => callCount >= onCallCount ? digest : null);
}

foreach ((string image, string digest) in externalImageDigestResults)
foreach ((string image, string digest, int onCallIndex) in externalImageDigestResults)
{
manifestServiceMock
.Setup(o => o.GetManifestDigestShaAsync(image, false))
.ReturnsAsync(digest);
.ReturnsAsync(callIndex => callIndex >= onCallIndex ? digest : throw new Exception());
}

foreach ((string image, IEnumerable<string> layers) in imageLayersResults)
Expand Down
Loading