Skip to content

Commit 6be996d

Browse files
0x416e746f6e0x416e746f6e
committed
feat: derive tags for indexed sub-images
1 parent 69c3d28 commit 6be996d

File tree

10 files changed

+271
-89
lines changed

10 files changed

+271
-89
lines changed

‎job/sync_container_registry_package.go

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"strings"
77

88
"github.com/flashbots/gh-artifacts-sync/config"
9+
"github.com/flashbots/gh-artifacts-sync/utils"
910
"github.com/google/go-github/v72/github"
1011
)
1112

@@ -53,15 +54,37 @@ func (j *SyncContainerRegistryPackage) meta() *Meta {
5354
return j.Meta
5455
}
5556

57+
func (j *SyncContainerRegistryPackage) IsTagless() bool {
58+
if j == nil ||
59+
j.Package == nil ||
60+
j.Package.PackageVersion == nil ||
61+
j.Package.PackageVersion.ContainerMetadata == nil ||
62+
j.Package.PackageVersion.ContainerMetadata.Tag == nil ||
63+
j.Package.PackageVersion.ContainerMetadata.Tag.Name == nil {
64+
// ---
65+
return true
66+
}
67+
return *j.Package.PackageVersion.ContainerMetadata.Tag.Name == ""
68+
}
69+
5670
func (j *SyncContainerRegistryPackage) GetDestinations() []*config.Destination {
5771
return j.Destinations
5872
}
5973

6074
func (j *SyncContainerRegistryPackage) GetDestinationReference(dst *config.Destination) string {
61-
tag := *j.Package.PackageVersion.ContainerMetadata.Tag.Name
75+
if j == nil ||
76+
j.Package == nil ||
77+
j.Package.PackageVersion == nil ||
78+
j.Package.PackageVersion.ContainerMetadata == nil ||
79+
j.Package.PackageVersion.ContainerMetadata.Tag == nil {
80+
// ---
81+
return ""
82+
}
83+
84+
tag := utils.MustString(j.Package.PackageVersion.ContainerMetadata.Tag.Name)
6285
if tag == "" {
6386
tag = strings.ReplaceAll(
64-
*j.Package.PackageVersion.ContainerMetadata.Tag.Digest, ":", "-",
87+
utils.MustString(j.Package.PackageVersion.ContainerMetadata.Tag.Digest), ":", "-",
6588
)
6689
}
6790
return dst.Path + "/" + dst.Package + ":" + tag

‎job/uploadable.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ type UploadableFile interface {
1212
}
1313

1414
type UploadableContainer interface {
15+
IsTagless() bool
1516
GetDestinations() []*config.Destination
1617
GetDestinationReference(*config.Destination) string
1718
GetTag() string

‎server/docker.go

Lines changed: 182 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,12 @@ import (
1414
"github.com/flashbots/gh-artifacts-sync/utils"
1515
"go.uber.org/zap"
1616

17+
crauthn "github.com/google/go-containerregistry/pkg/authn"
1718
crname "github.com/google/go-containerregistry/pkg/name"
1819
cr "github.com/google/go-containerregistry/pkg/v1"
1920
crempty "github.com/google/go-containerregistry/pkg/v1/empty"
2021
crmutate "github.com/google/go-containerregistry/pkg/v1/mutate"
22+
crremote "github.com/google/go-containerregistry/pkg/v1/remote"
2123
crtarball "github.com/google/go-containerregistry/pkg/v1/tarball"
2224
)
2325

@@ -28,45 +30,60 @@ type container struct {
2830
manifest *cr.Manifest
2931
}
3032

31-
func (s *Server) prepareIndexManifestForDestination(
33+
func (s *Server) dockerExtractImagesAndAttestations(
3234
indexManifest *cr.IndexManifest,
33-
dst *config.Destination,
34-
) error {
35-
if indexManifest == nil || dst == nil {
36-
return nil
35+
) (images map[string]*cr.Descriptor, attestations map[string]*cr.Descriptor, err error) {
36+
if indexManifest == nil {
37+
return nil, nil, nil
3738
}
3839

39-
var (
40-
attestations = make(map[string]*cr.Descriptor, 0)
41-
images = make(map[string]*cr.Descriptor, 0)
42-
errs = make([]error, 0)
43-
)
40+
images = make(map[string]*cr.Descriptor, 0)
41+
attestations = make(map[string]*cr.Descriptor, 0)
4442

45-
{ // separate images from their respective attestations
46-
for _, desc := range indexManifest.Manifests {
47-
if desc.Annotations["vnd.docker.reference.type"] != "attestation-manifest" {
48-
images[desc.Digest.String()] = &desc
49-
continue
50-
}
43+
errs := make([]error, 0)
44+
for _, desc := range indexManifest.Manifests {
45+
if desc.Annotations["vnd.docker.reference.type"] != "attestation-manifest" {
46+
images[desc.Digest.String()] = &desc
47+
continue
48+
}
5149

52-
digest := desc.Annotations["vnd.docker.reference.digest"]
53-
if digest == "" {
54-
err := fmt.Errorf("index contains reference w/o digest: %s",
55-
desc.Digest.String(),
56-
)
57-
errs = append(errs, err)
58-
continue
59-
}
50+
digest := desc.Annotations["vnd.docker.reference.digest"]
51+
if digest == "" {
52+
err := fmt.Errorf("index contains reference w/o digest: %s",
53+
desc.Digest.String(),
54+
)
55+
errs = append(errs, err)
56+
continue
57+
}
6058

61-
if another, collision := attestations[digest]; collision {
62-
err := fmt.Errorf("index contains multiple attestations for the same reference: %s: %s vs. %s",
63-
digest, desc.Digest.String(), another.Digest.String(),
64-
)
65-
errs = append(errs, err)
66-
continue
67-
}
59+
if another, collision := attestations[digest]; collision {
60+
err := fmt.Errorf("index contains multiple attestations for the same reference: %s: %s vs. %s",
61+
digest, desc.Digest.String(), another.Digest.String(),
62+
)
63+
errs = append(errs, err)
64+
continue
65+
}
66+
67+
attestations[digest] = &desc
68+
}
69+
70+
return images, attestations, utils.FlattenErrors(errs)
71+
}
72+
73+
func (s *Server) dockerFilterIndexManifest(
74+
indexManifest *cr.IndexManifest,
75+
dst *config.Destination,
76+
) error {
77+
var (
78+
attestations map[string]*cr.Descriptor
79+
images map[string]*cr.Descriptor
80+
err error
81+
)
6882

69-
attestations[digest] = &desc
83+
{ // separate images from their respective attestations
84+
images, attestations, err = s.dockerExtractImagesAndAttestations(indexManifest)
85+
if len(images) == 0 && len(attestations) == 0 {
86+
return err
7087
}
7188
}
7289

@@ -87,10 +104,10 @@ func (s *Server) prepareIndexManifestForDestination(
87104
}
88105
}
89106

90-
return utils.FlattenErrors(errs)
107+
return nil
91108
}
92109

93-
func (s *Server) prepareImageForUpload(
110+
func (s *Server) dockerPrepareImage(
94111
ctx context.Context,
95112
j job.UploadableContainer,
96113
stream *zip.ReadCloser,
@@ -131,7 +148,7 @@ func (s *Server) prepareImageForUpload(
131148
}
132149

133150
case ".tar":
134-
image, err := crtarball.Image(zipFileOpener(f), nil)
151+
image, err := crtarball.Image(helperZipFileOpener(f), nil)
135152
if err != nil {
136153
l.Error("Failed to open container tarball", zap.Error(err))
137154
errs = append(errs, err)
@@ -172,7 +189,7 @@ func (s *Server) prepareImageForUpload(
172189
}
173190
}
174191

175-
{ // filter platforms
192+
{ // filter by platform
176193
switch indexManifest {
177194
case nil:
178195
for originalDigest, container := range containers {
@@ -182,7 +199,7 @@ func (s *Server) prepareImageForUpload(
182199
}
183200

184201
default:
185-
if err := s.prepareIndexManifestForDestination(indexManifest, dst); err != nil {
202+
if err := s.dockerFilterIndexManifest(indexManifest, dst); err != nil {
186203
errs = append(errs, err)
187204
}
188205
_containers := make(map[string]*container)
@@ -194,6 +211,7 @@ func (s *Server) prepareImageForUpload(
194211
containers = _containers
195212
}
196213
}
214+
197215
if len(containers) == 0 {
198216
l.Info("No matching platforms, skipping...")
199217
return nil, nil, nil, utils.FlattenErrors(errs)
@@ -223,29 +241,138 @@ func (s *Server) prepareImageForUpload(
223241
}
224242

225243
var index cr.ImageIndex = crempty.Index
226-
for _, desc := range indexManifest.Manifests {
227-
originalDigest := desc.Digest.String()
228-
container := containers[originalDigest]
229-
annotations := desc.Annotations
230-
231-
if annotations["vnd.docker.reference.type"] == "attestation-manifest" {
232-
if annotationOriginalDigest, ok := annotations["vnd.docker.reference.digest"]; ok {
233-
if reference, ok := containers[annotationOriginalDigest]; ok {
234-
annotations["vnd.docker.reference.digest"] = reference.digest.String()
244+
{ // prepare container index
245+
for _, desc := range indexManifest.Manifests {
246+
originalDigest := desc.Digest.String()
247+
container := containers[originalDigest]
248+
annotations := desc.Annotations
249+
250+
if annotations["vnd.docker.reference.type"] == "attestation-manifest" {
251+
if originalReferenceDigest, ok := annotations["vnd.docker.reference.digest"]; ok {
252+
if reference, ok := containers[originalReferenceDigest]; ok {
253+
annotations["vnd.docker.reference.digest"] = reference.digest.String()
254+
}
235255
}
236256
}
257+
258+
index = crmutate.AppendManifests(index, crmutate.IndexAddendum{
259+
Add: container.image,
260+
261+
Descriptor: cr.Descriptor{
262+
Annotations: annotations,
263+
Digest: container.digest,
264+
Platform: container.config.Platform(),
265+
},
266+
})
237267
}
268+
}
269+
return ref, nil, index, utils.FlattenErrors(errs)
270+
}
271+
272+
func (s *Server) dockerTagRemoteSubImages(
273+
ctx context.Context,
274+
ref crname.Reference,
275+
auth crauthn.Authenticator,
276+
) error {
277+
l := logutils.LoggerFromContext(ctx)
238278

239-
index = crmutate.AppendManifests(index, crmutate.IndexAddendum{
240-
Add: container.image,
279+
desc, err := crremote.Get(ref, crremote.WithAuth(auth))
280+
if err != nil {
281+
return fmt.Errorf("failed to get a descriptor for container image: %s: %w",
282+
ref.Name(), err,
283+
)
284+
}
241285

242-
Descriptor: cr.Descriptor{
243-
Annotations: annotations,
244-
Digest: container.digest,
245-
Platform: container.config.Platform(),
246-
},
247-
})
286+
if !desc.MediaType.IsIndex() {
287+
return nil
248288
}
249289

250-
return ref, nil, index, utils.FlattenErrors(errs)
290+
index, err := crremote.Index(ref, crremote.WithAuth(auth))
291+
if err != nil {
292+
return fmt.Errorf("failed to retrieve container index: %s: %w",
293+
ref.Name(), err,
294+
)
295+
}
296+
297+
indexManifest, err := index.IndexManifest()
298+
if err != nil {
299+
return fmt.Errorf("failed to get image index manifest from a descriptor: %s: %s: %w",
300+
ref.Name(), desc.Digest.String(), err,
301+
)
302+
}
303+
304+
l.Debug("Downloaded an index",
305+
zap.String("digest", desc.Digest.String()),
306+
zap.String("reference", ref.Name()),
307+
zap.Any("annotations", indexManifest.Annotations),
308+
)
309+
310+
images, attestations, err := s.dockerExtractImagesAndAttestations(indexManifest)
311+
if len(images) == 0 && len(attestations) == 0 {
312+
return err
313+
}
314+
315+
errs := make([]error, 0)
316+
for _, desc := range images {
317+
image, err := index.Image(desc.Digest)
318+
if err != nil {
319+
errs = append(errs, fmt.Errorf("failed to get image from an index: %s: %s: %w",
320+
ref.Name(), desc.Digest.String(), err,
321+
))
322+
continue
323+
}
324+
325+
_tag := fmt.Sprintf("%s:%s-%s-%s",
326+
ref.Context().Name(), ref.Identifier(), desc.Platform.OS, desc.Platform.Architecture,
327+
)
328+
tag, err := crname.NewTag(_tag)
329+
if err != nil {
330+
errs = append(errs, fmt.Errorf("failed to parse a tag: %s: %w",
331+
_tag, err,
332+
))
333+
continue
334+
}
335+
336+
if err := crremote.Tag(tag, image, crremote.WithAuth(auth)); err != nil {
337+
errs = append(errs, fmt.Errorf("failed to tag sub-image: %s: %w",
338+
_tag, err,
339+
))
340+
continue
341+
}
342+
}
343+
344+
for digest, desc := range attestations {
345+
reference, ok := images[digest]
346+
if !ok {
347+
continue
348+
}
349+
350+
image, err := index.Image(desc.Digest)
351+
if err != nil {
352+
errs = append(errs, fmt.Errorf("failed to get image from an index: %s: %s: %w",
353+
ref.Name(), desc.Digest.String(), err,
354+
))
355+
continue
356+
}
357+
358+
_tag := fmt.Sprintf("%s:%s-%s-%s-attestation",
359+
ref.Context().Name(), ref.Identifier(), reference.Platform.OS, reference.Platform.Architecture,
360+
)
361+
tag, err := crname.NewTag(_tag)
362+
if err != nil {
363+
errs = append(errs, fmt.Errorf("failed to parse a tag: %s: %w",
364+
_tag, err,
365+
))
366+
continue
367+
}
368+
369+
if err := crremote.Tag(tag, image, crremote.WithAuth(auth)); err != nil {
370+
errs = append(errs, fmt.Errorf("failed to tag sub-image: %s: %w",
371+
_tag, err,
372+
))
373+
continue
374+
}
375+
}
376+
377+
return utils.FlattenErrors(errs)
251378
}

0 commit comments

Comments
 (0)