Skip to content

Commit 026ccf1

Browse files
yosifkityosifkit
authored
Merge pull request #61 from infosiftr/oci-import
Add "Builder: oci-import" support
2 parents 18db6c5 + 449eb48 commit 026ccf1

15 files changed

+1824
-81
lines changed

‎cmd/bashbrew/cmd-build.go

+33-22
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"fmt"
5+
"strings"
56

67
"github.com/urfave/cli"
78
)
@@ -76,6 +77,8 @@ func cmdBuild(c *cli.Context) error {
7677
if err != nil {
7778
return cli.NewMultiError(fmt.Errorf(`failed calculating "cache hash" for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
7879
}
80+
imageTags := r.Tags(namespace, uniq, entry)
81+
tags := append([]string{cacheTag}, imageTags...)
7982

8083
// check whether we've already built this artifact
8184
_, err = dockerInspect("{{.Id}}", cacheTag)
@@ -87,48 +90,56 @@ func cmdBuild(c *cli.Context) error {
8790
return cli.NewMultiError(fmt.Errorf(`failed fetching git repo for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
8891
}
8992

90-
archive, err := gitArchive(commit, entry.ArchDirectory(arch))
91-
if err != nil {
92-
return cli.NewMultiError(fmt.Errorf(`failed generating git archive for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
93-
}
94-
defer archive.Close()
95-
96-
// TODO use "meta.StageNames" to do "docker build --target" so we can tag intermediate stages too for cache (streaming "git archive" directly to "docker build" makes that a little hard to accomplish without re-streaming)
97-
9893
switch builder := entry.ArchBuilder(arch); builder {
99-
case "classic", "":
94+
case "buildkit", "classic", "":
10095
var platform string
10196
if fromScratch {
10297
platform = ociArch.String()
10398
}
104-
err = dockerBuild(cacheTag, entry.ArchFile(arch), archive, platform)
99+
100+
archive, err := gitArchive(commit, entry.ArchDirectory(arch))
105101
if err != nil {
106-
return cli.NewMultiError(fmt.Errorf(`failed building %q (tags %q)`, r.RepoName, entry.TagsString()), err)
102+
return cli.NewMultiError(fmt.Errorf(`failed generating git archive for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
107103
}
108-
case "buildkit":
109-
var platform string
110-
if fromScratch {
111-
platform = ociArch.String()
104+
defer archive.Close()
105+
106+
if builder == "buildkit" {
107+
err = dockerBuildxBuild(tags, entry.ArchFile(arch), archive, platform)
108+
} else {
109+
// TODO use "meta.StageNames" to do "docker build --target" so we can tag intermediate stages too for cache (streaming "git archive" directly to "docker build" makes that a little hard to accomplish without re-streaming)
110+
err = dockerBuild(tags, entry.ArchFile(arch), archive, platform)
112111
}
113-
err = dockerBuildxBuild(cacheTag, entry.ArchFile(arch), archive, platform)
114112
if err != nil {
115113
return cli.NewMultiError(fmt.Errorf(`failed building %q (tags %q)`, r.RepoName, entry.TagsString()), err)
116114
}
115+
116+
archive.Close() // be sure this happens sooner rather than later (defer might take a while, and we want to reap zombies more aggressively)
117+
118+
case "oci-import":
119+
err := ociImportBuild(tags, commit, entry.ArchDirectory(arch), entry.ArchFile(arch))
120+
if err != nil {
121+
return cli.NewMultiError(fmt.Errorf(`failed oci-import build of %q (tags %q)`, r.RepoName, entry.TagsString()), err)
122+
}
123+
124+
fmt.Printf("Importing %s into Docker\n", r.EntryIdentifier(entry))
125+
err = ociImportDockerLoad(imageTags)
126+
if err != nil {
127+
return cli.NewMultiError(fmt.Errorf(`failed oci-import into Docker of %q (tags %q)`, r.RepoName, entry.TagsString()), err)
128+
}
129+
117130
default:
118131
return cli.NewMultiError(fmt.Errorf(`unknown builder %q`, builder))
119132
}
120-
archive.Close() // be sure this happens sooner rather than later (defer might take a while, and we want to reap zombies more aggressively)
121133
}
122134
} else {
123135
fmt.Printf("Using %s (%s)\n", cacheTag, r.EntryIdentifier(entry))
124-
}
125136

126-
for _, tag := range r.Tags(namespace, uniq, entry) {
127-
fmt.Printf("Tagging %s\n", tag)
128137
if !dryRun {
129-
err := dockerTag(cacheTag, tag)
138+
// https://github.com/docker-library/bashbrew/pull/61/files#r1044926620
139+
// abusing "docker build" for "tag something a lot of times, but efficiently" 👀
140+
err := dockerBuild(imageTags, "", strings.NewReader("FROM "+cacheTag), "")
130141
if err != nil {
131-
return cli.NewMultiError(fmt.Errorf(`failed tagging %q as %q`, cacheTag, tag), err)
142+
return cli.NewMultiError(fmt.Errorf(`failed tagging %q: %q`, cacheTag, strings.Join(imageTags, ", ")), err)
132143
}
133144
}
134145
}

‎cmd/bashbrew/cmd-push.go

+54-13
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"os"
66
"path"
7+
"strings"
78

89
"github.com/urfave/cli"
910
)
@@ -38,29 +39,69 @@ func cmdPush(c *cli.Context) error {
3839
continue
3940
}
4041

42+
tags := []string{}
4143
// we can't use "r.Tags()" here because it will include SharedTags, which we never want to push directly (see "cmd-put-shared.go")
42-
TagsLoop:
4344
for i, tag := range entry.Tags {
4445
if uniq && i > 0 {
4546
break
4647
}
4748
tag = tagRepo + ":" + tag
49+
tags = append(tags, tag)
50+
}
4851

49-
if !force {
50-
localImageId, _ := dockerInspect("{{.Id}}", tag)
51-
registryImageIds := fetchRegistryImageIds(tag)
52-
for _, registryImageId := range registryImageIds {
53-
if localImageId == registryImageId {
54-
fmt.Fprintf(os.Stderr, "skipping %s (remote image matches local)\n", tag)
55-
continue TagsLoop
56-
}
57-
}
52+
switch builder := entry.ArchBuilder(arch); builder {
53+
case "oci-import":
54+
cacheTag, err := r.DockerCacheName(entry)
55+
if err != nil {
56+
return cli.NewMultiError(fmt.Errorf(`failed calculating "cache hash" for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
57+
}
58+
desc, err := ociImportLookup(cacheTag)
59+
if err != nil {
60+
return cli.NewMultiError(fmt.Errorf(`failed looking up descriptor for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
5861
}
59-
fmt.Printf("Pushing %s\n", tag)
62+
skip, update, err := ociImportPushFilter(*desc, tags)
63+
if err != nil {
64+
return cli.NewMultiError(fmt.Errorf(`failed looking up tags for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
65+
}
66+
if len(skip) > 0 && len(update) == 0 {
67+
fmt.Fprintf(os.Stderr, "skipping %s (remote tags all up-to-date)\n", r.EntryIdentifier(entry))
68+
continue
69+
} else if len(skip) > 0 {
70+
fmt.Fprintf(os.Stderr, "partially skipping %s (remote tags up-to-date: %s)\n", r.EntryIdentifier(entry), strings.Join(skip, ", "))
71+
}
72+
fmt.Printf("Pushing %s to %s\n", desc.Digest, strings.Join(update, ", "))
6073
if !dryRun {
61-
err = dockerPush(tag)
74+
err := ociImportPush(*desc, update)
6275
if err != nil {
63-
return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, tag), err)
76+
return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, r.EntryIdentifier(entry)), err)
77+
}
78+
}
79+
80+
default:
81+
TagsLoop:
82+
for _, tag := range tags {
83+
if !force {
84+
localImageId, _ := dockerInspect("{{.Id}}", tag)
85+
if debugFlag {
86+
fmt.Printf("DEBUG: docker inspect %q -> %q\n", tag, localImageId)
87+
}
88+
registryImageIds := fetchRegistryImageIds(tag)
89+
if debugFlag {
90+
fmt.Printf("DEBUG: registry inspect %q -> %+v\n", tag, registryImageIds)
91+
}
92+
for _, registryImageId := range registryImageIds {
93+
if localImageId == registryImageId {
94+
fmt.Fprintf(os.Stderr, "skipping %s (remote image matches local)\n", tag)
95+
continue TagsLoop
96+
}
97+
}
98+
}
99+
fmt.Printf("Pushing %s\n", tag)
100+
if !dryRun {
101+
err = dockerPush(tag)
102+
if err != nil {
103+
return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, tag), err)
104+
}
64105
}
65106
}
66107
}

‎cmd/bashbrew/containerd.go

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"os"
6+
"path/filepath"
7+
"time"
8+
9+
"github.com/containerd/containerd"
10+
"github.com/containerd/containerd/content/local"
11+
"github.com/containerd/containerd/metadata"
12+
"github.com/containerd/containerd/namespaces"
13+
14+
"go.etcd.io/bbolt"
15+
)
16+
17+
func newBuiltinContainerdServices(ctx context.Context) (containerd.ClientOpt, error) {
18+
// thanks to https://github.com/Azure/image-rootfs-scanner/blob/e7041e47d1a13e15d73d9c85644542e6758f9f3a/containerd.go#L42-L87 for inspiring this magic
19+
20+
root := filepath.Join(defaultCache, "containerd")
21+
dbPath := filepath.Join(root, "metadata.db")
22+
contentRoot := filepath.Join(root, "content")
23+
24+
cs, err := local.NewStore(contentRoot)
25+
if err != nil {
26+
return nil, err
27+
}
28+
29+
db, err := bbolt.Open(dbPath, 0600, &bbolt.Options{
30+
Timeout: 1 * time.Minute,
31+
})
32+
33+
mdb := metadata.NewDB(db, cs, nil)
34+
return containerd.WithServices(
35+
containerd.WithContentStore(mdb.ContentStore()),
36+
containerd.WithImageStore(metadata.NewImageStore(mdb)),
37+
containerd.WithLeasesService(metadata.NewLeaseManager(mdb)),
38+
), nil
39+
}
40+
41+
var containerdClientCache *containerd.Client = nil
42+
43+
// the returned client is cached, don't Close() it!
44+
func newContainerdClient(ctx context.Context) (context.Context, *containerd.Client, error) {
45+
ns := "bashbrew"
46+
for _, envKey := range []string{
47+
`BASHBREW_CONTAINERD_NAMESPACE`,
48+
`CONTAINERD_NAMESPACE`,
49+
} {
50+
if env, ok := os.LookupEnv(envKey); ok {
51+
if env != "" {
52+
// set-but-empty environment variable means use default explicitly
53+
ns = env
54+
}
55+
break
56+
}
57+
}
58+
ctx = namespaces.WithNamespace(ctx, ns)
59+
60+
if containerdClientCache != nil {
61+
return ctx, containerdClientCache, nil
62+
}
63+
64+
for _, envKey := range []string{
65+
`BASHBREW_CONTAINERD_CONTENT_ADDRESS`, // TODO if we ever need to connnect to a containerd instance for something more interesting like running containers, we need to have *that* codepath not use _CONTENT_ variants
66+
`BASHBREW_CONTAINERD_ADDRESS`,
67+
`CONTAINERD_CONTENT_ADDRESS`,
68+
`CONTAINERD_ADDRESS`,
69+
} {
70+
if socket, ok := os.LookupEnv(envKey); ok {
71+
if socket == "" {
72+
// we'll use a set-but-empty variable as an explicit request to use our built-in implementation
73+
break
74+
}
75+
client, err := containerd.New(socket)
76+
containerdClientCache = client
77+
return ctx, client, err
78+
}
79+
}
80+
81+
// if we don't have an explicit variable asking us to connect to an existing containerd instance, we set up and use our own in-process content/image store
82+
services, err := newBuiltinContainerdServices(ctx)
83+
if err != nil {
84+
return ctx, nil, err
85+
}
86+
client, err := containerd.New("", services)
87+
containerdClientCache = client
88+
return ctx, client, err
89+
}

‎cmd/bashbrew/docker.go

+23-7
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,14 @@ func (r Repo) dockerfileMetadata(entry *manifest.Manifest2822Entry) (*dockerfile
5353
var dockerfileMetadataCache = map[string]*dockerfileMetadata{}
5454

5555
func (r Repo) archDockerfileMetadata(arch string, entry *manifest.Manifest2822Entry) (*dockerfileMetadata, error) {
56+
if builder := entry.ArchBuilder(arch); builder == "oci-import" {
57+
return &dockerfileMetadata{
58+
Froms: []string{
59+
"scratch",
60+
},
61+
}, nil
62+
}
63+
5664
commit, err := r.fetchGitRepo(arch, entry)
5765
if err != nil {
5866
return nil, cli.NewMultiError(fmt.Errorf("failed fetching Git repo for arch %q from entry %q", arch, entry.String()), err)
@@ -242,9 +250,16 @@ func (r Repo) dockerBuildUniqueBits(entry *manifest.Manifest2822Entry) ([]string
242250
return uniqueBits, nil
243251
}
244252

245-
func dockerBuild(tag string, file string, context io.Reader, platform string) error {
246-
args := []string{"build", "--tag", tag, "--file", file, "--rm", "--force-rm"}
247-
args = append(args, "-")
253+
func dockerBuild(tags []string, file string, context io.Reader, platform string) error {
254+
args := []string{"build"}
255+
for _, tag := range tags {
256+
args = append(args, "--tag", tag)
257+
}
258+
if file != "" {
259+
args = append(args, "--file", file)
260+
}
261+
args = append(args, "--rm", "--force-rm", "-")
262+
248263
cmd := exec.Command("docker", args...)
249264
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=0")
250265
if debugFlag {
@@ -278,7 +293,7 @@ func dockerBuild(tag string, file string, context io.Reader, platform string) er
278293

279294
const dockerfileSyntaxEnv = "BASHBREW_BUILDKIT_SYNTAX"
280295

281-
func dockerBuildxBuild(tag string, file string, context io.Reader, platform string) error {
296+
func dockerBuildxBuild(tags []string, file string, context io.Reader, platform string) error {
282297
dockerfileSyntax, ok := os.LookupEnv(dockerfileSyntaxEnv)
283298
if !ok {
284299
return fmt.Errorf("missing %q", dockerfileSyntaxEnv)
@@ -289,13 +304,14 @@ func dockerBuildxBuild(tag string, file string, context io.Reader, platform stri
289304
"build",
290305
"--progress", "plain",
291306
"--build-arg", "BUILDKIT_SYNTAX=" + dockerfileSyntax,
292-
"--tag", tag,
293-
"--file", file,
294307
}
295308
if platform != "" {
296309
args = append(args, "--platform", platform)
297310
}
298-
args = append(args, "-")
311+
for _, tag := range tags {
312+
args = append(args, "--tag", tag)
313+
}
314+
args = append(args, "--file", file, "-")
299315

300316
cmd := exec.Command("docker", args...)
301317
cmd.Stdin = context

‎cmd/bashbrew/git.go

+9
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"fmt"
55
"io"
6+
"io/fs"
67
"io/ioutil"
78
"os"
89
"os/exec"
@@ -15,6 +16,7 @@ import (
1516

1617
"github.com/docker-library/bashbrew/manifest"
1718
"github.com/docker-library/bashbrew/pkg/execpipe"
19+
"github.com/docker-library/bashbrew/pkg/gitfs"
1820

1921
goGit "github.com/go-git/go-git/v5"
2022
goGitConfig "github.com/go-git/go-git/v5/config"
@@ -94,6 +96,13 @@ func getGitCommit(commit string) (string, error) {
9496
return h.String(), nil
9597
}
9698

99+
func gitCommitFS(commit string) (fs.FS, error) {
100+
if err := ensureGitInit(); err != nil {
101+
return nil, err
102+
}
103+
return gitfs.CommitHash(gitRepo, commit)
104+
}
105+
97106
func gitStream(args ...string) (io.ReadCloser, error) {
98107
return execpipe.Run(gitCommand(args...))
99108
}

‎cmd/bashbrew/main.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,8 @@ func main() {
167167
if !debugFlag {
168168
// containerd uses logrus, but it defaults to "info" (which is a bit leaky where we use containerd)
169169
logrus.SetLevel(logrus.WarnLevel)
170+
} else {
171+
logrus.SetLevel(logrus.DebugLevel)
170172
}
171173

172174
arch = c.GlobalString("arch")
@@ -401,9 +403,9 @@ func main() {
401403
Category: "plumbing",
402404
},
403405
{
404-
Name: "remote",
405-
Usage: "query registries for bashbrew-related data",
406-
Before: subcommandBeforeFactory("remote"),
406+
Name: "remote",
407+
Usage: "query registries for bashbrew-related data",
408+
Before: subcommandBeforeFactory("remote"),
407409
Category: "plumbing",
408410
Subcommands: []cli.Command{
409411
{

0 commit comments

Comments
 (0)