Skip to content

Commit f63f73a

Browse files
committed
Configure shims from runtime config
In dockerd we already have a concept of a "runtime", which specifies the OCI runtime to use (e.g. runc). This PR extends that config to add containerd shim configuration. This option is only exposed within the daemon itself (cannot be configured in daemon.json). This is due to issues in supporting unknown shims which will require more design work. What this change allows us to do is keep all the runtime config in one place. So the default "runc" runtime will just have it's already existing shim config codified within the runtime config alone. I've also added 2 more "stock" runtimes which are basically runc+shimv1 and runc+shimv2. These new runtime configurations are: - io.containerd.runtime.v1.linux - runc + v1 shim using the V1 shim API - io.containerd.runc.v2 - runc + shim v2 These names coincide with the actual names of the containerd shims. This allows the user to essentially control what shim is going to be used by either specifying these as a `--runtime` on container create or by setting `--default-runtime` on the daemon. For custom/user-specified runtimes, the default shim config (currently shim v1) is used. Signed-off-by: Brian Goff <[email protected]>
1 parent 6fd94aa commit f63f73a

17 files changed

+237
-177
lines changed

api/types/types.go

+10
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,16 @@ type Checkpoint struct {
511511
type Runtime struct {
512512
Path string `json:"path"`
513513
Args []string `json:"runtimeArgs,omitempty"`
514+
515+
// This is exposed here only for internal use
516+
// It is not currently supported to specify custom shim configs
517+
Shim *ShimConfig `json:"-"`
518+
}
519+
520+
// ShimConfig is used by runtime to configure containerd shims
521+
type ShimConfig struct {
522+
Binary string
523+
Opts interface{}
514524
}
515525

516526
// DiskUsage contains response of Engine API:

daemon/config/config.go

+22-7
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,6 @@ const (
3636
// maximum number of attempts that
3737
// may take place at a time for each pull when the connection is lost.
3838
DefaultDownloadAttempts = 5
39-
// StockRuntimeName is the reserved name/alias used to represent the
40-
// OCI runtime being shipped with the docker daemon package.
41-
StockRuntimeName = "runc"
4239
// DefaultShmSize is the default value for container's shm size
4340
DefaultShmSize = int64(67108864)
4441
// DefaultNetworkMtu is the default value for network MTU
@@ -47,8 +44,24 @@ const (
4744
DisableNetworkBridge = "none"
4845
// DefaultInitBinary is the name of the default init binary
4946
DefaultInitBinary = "docker-init"
47+
48+
// StockRuntimeName is the reserved name/alias used to represent the
49+
// OCI runtime being shipped with the docker daemon package.
50+
StockRuntimeName = "runc"
51+
// LinuxV1RuntimeName is the runtime used to specify the containerd v1 shim with the runc binary
52+
// Note this is different than io.containerd.runc.v1 which would be the v1 shim using the v2 shim API.
53+
// This is specifically for the v1 shim using the v1 shim API.
54+
LinuxV1RuntimeName = "io.containerd.runtime.v1.linux"
55+
// LinuxV2RuntimeName is the runtime used to specify the containerd v2 runc shim
56+
LinuxV2RuntimeName = "io.containerd.runc.v2"
5057
)
5158

59+
var builtinRuntimes = map[string]bool{
60+
StockRuntimeName: true,
61+
LinuxV1RuntimeName: true,
62+
LinuxV2RuntimeName: true,
63+
}
64+
5265
// flatOptions contains configuration keys
5366
// that MUST NOT be parsed as deep structures.
5467
// Use this to differentiate these options
@@ -571,10 +584,12 @@ func Validate(config *Config) error {
571584
return err
572585
}
573586

574-
if defaultRuntime := config.GetDefaultRuntimeName(); defaultRuntime != "" && defaultRuntime != StockRuntimeName {
575-
runtimes := config.GetAllRuntimes()
576-
if _, ok := runtimes[defaultRuntime]; !ok {
577-
return fmt.Errorf("specified default runtime '%s' does not exist", defaultRuntime)
587+
if defaultRuntime := config.GetDefaultRuntimeName(); defaultRuntime != "" {
588+
if !builtinRuntimes[defaultRuntime] {
589+
runtimes := config.GetAllRuntimes()
590+
if _, ok := runtimes[defaultRuntime]; !ok {
591+
return fmt.Errorf("specified default runtime '%s' does not exist", defaultRuntime)
592+
}
578593
}
579594
}
580595

daemon/daemon.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -932,7 +932,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
932932
}
933933
}
934934

935-
return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m, d.useShimV2())
935+
var rt types.Runtime
936+
if runtime := config.GetRuntime(config.GetDefaultRuntimeName()); runtime != nil {
937+
rt = *runtime
938+
}
939+
return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m, rt)
936940
}
937941

938942
// Plugin system initialization should happen before restore. Do not change order.
@@ -1081,7 +1085,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
10811085

10821086
go d.execCommandGC()
10831087

1084-
d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d, d.useShimV2())
1088+
d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d)
10851089
if err != nil {
10861090
return nil, err
10871091
}

daemon/daemon_unix.go

+11-63
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import (
3030
"github.com/docker/docker/opts"
3131
"github.com/docker/docker/pkg/containerfs"
3232
"github.com/docker/docker/pkg/idtools"
33-
"github.com/docker/docker/pkg/ioutils"
3433
"github.com/docker/docker/pkg/parsers"
3534
"github.com/docker/docker/pkg/parsers/kernel"
3635
"github.com/docker/docker/pkg/sysinfo"
@@ -78,10 +77,6 @@ const (
7877
cgroupFsDriver = "cgroupfs"
7978
cgroupSystemdDriver = "systemd"
8079
cgroupNoneDriver = "none"
81-
82-
// DefaultRuntimeName is the default runtime to be used by
83-
// containerd if none is specified
84-
DefaultRuntimeName = "runc"
8580
)
8681

8782
type containerGetter interface {
@@ -729,55 +724,11 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.
729724
}
730725
}
731726

732-
return warnings, nil
733-
}
734-
735-
func (daemon *Daemon) loadRuntimes() error {
736-
return daemon.initRuntimes(daemon.configStore.Runtimes)
737-
}
738-
739-
func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error) {
740-
runtimeDir := filepath.Join(daemon.configStore.Root, "runtimes")
741-
// Remove old temp directory if any
742-
os.RemoveAll(runtimeDir + "-old")
743-
tmpDir, err := ioutils.TempDir(daemon.configStore.Root, "gen-runtimes")
744-
if err != nil {
745-
return errors.Wrap(err, "failed to get temp dir to generate runtime scripts")
727+
if hostConfig.Runtime == config.LinuxV1RuntimeName || (hostConfig.Runtime == "" && daemon.configStore.DefaultRuntime == config.LinuxV1RuntimeName) {
728+
warnings = append(warnings, fmt.Sprintf("Configured runtime %q is deprecated and will be removed in the next release.", config.LinuxV1RuntimeName))
746729
}
747-
defer func() {
748-
if err != nil {
749-
if err1 := os.RemoveAll(tmpDir); err1 != nil {
750-
logrus.WithError(err1).WithField("dir", tmpDir).
751-
Warn("failed to remove tmp dir")
752-
}
753-
return
754-
}
755-
756-
if err = os.Rename(runtimeDir, runtimeDir+"-old"); err != nil {
757-
return
758-
}
759-
if err = os.Rename(tmpDir, runtimeDir); err != nil {
760-
err = errors.Wrap(err, "failed to setup runtimes dir, new containers may not start")
761-
return
762-
}
763-
if err = os.RemoveAll(runtimeDir + "-old"); err != nil {
764-
logrus.WithError(err).WithField("dir", tmpDir).
765-
Warn("failed to remove old runtimes dir")
766-
}
767-
}()
768730

769-
for name, rt := range runtimes {
770-
if len(rt.Args) == 0 {
771-
continue
772-
}
773-
774-
script := filepath.Join(tmpDir, name)
775-
content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
776-
if err := ioutil.WriteFile(script, []byte(content), 0700); err != nil {
777-
return err
778-
}
779-
}
780-
return nil
731+
return warnings, nil
781732
}
782733

783734
// verifyDaemonSettings performs validation of daemon config struct
@@ -808,14 +759,15 @@ func verifyDaemonSettings(conf *config.Config) error {
808759
return fmt.Errorf("exec-opt native.cgroupdriver=systemd requires cgroup v2 for rootless mode")
809760
}
810761

811-
if conf.DefaultRuntime == "" {
812-
conf.DefaultRuntime = config.StockRuntimeName
813-
}
814-
if conf.Runtimes == nil {
815-
conf.Runtimes = make(map[string]types.Runtime)
762+
configureRuntimes(conf)
763+
if rtName := conf.GetDefaultRuntimeName(); rtName != "" {
764+
if conf.GetRuntime(rtName) == nil {
765+
return fmt.Errorf("specified default runtime '%s' does not exist", rtName)
766+
}
767+
if rtName == config.LinuxV1RuntimeName {
768+
logrus.Warnf("Configured default runtime %q is deprecated and will be removed in the next release.", config.LinuxV1RuntimeName)
769+
}
816770
}
817-
conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeName}
818-
819771
return nil
820772
}
821773

@@ -1756,10 +1708,6 @@ func (daemon *Daemon) setupSeccompProfile() error {
17561708
return nil
17571709
}
17581710

1759-
func (daemon *Daemon) useShimV2() bool {
1760-
return cgroups.IsCgroup2UnifiedMode()
1761-
}
1762-
17631711
// RawSysInfo returns *sysinfo.SysInfo .
17641712
func (daemon *Daemon) RawSysInfo(quiet bool) *sysinfo.SysInfo {
17651713
var opts []sysinfo.Opt

daemon/info.go

+5
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/docker/docker/api"
1212
"github.com/docker/docker/api/types"
1313
"github.com/docker/docker/cli/debug"
14+
"github.com/docker/docker/daemon/config"
1415
"github.com/docker/docker/daemon/logger"
1516
"github.com/docker/docker/dockerversion"
1617
"github.com/docker/docker/pkg/fileutils"
@@ -78,6 +79,10 @@ func (daemon *Daemon) SystemInfo() *types.Info {
7879
daemon.fillSecurityOptions(v, sysInfo)
7980
daemon.fillLicense(v)
8081

82+
if v.DefaultRuntime == config.LinuxV1RuntimeName {
83+
v.Warnings = append(v.Warnings, fmt.Sprintf("Configured default runtime %q is deprecated and will be removed in the next release.", config.LinuxV1RuntimeName))
84+
}
85+
8186
return v
8287
}
8388

daemon/runtime_unix.go

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
// +build !windows
2+
3+
package daemon
4+
5+
import (
6+
"fmt"
7+
"io/ioutil"
8+
"os"
9+
"os/exec"
10+
"path/filepath"
11+
"strings"
12+
13+
"github.com/containerd/containerd/runtime/linux/runctypes"
14+
v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
15+
"github.com/docker/docker/api/types"
16+
"github.com/docker/docker/daemon/config"
17+
"github.com/docker/docker/pkg/ioutils"
18+
"github.com/opencontainers/runc/libcontainer/cgroups"
19+
"github.com/pkg/errors"
20+
"github.com/sirupsen/logrus"
21+
)
22+
23+
const (
24+
defaultRuntimeName = "runc"
25+
26+
linuxShimV1 = "io.containerd.runtime.v1.linux"
27+
linuxShimV2 = "io.containerd.runc.v2"
28+
)
29+
30+
func configureRuntimes(conf *config.Config) {
31+
if conf.DefaultRuntime == "" {
32+
conf.DefaultRuntime = config.StockRuntimeName
33+
}
34+
if conf.Runtimes == nil {
35+
conf.Runtimes = make(map[string]types.Runtime)
36+
}
37+
conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: defaultRuntimeName, Shim: getShimConfig(conf, defaultRuntimeName)}
38+
conf.Runtimes[config.LinuxV1RuntimeName] = types.Runtime{Path: defaultRuntimeName, Shim: defaultV1ShimConfig(conf, defaultRuntimeName)}
39+
conf.Runtimes[config.LinuxV2RuntimeName] = types.Runtime{Path: defaultRuntimeName, Shim: defaultV2ShimConfig(conf, defaultRuntimeName)}
40+
}
41+
42+
func getShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
43+
if cgroups.IsCgroup2UnifiedMode() {
44+
return defaultV2ShimConfig(conf, runtimePath)
45+
}
46+
return defaultV1ShimConfig(conf, runtimePath)
47+
}
48+
49+
func defaultV2ShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
50+
return &types.ShimConfig{
51+
Binary: linuxShimV2,
52+
Opts: &v2runcoptions.Options{
53+
BinaryName: runtimePath,
54+
Root: filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName),
55+
SystemdCgroup: UsingSystemd(conf),
56+
NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
57+
},
58+
}
59+
}
60+
61+
func defaultV1ShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
62+
return &types.ShimConfig{
63+
Binary: linuxShimV1,
64+
Opts: &runctypes.RuncOptions{
65+
Runtime: runtimePath,
66+
RuntimeRoot: filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName),
67+
SystemdCgroup: UsingSystemd(conf),
68+
},
69+
}
70+
}
71+
72+
func (daemon *Daemon) loadRuntimes() error {
73+
return daemon.initRuntimes(daemon.configStore.Runtimes)
74+
}
75+
76+
func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error) {
77+
runtimeDir := filepath.Join(daemon.configStore.Root, "runtimes")
78+
// Remove old temp directory if any
79+
os.RemoveAll(runtimeDir + "-old")
80+
tmpDir, err := ioutils.TempDir(daemon.configStore.Root, "gen-runtimes")
81+
if err != nil {
82+
return errors.Wrap(err, "failed to get temp dir to generate runtime scripts")
83+
}
84+
defer func() {
85+
if err != nil {
86+
if err1 := os.RemoveAll(tmpDir); err1 != nil {
87+
logrus.WithError(err1).WithField("dir", tmpDir).
88+
Warn("failed to remove tmp dir")
89+
}
90+
return
91+
}
92+
93+
if err = os.Rename(runtimeDir, runtimeDir+"-old"); err != nil {
94+
return
95+
}
96+
if err = os.Rename(tmpDir, runtimeDir); err != nil {
97+
err = errors.Wrap(err, "failed to setup runtimes dir, new containers may not start")
98+
return
99+
}
100+
if err = os.RemoveAll(runtimeDir + "-old"); err != nil {
101+
logrus.WithError(err).WithField("dir", tmpDir).
102+
Warn("failed to remove old runtimes dir")
103+
}
104+
}()
105+
106+
for name, rt := range runtimes {
107+
if len(rt.Args) == 0 {
108+
continue
109+
}
110+
111+
script := filepath.Join(tmpDir, name)
112+
content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
113+
if err := ioutil.WriteFile(script, []byte(content), 0700); err != nil {
114+
return err
115+
}
116+
}
117+
return nil
118+
}
119+
120+
// rewriteRuntimePath is used for runtimes which have custom arguments supplied.
121+
// This is needed because the containerd API only calls the OCI runtime binary, there is no options for extra arguments.
122+
// To support this case, the daemon wraps the specified runtime in a script that passes through those arguments.
123+
func (daemon *Daemon) rewriteRuntimePath(name, p string, args []string) (string, error) {
124+
if len(args) == 0 {
125+
return p, nil
126+
}
127+
128+
// Check that the runtime path actually exists here so that we can return a well known error.
129+
if _, err := exec.LookPath(p); err != nil {
130+
return "", errors.Wrap(err, "error while looking up the specified runtime path")
131+
}
132+
133+
return filepath.Join(daemon.configStore.Root, "runtimes", name), nil
134+
}

daemon/start.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
175175
}
176176
}
177177

178-
createOptions, err := daemon.getLibcontainerdCreateOptions(container)
178+
shim, createOptions, err := daemon.getLibcontainerdCreateOptions(container)
179179
if err != nil {
180180
return err
181181
}
@@ -187,7 +187,7 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
187187
return err
188188
}
189189

190-
err = daemon.containerd.Create(ctx, container.ID, spec, createOptions, withImageName(imageRef.String()))
190+
err = daemon.containerd.Create(ctx, container.ID, spec, shim, createOptions, withImageName(imageRef.String()))
191191
if err != nil {
192192
if errdefs.IsConflict(err) {
193193
logrus.WithError(err).WithField("container", container.ID).Error("Container not cleaned up from containerd from previous run")
@@ -196,7 +196,7 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
196196
if err := daemon.containerd.Delete(ctx, container.ID); err != nil && !errdefs.IsNotFound(err) {
197197
logrus.WithError(err).WithField("container", container.ID).Error("Error cleaning up stale containerd container object")
198198
}
199-
err = daemon.containerd.Create(ctx, container.ID, spec, createOptions, withImageName(imageRef.String()))
199+
err = daemon.containerd.Create(ctx, container.ID, spec, shim, createOptions, withImageName(imageRef.String()))
200200
}
201201
if err != nil {
202202
return translateContainerdStartErr(container.Path, container.SetExitCode, err)

0 commit comments

Comments
 (0)