Fix containerd CRI plugin sandbox image

banzaicloud · Mar 21, 2019 · c4a2c02 · c4a2c02
1 parent ab1c6f1
commit c4a2c02
Show file tree

Hide file tree

Showing 10 changed files with 77 additions and 31 deletions.
diff --git a/cmd/pke/app/phases/kubeadm/controlplane/controlplane.go b/cmd/pke/app/phases/kubeadm/controlplane/controlplane.go
@@ -434,6 +434,7 @@ nodeRegistration:
   kubeletExtraArgs:
   {{if .Nodepool }}
     node-labels: "nodepool.banzaicloud.io/name={{ .Nodepool }}"{{end}}
+    # pod-infra-container-image: {{ .ImageRepository }}/pause:3.1 # only needed by docker
   {{if .CloudProvider }}
     cloud-provider: "{{ .CloudProvider }}"{{end}}
     read-only-port: "0"

diff --git a/cmd/pke/app/phases/kubeadm/node/node.go b/cmd/pke/app/phases/kubeadm/node/node.go
@@ -17,10 +17,10 @@ package node
 import (
 	"context"
 	"fmt"
-	"html/template"
 	"io"
 	"os"
 	"path/filepath"
+	"text/template"
 
 	"github.com/banzaicloud/pipeline/client"
 	"github.com/banzaicloud/pke/cmd/pke/app/constants"

diff --git a/cmd/pke/app/phases/runtime/container/container.go b/cmd/pke/app/phases/runtime/container/container.go
@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/banzaicloud/pke/cmd/pke/app/constants"
 	"github.com/banzaicloud/pke/cmd/pke/app/phases"
+	"github.com/banzaicloud/pke/cmd/pke/app/util/validator"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -30,7 +32,9 @@ const (
 
 var _ phases.Runnable = (*Runtime)(nil)
 
-type Runtime struct{}
+type Runtime struct {
+	imageRepository string
+}
 
 func NewCommand(out io.Writer) *cobra.Command {
 	return phases.NewCommand(out, &Runtime{})
@@ -44,14 +48,29 @@ func (r *Runtime) Short() string {
 	return short
 }
 
-func (r *Runtime) RegisterFlags(flags *pflag.FlagSet) {}
+func (r *Runtime) RegisterFlags(flags *pflag.FlagSet) {
+	// Image repository
+	flags.String(constants.FlagImageRepository, "banzaicloud", "Prefix for image repository")
+}
 
 func (r *Runtime) Validate(cmd *cobra.Command) error {
+	var err error
+	r.imageRepository, err = cmd.Flags().GetString(constants.FlagImageRepository)
+	if err != nil {
+		return err
+	}
+
+	if err := validator.NotEmpty(map[string]interface{}{
+		constants.FlagImageRepository: r.imageRepository,
+	}); err != nil {
+		return err
+	}
+
 	return nil
 }
 
 func (r *Runtime) Run(out io.Writer) error {
 	_, _ = fmt.Fprintf(out, "[RUNNING] %s\n", r.Use())
 
-	return installRuntime(out)
+	return r.installRuntime(out)
 }
diff --git a/cmd/pke/app/phases/runtime/container/containerd_linux.go b/cmd/pke/app/phases/runtime/container/containerd_linux.go
@@ -21,6 +21,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"text/template"
 
 	"github.com/banzaicloud/pke/cmd/pke/app/constants"
 	"github.com/banzaicloud/pke/cmd/pke/app/util/file"
@@ -33,31 +34,27 @@ const (
 	containerDSHA256      = "3391758c62d17a56807ddac98b05487d9e78e5beb614a0602caab747b0eda9e0"
 	containerDURL         = "https://storage.googleapis.com/cri-containerd-release/cri-containerd-%s.linux-amd64.tar.gz"
 	containerDVersionPath = "/opt/containerd/cluster/version"
+	containerDConf        = "/etc/containerd/config.toml"
 
 	criConfFile = "/etc/sysctl.d/99-kubernetes-cri.conf"
 	criConf     = `net.bridge.bridge-nf-call-iptables  = 1
 net.bridge.bridge-nf-call-ip6tables = 1
 net.ipv4.ip_forward                 = 1
-`
-
-	containerDConfFile = "/etc/systemd/system/kubelet.service.d/0-containerd.conf"
-	containerDConf     = `[Service]
-Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
 `
 )
 
-func installRuntime(out io.Writer) error {
+func (r *Runtime) installRuntime(out io.Writer) error {
 	if ver, err := linux.CentOSVersion(out); err == nil {
 		if ver == "7" {
-			return installCentOS7(out)
+			return installCentOS7(out, r.imageRepository)
 		}
 		return constants.ErrUnsupportedOS
 	}
 
 	return constants.ErrUnsupportedOS
 }
 
-func installCentOS7(out io.Writer) error {
+func installCentOS7(out io.Writer, imageRepository string) error {
 	// modprobe overlay
 	if err := linux.ModprobeOverlay(out); err != nil {
 		return errors.Wrap(err, "missing overlay Linux Kernel module")
@@ -90,7 +87,7 @@ func installCentOS7(out io.Writer) error {
 	_ = linux.SystemctlDisableAndStop(out, "containerd")
 
 	// Check ContainerD installed or not
-	if err := installContainerD(out); err != nil {
+	if err := installContainerD(out, imageRepository); err != nil {
 		return err
 	}
 
@@ -101,17 +98,6 @@ func installCentOS7(out io.Writer) error {
 
 	_ = linux.SystemctlDisableAndStop(out, "kubelet")
 
-	// cat > /etc/systemd/system/kubelet.service.d/0-containerd.conf <<EOF
-	// [Service]
-	// Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
-	// EOF
-	if err := os.MkdirAll(filepath.Dir(containerDConfFile), 0750); err != nil {
-		return err
-	}
-	if err := file.Overwrite(containerDConfFile, containerDConf); err != nil {
-		return err
-	}
-
 	// systemctl daemon-reload
 	if err := linux.SystemctlReload(out); err != nil {
 		return err
@@ -120,7 +106,7 @@ func installCentOS7(out io.Writer) error {
 	return nil
 }
 
-func installContainerD(out io.Writer) error {
+func installContainerD(out io.Writer, imageRepository string) error {
 	// Check ContainerD installed or not
 	if _, err := os.Stat(containerDVersionPath); !os.IsNotExist(err) {
 		// TODO: check ContainerD version
@@ -162,5 +148,40 @@ func installContainerD(out io.Writer) error {
 		return err
 	}
 
-	return nil
+	return writeContainerDConfig(out, containerDConf, imageRepository)
+}
+
+func writeContainerDConfig(out io.Writer, filename, imageRepository string) error {
+	dir := filepath.Dir(filename)
+
+	_, _ = fmt.Fprintf(out, "[%s] creating directory: %q\n", use, dir)
+	err := os.MkdirAll(dir, 0750)
+	if err != nil {
+		return err
+	}
+
+	conf := `[plugins.cri]
+  sandbox_image = "{{ .ImageRepository }}/pause:3.1"`
+
+	tmpl, err := template.New("containerd-config").Parse(conf)
+	if err != nil {
+		return err
+	}
+
+	// create and truncate write only file
+	w, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = w.Close() }()
+
+	type data struct {
+		ImageRepository string
+	}
+
+	d := data{
+		ImageRepository: imageRepository,
+	}
+
+	return tmpl.Execute(w, d)
 }
diff --git a/cmd/pke/app/phases/runtime/container/containerd_other.go b/cmd/pke/app/phases/runtime/container/containerd_other.go
@@ -22,6 +22,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-func installRuntime(w io.Writer) error {
+func (r *Runtime) installRuntime(w io.Writer) error {
 	return errors.Errorf("unsupported operating system")
 }
diff --git a/cmd/pke/docs/pke_install_master_container-runtime.md b/cmd/pke/docs/pke_install_master_container-runtime.md
@@ -13,7 +13,8 @@ pke install master container-runtime [flags]
 ### Options
 
 ```
-  -h, --help   help for container-runtime
+  -h, --help                      help for container-runtime
+      --image-repository string   Prefix for image repository (default "banzaicloud")
 ```
 
 ### SEE ALSO

diff --git a/cmd/pke/docs/pke_install_single_container-runtime.md b/cmd/pke/docs/pke_install_single_container-runtime.md
@@ -13,7 +13,8 @@ pke install single container-runtime [flags]
 ### Options
 
 ```
-  -h, --help   help for container-runtime
+  -h, --help                      help for container-runtime
+      --image-repository string   Prefix for image repository (default "banzaicloud")
 ```
 
 ### SEE ALSO

diff --git a/cmd/pke/docs/pke_install_worker.md b/cmd/pke/docs/pke_install_worker.md
@@ -14,6 +14,7 @@ pke install worker [flags]
 
 ```
   -h, --help                                        help for worker
+      --image-repository string                     Prefix for image repository (default "banzaicloud")
       --kubernetes-api-server string                Kubernetes API Server host port
       --kubernetes-api-server-ca-cert-hash string   CA cert hash
       --kubernetes-cloud-provider string            cloud provider. example: aws

diff --git a/cmd/pke/docs/pke_install_worker_container-runtime.md b/cmd/pke/docs/pke_install_worker_container-runtime.md
@@ -13,7 +13,8 @@ pke install worker container-runtime [flags]
 ### Options
 
 ```
-  -h, --help   help for container-runtime
+  -h, --help                      help for container-runtime
+      --image-repository string   Prefix for image repository (default "banzaicloud")
 ```
 
 ### SEE ALSO

diff --git a/cmd/pke/docs/pke_machine-image_container-runtime.md b/cmd/pke/docs/pke_machine-image_container-runtime.md
@@ -13,7 +13,8 @@ pke machine-image container-runtime [flags]
 ### Options
 
 ```
-  -h, --help   help for container-runtime
+  -h, --help                      help for container-runtime
+      --image-repository string   Prefix for image repository (default "banzaicloud")
 ```
 
 ### SEE ALSO