|
| 1 | +/** |
| 2 | +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | +**/ |
| 16 | + |
| 17 | +package cudacompat |
| 18 | + |
| 19 | +import ( |
| 20 | + "fmt" |
| 21 | + "os" |
| 22 | + "path/filepath" |
| 23 | + "strings" |
| 24 | + |
| 25 | + "github.com/moby/sys/symlink" |
| 26 | + "github.com/urfave/cli/v2" |
| 27 | + |
| 28 | + "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" |
| 29 | + "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" |
| 30 | +) |
| 31 | + |
| 32 | +const ( |
| 33 | + cudaCompatPath = "/usr/local/cuda/compat" |
| 34 | +) |
| 35 | + |
| 36 | +type command struct { |
| 37 | + logger logger.Interface |
| 38 | +} |
| 39 | + |
| 40 | +type options struct { |
| 41 | + driverVersion string |
| 42 | + containerSpec string |
| 43 | +} |
| 44 | + |
| 45 | +// NewCommand constructs an cuda-compat command with the specified logger |
| 46 | +func NewCommand(logger logger.Interface) *cli.Command { |
| 47 | + c := command{ |
| 48 | + logger: logger, |
| 49 | + } |
| 50 | + return c.build() |
| 51 | +} |
| 52 | + |
| 53 | +// build the cuda-compat command |
| 54 | +func (m command) build() *cli.Command { |
| 55 | + cfg := options{} |
| 56 | + |
| 57 | + // Create the 'cuda-compat' command |
| 58 | + c := cli.Command{ |
| 59 | + Name: "cuda-compat", |
| 60 | + Usage: "This hook ensures that the folder containing the CUDA compat libraries is added to the ldconfig search path if required.", |
| 61 | + Before: func(c *cli.Context) error { |
| 62 | + return m.validateFlags(c, &cfg) |
| 63 | + }, |
| 64 | + Action: func(c *cli.Context) error { |
| 65 | + return m.run(c, &cfg) |
| 66 | + }, |
| 67 | + } |
| 68 | + |
| 69 | + c.Flags = []cli.Flag{ |
| 70 | + &cli.StringFlag{ |
| 71 | + Name: "driver-version", |
| 72 | + Usage: "Specify the host driver version. If the CUDA compat libraries detected in the container do not have a higher MAJOR version, the hook is a no-op.", |
| 73 | + Destination: &cfg.driverVersion, |
| 74 | + }, |
| 75 | + &cli.StringFlag{ |
| 76 | + Name: "container-spec", |
| 77 | + Hidden: true, |
| 78 | + Category: "testing-only", |
| 79 | + Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN", |
| 80 | + Destination: &cfg.containerSpec, |
| 81 | + }, |
| 82 | + } |
| 83 | + |
| 84 | + return &c |
| 85 | +} |
| 86 | + |
| 87 | +func (m command) validateFlags(c *cli.Context, cfg *options) error { |
| 88 | + return nil |
| 89 | +} |
| 90 | + |
| 91 | +func (m command) run(c *cli.Context, cfg *options) error { |
| 92 | + s, err := oci.LoadContainerState(cfg.containerSpec) |
| 93 | + if err != nil { |
| 94 | + return fmt.Errorf("failed to load container state: %w", err) |
| 95 | + } |
| 96 | + |
| 97 | + containerRoot, err := s.GetContainerRoot() |
| 98 | + if err != nil { |
| 99 | + return fmt.Errorf("failed to determined container root: %w", err) |
| 100 | + } |
| 101 | + |
| 102 | + return m.updateCUDACompatLibs(root(containerRoot), cfg) |
| 103 | +} |
| 104 | + |
| 105 | +func (m command) updateCUDACompatLibs(containerRoot root, cfg *options) error { |
| 106 | + if !containerRoot.hasPath(cudaCompatPath) { |
| 107 | + return nil |
| 108 | + } |
| 109 | + |
| 110 | + if !containerRoot.hasPath("/etc/ld.so.cache") { |
| 111 | + // If there is no ldcache in the container, the hook is a no-op. |
| 112 | + return nil |
| 113 | + } |
| 114 | + if !containerRoot.hasPath("/etc/ld.so.conf.d") { |
| 115 | + // If the /etc/ld.so.conf.d folder does not exist in the container, the hook is a no-op. |
| 116 | + return nil |
| 117 | + } |
| 118 | + |
| 119 | + libs, err := containerRoot.glob(filepath.Join(cudaCompatPath, "libcuda.so.*.*")) |
| 120 | + if err != nil { |
| 121 | + m.logger.Warningf("Failed to find CUDA compat library: %w", err) |
| 122 | + return nil |
| 123 | + } |
| 124 | + |
| 125 | + if len(libs) == 0 { |
| 126 | + return nil |
| 127 | + } |
| 128 | + |
| 129 | + if len(libs) != 1 { |
| 130 | + m.logger.Warningf("Unexpected number of CUDA compat libraries: %v", libs) |
| 131 | + return nil |
| 132 | + } |
| 133 | + |
| 134 | + compatVersion := strings.TrimPrefix(filepath.Base(libs[0]), "libcuda.so.") |
| 135 | + compatMajor := strings.SplitN(compatVersion, ".", 2)[0] |
| 136 | + |
| 137 | + driverVersion := cfg.driverVersion |
| 138 | + driverMajor := strings.SplitN(driverVersion, ".", 2)[0] |
| 139 | + |
| 140 | + if driverMajor < compatMajor { |
| 141 | + return m.createConfig(string(containerRoot), []string{cudaCompatPath}) |
| 142 | + } |
| 143 | + return nil |
| 144 | +} |
| 145 | + |
| 146 | +// A root is used to add basic path functionality to a string. |
| 147 | +type root string |
| 148 | + |
| 149 | +// hasPath checks whether the specified path exists in the root. |
| 150 | +func (r root) hasPath(path string) bool { |
| 151 | + resolved, err := r.resolve(path) |
| 152 | + if err != nil { |
| 153 | + return false |
| 154 | + } |
| 155 | + if _, err := os.Stat(resolved); err != nil && os.IsNotExist(err) { |
| 156 | + return false |
| 157 | + } |
| 158 | + return true |
| 159 | +} |
| 160 | + |
| 161 | +// glob matches the specified pattern in the root. |
| 162 | +func (r root) glob(pattern string) ([]string, error) { |
| 163 | + patternPath, err := r.resolve(pattern) |
| 164 | + if err != nil { |
| 165 | + return nil, err |
| 166 | + } |
| 167 | + return filepath.Glob(patternPath) |
| 168 | +} |
| 169 | + |
| 170 | +// resolve returns the absolute path including root path. |
| 171 | +// Symlinks are resolved, but are guaranteed to resolve in the root. |
| 172 | +func (r root) resolve(path string) (string, error) { |
| 173 | + absolute := filepath.Clean(filepath.Join(string(r), path)) |
| 174 | + return symlink.FollowSymlinkInScope(absolute, string(r)) |
| 175 | +} |
| 176 | + |
| 177 | +// createConfig creates (or updates) /etc/ld.so.conf.d/00-compat-<RANDOM_STRING>.conf in the container |
| 178 | +// to include the required paths. |
| 179 | +// Note that the 00-compat prefix is chosen to ensure that these libraries have |
| 180 | +// a higher precedence than other libraries on the system. |
| 181 | +func (m command) createConfig(root string, folders []string) error { |
| 182 | + if len(folders) == 0 { |
| 183 | + m.logger.Debugf("No folders to add to /etc/ld.so.conf") |
| 184 | + return nil |
| 185 | + } |
| 186 | + |
| 187 | + if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil { |
| 188 | + return fmt.Errorf("failed to create ld.so.conf.d: %w", err) |
| 189 | + } |
| 190 | + |
| 191 | + configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "00-compat-*.conf") |
| 192 | + if err != nil { |
| 193 | + return fmt.Errorf("failed to create config file: %w", err) |
| 194 | + } |
| 195 | + defer configFile.Close() |
| 196 | + |
| 197 | + m.logger.Debugf("Adding folders %v to %v", folders, configFile.Name()) |
| 198 | + |
| 199 | + configured := make(map[string]bool) |
| 200 | + for _, folder := range folders { |
| 201 | + if configured[folder] { |
| 202 | + continue |
| 203 | + } |
| 204 | + _, err = configFile.WriteString(fmt.Sprintf("%s\n", folder)) |
| 205 | + if err != nil { |
| 206 | + return fmt.Errorf("failed to update ld.so.conf.d: %w", err) |
| 207 | + } |
| 208 | + configured[folder] = true |
| 209 | + } |
| 210 | + |
| 211 | + // The created file needs to be world readable for the cases where the container is run as a non-root user. |
| 212 | + if err := os.Chmod(configFile.Name(), 0644); err != nil { |
| 213 | + return fmt.Errorf("failed to chmod config file: %w", err) |
| 214 | + } |
| 215 | + |
| 216 | + return nil |
| 217 | +} |
0 commit comments