Create .so symlinks for driver libraries in container

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

CHANGELOG.md

@@ -236,6 +236,7 @@
 
 ## v1.15.0
 
+* Add a hook to create `.so` symlinks for driver libraries in a container.
 * Remove `nvidia-container-runtime` and `nvidia-docker2` packages.
 * Use `XDG_DATA_DIRS` environment variable when locating config files such as graphics config files.
 * Add support for v0.7.0 Container Device Interface (CDI) specification.

cmd/nvidia-ctk/info/info.go

@@ -17,6 +17,10 @@
 package info
 
 import (
+	"context"
+	"debug/elf"
+	"fmt"
+
 	"github.com/urfave/cli/v3"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
@@ -40,7 +44,27 @@ func (m command) build() *cli.Command {
 	info := cli.Command{
 		Name:  "info",
 		Usage: "Provide information about the system",
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return run()
+		},
 	}
 
 	return &info
 }
+
+func run() error {
+	filename := "/usr/lib/x86_64-linux-gnu/libcuda.so.570.133.20"
+
+	lib, err := elf.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer lib.Close()
+
+	sonames, err := lib.DynString(elf.DT_SONAME)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("sonames=%v\n", sonames)
+	return nil
+}

internal/discover/symlinks.go

@@ -17,10 +17,22 @@
 package discover
 
 import (
+	"debug/elf"
 	"fmt"
 	"path/filepath"
+	"strings"
 )
 
+// TODO: move to the symlinks package
+type Symlink struct {
+	target string
+	link   string
+}
+
+func (s *Symlink) String() string {
+	return fmt.Sprintf("%s::%s", s.target, s.link)
+}
+
 type additionalSymlinks struct {
 	Discover
 	version     string
@@ -60,7 +72,15 @@ func (d *additionalSymlinks) Hooks() ([]Hook, error) {
 		}
 		processedPaths[mount.Path] = true
 
-		for _, link := range d.getLinksForMount(mount.Path) {
+		linksForMount := d.getLinksForMount(mount.Path)
+		soSymlinks, err := d.getDotSoSymlinks(mount.HostPath)
+		if err != nil {
+			// TODO: Log this error.
+			soSymlinks = nil
+		}
+		linksForMount = append(linksForMount, soSymlinks...)
+
+		for _, link := range linksForMount {
 			if processedLinks[link] {
 				continue
 			}
@@ -110,3 +130,47 @@ func (d additionalSymlinks) isDriverLibrary(libraryName string, filename string)
 	match, _ := filepath.Match(pattern, filename)
 	return match
 }
+
+func (d *additionalSymlinks) getDotSoSymlinks(libraryPath string) ([]string, error) {
+	libraryName := filepath.Base(libraryPath)
+	if !d.isDriverLibrary("*", libraryName) {
+		return nil, nil
+	}
+
+	var soSymlinks []string
+	lib, err := elf.Open(libraryPath)
+	if err != nil {
+		return nil, err
+	}
+	defer lib.Close()
+
+	sonames, err := lib.DynString(elf.DT_SONAME)
+	if err != nil {
+		return nil, err
+	}
+
+	var sonameLinkPath string
+	for _, soname := range sonames {
+		linkPath := filepath.Join(filepath.Dir(libraryPath), soname)
+		if sonameLinkPath == "" {
+			sonameLinkPath = linkPath
+		}
+		s := Symlink{
+			target: libraryName,
+			link:   linkPath,
+		}
+		soSymlinks = append(soSymlinks, s.String())
+	}
+
+	if sonameLinkPath != "" {
+		sonameLinkPathExt := filepath.Ext(sonameLinkPath)
+		soLinkPath := strings.TrimSuffix(sonameLinkPath, sonameLinkPathExt)
+		s := Symlink{
+			target: filepath.Base(sonameLinkPath),
+			link:   soLinkPath,
+		}
+		soSymlinks = append(soSymlinks, s.String())
+	}
+
+	return soSymlinks, nil
+}

internal/lookup/root/options.go

@@ -43,3 +43,9 @@ func WithConfigSearchPaths(paths ...string) Option {
 		d.configSearchPaths = paths
 	}
 }
+
+func WithVersion(version string) Option {
+	return func(d *Driver) {
+		d.version = version
+	}
+}

internal/lookup/root/root.go

@@ -17,23 +17,30 @@
 package root
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 // Driver represents a filesystem in which a set of drivers or devices is defined.
 type Driver struct {
+	sync.Mutex
 	logger logger.Interface
 	// Root represents the root from the perspective of the driver libraries and binaries.
 	Root string
 	// librarySearchPaths specifies explicit search paths for discovering libraries.
 	librarySearchPaths []string
 	// configSearchPaths specified explicit search paths for discovering driver config files.
 	configSearchPaths []string
+	// version stores the driver version. This can be specified at construction or cached on subsequent calls.
+	version string
+	// libraryRoot stores the absolute path where the driver libraries (libcuda.so.<VERSION>) can be found.
+	libraryRoot string
 }
 
 // New creates a new Driver root using the specified options.
@@ -103,6 +110,62 @@ func (r *Driver) configSearchOptions() []lookup.Option {
 	}
 }
 
+// Version returns the driver version as a string.
+func (r *Driver) Version() (string, error) {
+	r.Lock()
+	defer r.Unlock()
+	if r.version != "" {
+		return r.version, nil
+	}
+
+	libcudaPath, err := r.libcudaPath()
+	if err != nil {
+		return "", fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+
+	version := strings.TrimPrefix(filepath.Base(libcudaPath), "libcuda.so.")
+	if version == "" {
+		return "", fmt.Errorf("failed to determine libcuda.so version from path: %q", libcudaPath)
+	}
+
+	r.version = version
+	return r.version, nil
+}
+
+// LibraryRoot returns the folder in which the driver libraries can be found.
+func (r *Driver) LibraryRoot() (string, error) {
+	r.Lock()
+	defer r.Unlock()
+	if r.libraryRoot != "" {
+		return r.libraryRoot, nil
+	}
+
+	libcudaPath, err := r.libcudaPath()
+	if err != nil {
+		return "", fmt.Errorf("failed to locate libcuda.so: %v", err)
+	}
+
+	r.libraryRoot = filepath.Dir(libcudaPath)
+	return r.libraryRoot, nil
+}
+
+// libcudaPath returns the path to libcuda.so.*.* in the driver root.
+func (r *Driver) libcudaPath() (string, error) {
+	pattern := "libcuda.so.*.*"
+
+	locator := r.Libraries()
+	paths, err := locator.Locate(pattern)
+	if err != nil {
+		return "", fmt.Errorf("failed to locate %v: %v", pattern, err)
+	}
+
+	libcudaPath := paths[0]
+	if len(paths) > 1 {
+		r.logger.Warningf("Selecting %v out of multiple libcuda.so paths.", libcudaPath, paths)
+	}
+	return libcudaPath, nil
+}
+
 // normalizeSearchPaths takes a list of paths and normalized these.
 // Each of the elements in the list is expanded if it is a path list and the
 // resultant list is returned.

internal/runtime/runtime_factory_test.go

@@ -67,6 +67,7 @@ func TestFactoryMethod(t *testing.T) {
 	logger, _ := testlog.NewNullLogger()
 	driver := root.New(
 		root.WithDriverRoot("/nvidia/driver/root"),
+		root.WithVersion("999.88.77"),
 	)
 
 	testCases := []struct {

pkg/nvcdi/driver-nvml.go

@@ -32,7 +32,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root"
 )
 
-// NewDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
+// newDriverDiscoverer creates a discoverer for the libraries and binaries associated with a driver installation.
 // The supplied NVML Library is used to query the expected driver version.
 func (l *nvmllib) NewDriverDiscoverer() (discover.Discover, error) {
 	if r := l.nvmllib.Init(); r != nvml.SUCCESS {