Load NVIDIA Kernel Modules for JIT-CDI mode #975
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,6 +25,7 @@ import ( | |
| "github.com/NVIDIA/nvidia-container-toolkit/internal/config" | ||
| "github.com/NVIDIA/nvidia-container-toolkit/internal/config/image" | ||
| "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" | ||
| "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root" | ||
| "github.com/NVIDIA/nvidia-container-toolkit/internal/modifier/cdi" | ||
| "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" | ||
| "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi" | ||
|
|
@@ -34,7 +35,7 @@ import ( | |
| // NewCDIModifier creates an OCI spec modifier that determines the modifications to make based on the | ||
| // CDI specifications available on the system. The NVIDIA_VISIBLE_DEVICES environment variable is | ||
| // used to select the devices to include. | ||
| func NewCDIModifier(logger logger.Interface, cfg *config.Config, driver *root.Driver, ociSpec oci.Spec) (oci.SpecModifier, error) { | ||
| devices, err := getDevicesFromSpec(logger, ociSpec, cfg) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get required devices from OCI specification: %v", err) | ||
|
|
@@ -50,7 +51,7 @@ func NewCDIModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spe | |
| return nil, fmt.Errorf("requesting a CDI device with vendor 'runtime.nvidia.com' is not supported when requesting other CDI devices") | ||
| } | ||
| if len(automaticDevices) > 0 { | ||
| automaticModifier, err := newAutomaticCDISpecModifier(logger, cfg, driver, automaticDevices) | ||
| if err == nil { | ||
| return automaticModifier, nil | ||
| } | ||
|
|
@@ -163,9 +164,9 @@ func filterAutomaticDevices(devices []string) []string { | |
| return automatic | ||
| } | ||
|
|
||
| func newAutomaticCDISpecModifier(logger logger.Interface, cfg *config.Config, driver *root.Driver, devices []string) (oci.SpecModifier, error) { | ||
| logger.Debugf("Generating in-memory CDI specs for devices %v", devices) | ||
| spec, err := generateAutomaticCDISpec(logger, cfg, driver, devices) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to generate CDI spec: %w", err) | ||
| } | ||
|
|
@@ -180,7 +181,7 @@ func newAutomaticCDISpecModifier(logger logger.Interface, cfg *config.Config, de | |
| return cdiModifier, nil | ||
| } | ||
|
|
||
| func generateAutomaticCDISpec(logger logger.Interface, cfg *config.Config, driver *root.Driver, devices []string) (spec.Interface, error) { | ||
| cdilib, err := nvcdi.New( | ||
| nvcdi.WithLogger(logger), | ||
| nvcdi.WithNVIDIACDIHookPath(cfg.NVIDIACTKConfig.Path), | ||
|
|
@@ -192,6 +193,11 @@ func generateAutomaticCDISpec(logger logger.Interface, cfg *config.Config, devic | |
| return nil, fmt.Errorf("failed to construct CDI library: %w", err) | ||
| } | ||
|
|
||
| // TODO: Consider moving this into the nvcdi API. | ||
| if err := driver.LoadKernelModules(cfg.NVIDIAContainerRuntimeConfig.Modes.JitCDI.LoadKernelModules...); err != nil { | ||
|
There was a problem hiding this comment. @klueska are there any cases where we DON'T want to load / try to load the kernel modules? Note that we aslo skip this when running in a user namespace in |
||
| logger.Warningf("Ignoring error(s) loading kernel modules: %v", err) | ||
| } | ||
|
|
||
| identifiers := []string{} | ||
| for _, device := range devices { | ||
| _, _, id := parser.ParseDevice(device) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -77,7 +77,7 @@ func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Sp | |
| mode := info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode, image) | ||
| // We update the mode here so that we can continue passing just the config to other functions. | ||
| cfg.NVIDIAContainerRuntimeConfig.Mode = mode | ||
| modeModifier, err := newModeModifier(logger, mode, cfg, driver, ociSpec, image) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -107,14 +107,14 @@ func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Sp | |
| return modifiers, nil | ||
| } | ||
|
|
||
| func newModeModifier(logger logger.Interface, mode string, cfg *config.Config, driver *root.Driver, ociSpec oci.Spec, image image.CUDA) (oci.SpecModifier, error) { | ||
| switch mode { | ||
| case "legacy": | ||
| return modifier.NewStableRuntimeModifier(logger, cfg.NVIDIAContainerRuntimeHookConfig.Path), nil | ||
| case "csv": | ||
| return modifier.NewCSVModifier(logger, cfg, image) | ||
| case "cdi": | ||
| return modifier.NewCDIModifier(logger, cfg, driver, ociSpec) | ||
| } | ||
|
|
||
| return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode) | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How did we end up making this fine selection? 🍷 🍇 🧀
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
These are the kernel modules that are required for various GPU functionalities. In the
nvidia-container-cliwe do this throughnvidia-modprobe:nvidia: https://github.com/NVIDIA/libnvidia-container/blob/95d3e86522976061e856724867ebcaf75c4e9b60/src/nvc.c#L279nvidia-uvm: https://github.com/NVIDIA/libnvidia-container/blob/95d3e86522976061e856724867ebcaf75c4e9b60/src/nvc.c#L305nvidia-modeset: https://github.com/NVIDIA/libnvidia-container/blob/95d3e86522976061e856724867ebcaf75c4e9b60/src/nvc.c#L314Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is actually a typo. The module names should be
nvidia,nvidia_uvmandnvidia_modeset.