Skip to content

Commit bcd738c

Browse files
vishesh92vishesh92
authored
Fix GPU discovery script to make it run with mdev for SR-IOV enabled devices (#11340)
1 parent e805e45 commit bcd738c

File tree

5 files changed

+83
-63
lines changed

5 files changed

+83
-63
lines changed

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ private void generateMdevXml(StringBuilder gpuBuilder) {
4949
String mdevUuid = vgpuType.getBusAddress(); // For MDEV devices, busAddress contains the UUID
5050
String displayAttribute = vgpuType.isDisplay() ? "on" : "off";
5151

52-
gpuBuilder.append("<hostdev mode='subsystem' type='mdev' managed='no' display='").append(displayAttribute).append("'>\n");
52+
gpuBuilder.append("<hostdev mode='subsystem' type='mdev' model='vfio-pci' display='").append(displayAttribute).append("'>\n");
5353
gpuBuilder.append(" <source>\n");
5454
gpuBuilder.append(" <address uuid='").append(mdevUuid).append("'/>\n");
5555
gpuBuilder.append(" </source>\n");

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,9 @@ public void testGpuDef_withMdevDevice() {
6464

6565
String gpuXml = gpuDef.toString();
6666

67-
assertTrue(gpuXml.contains("<hostdev mode='subsystem' type='mdev' managed='no' display='off'>"));
67+
assertTrue(gpuXml.contains("<hostdev mode='subsystem' type='mdev' model='vfio-pci' display='off'>"));
6868
assertTrue(gpuXml.contains("<address uuid='4b20d080-1b54-4048-85b3-a6a62d165c01'/>"));
6969
assertTrue(gpuXml.contains("</hostdev>"));
70-
assertFalse(gpuXml.contains("vfio")); // MDEV should not contain vfio driver element
7170
}
7271

7372
@Test

‎scripts/vm/hypervisor/kvm/gpudiscovery.sh

Lines changed: 66 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ for VM in "${VMS[@]}"; do
473473
# -- MDEV hostdevs: use xmlstarlet to extract UUIDs --
474474
while IFS= read -r UUID; do
475475
[[ -n "$UUID" ]] && mdev_to_vm["$UUID"]="$VM"
476-
done < <(echo "$xml" | xmlstarlet sel -T -t -m "//hostdev[@type='mdev']" -v "@uuid" -n 2>/dev/null || true)
476+
done < <(echo "$xml" | xmlstarlet sel -T -t -m "//hostdev[@type='mdev']/source/address" -v "@uuid" -n 2>/dev/null || true)
477477
done
478478

479479
# Helper: convert a VM name to JSON value (quoted string or null)
@@ -516,6 +516,55 @@ parse_and_add_gpu_properties() {
516516
fi
517517
}
518518

519+
# Finds and formats mdev instances for a given PCI device (PF or VF).
520+
# Appends JSON strings for each found mdev instance to the global 'vlist' array.
521+
# Arguments:
522+
# $1: mdev_base_path (e.g., /sys/bus/pci/devices/.../mdev_supported_types)
523+
# $2: bdf (e.g., 01:00.0)
524+
process_mdev_instances() {
525+
local mdev_base_path="$1"
526+
local bdf="$2"
527+
528+
if [[ ! -d "$mdev_base_path" ]]; then
529+
return
530+
fi
531+
532+
for PROF_DIR in "$mdev_base_path"/*; do
533+
[[ -d "$PROF_DIR" ]] || continue
534+
535+
local PROFILE_NAME
536+
if [[ -f "$PROF_DIR/name" ]]; then
537+
PROFILE_NAME=$(<"$PROF_DIR/name")
538+
else
539+
PROFILE_NAME=$(basename "$PROF_DIR")
540+
fi
541+
542+
parse_and_add_gpu_properties "$PROF_DIR/description"
543+
544+
local DEVICE_DIR="$PROF_DIR/devices"
545+
if [[ -d "$DEVICE_DIR" ]]; then
546+
for UDIR in "$DEVICE_DIR"/*; do
547+
[[ -d "$UDIR" ]] || continue
548+
local MDEV_UUID
549+
MDEV_UUID=$(basename "$UDIR")
550+
551+
local DOMAIN="0x0000"
552+
local BUS="0x${bdf:0:2}"
553+
local SLOT="0x${bdf:3:2}"
554+
local FUNC="0x${bdf:6:1}"
555+
556+
local raw
557+
raw="${mdev_to_vm[$MDEV_UUID]:-}"
558+
local USED_JSON
559+
USED_JSON=$(to_json_vm "$raw")
560+
561+
vlist+=(
562+
"{\"mdev_uuid\":\"$MDEV_UUID\",\"profile_name\":$(json_escape "$PROFILE_NAME"),\"max_instances\":$MAX_INSTANCES,\"video_ram\":$VIDEO_RAM,\"max_heads\":$MAX_HEADS,\"max_resolution_x\":$MAX_RESOLUTION_X,\"max_resolution_y\":$MAX_RESOLUTION_Y,\"libvirt_address\":{\"domain\":\"$DOMAIN\",\"bus\":\"$BUS\",\"slot\":\"$SLOT\",\"function\":\"$FUNC\"},\"used_by_vm\":$USED_JSON}")
563+
done
564+
fi
565+
done
566+
}
567+
519568
# === GPU Discovery ===
520569

521570
mapfile -t LINES < <(lspci -nnm)
@@ -588,51 +637,9 @@ for LINE in "${LINES[@]}"; do
588637
# === vGPU (MDEV) instances ===
589638
VGPU_ARRAY="[]"
590639
declare -a vlist=()
640+
# Process mdev on the Physical Function
591641
MDEV_BASE="/sys/bus/pci/devices/0000:$PCI_ADDR/mdev_supported_types"
592-
if [[ -d "$MDEV_BASE" ]]; then
593-
for PROF_DIR in "$MDEV_BASE"/*; do
594-
[[ -d "$PROF_DIR" ]] || continue
595-
596-
# Read the human-readable profile name from the 'name' file
597-
if [[ -f "$PROF_DIR/name" ]]; then
598-
PROFILE_NAME=$(<"$PROF_DIR/name")
599-
else
600-
PROFILE_NAME=$(basename "$PROF_DIR")
601-
fi
602-
603-
# Fetch max_instance from the description file, if present
604-
parse_and_add_gpu_properties "$PROF_DIR/description"
605-
606-
# Under each profile, existing UUIDs appear in:
607-
# /sys/bus/pci/devices/0000:$PCI_ADDR/mdev_supported_types/<PROFILE>/devices/*
608-
DEVICE_DIR="$PROF_DIR/devices"
609-
if [[ -d "$DEVICE_DIR" ]]; then
610-
for UDIR in "$DEVICE_DIR"/*; do
611-
[[ -d $UDIR ]] || continue
612-
MDEV_UUID=$(basename "$UDIR")
613-
614-
# libvirt_address uses PF BDF
615-
DOMAIN="0x0000"
616-
BUS="0x${PCI_ADDR:0:2}"
617-
SLOT="0x${PCI_ADDR:3:2}"
618-
FUNC="0x${PCI_ADDR:6:1}"
619-
620-
# Determine which VM uses this UUID
621-
raw="${mdev_to_vm[$MDEV_UUID]:-}"
622-
USED_JSON=$(to_json_vm "$raw")
623-
624-
vlist+=(
625-
"{\"mdev_uuid\":\"$MDEV_UUID\",\"profile_name\":$(json_escape "$PROFILE_NAME"),\"max_instances\":$MAX_INSTANCES,\"video_ram\":$VIDEO_RAM,\"max_heads\":$MAX_HEADS,\"max_resolution_x\":$MAX_RESOLUTION_X,\"max_resolution_y\":$MAX_RESOLUTION_Y,\"libvirt_address\":{\"domain\":\"$DOMAIN\",\"bus\":\"$BUS\",\"slot\":\"$SLOT\",\"function\":\"$FUNC\"},\"used_by_vm\":$USED_JSON}")
626-
done
627-
fi
628-
done
629-
if [ ${#vlist[@]} -gt 0 ]; then
630-
VGPU_ARRAY="[$(
631-
IFS=,
632-
echo "${vlist[*]}"
633-
)]"
634-
fi
635-
fi
642+
process_mdev_instances "$MDEV_BASE" "$PCI_ADDR"
636643

637644
# === VF instances (SR-IOV / MIG) ===
638645
VF_ARRAY="[]"
@@ -644,6 +651,12 @@ for LINE in "${LINES[@]}"; do
644651
VF_ADDR=${VF_PATH##*/} # e.g. "0000:65:00.2"
645652
VF_BDF="${VF_ADDR:5}" # "65:00.2"
646653

654+
# For NVIDIA SR-IOV, check for vGPU (mdev) on the VF itself
655+
if [[ "$VENDOR_ID" == "10de" ]]; then
656+
VF_MDEV_BASE="$VF_PATH/mdev_supported_types"
657+
process_mdev_instances "$VF_MDEV_BASE" "$VF_BDF"
658+
fi
659+
647660
DOMAIN="0x0000"
648661
BUS="0x${VF_BDF:0:2}"
649662
SLOT="0x${VF_BDF:3:2}"
@@ -674,6 +687,14 @@ for LINE in "${LINES[@]}"; do
674687
fi
675688
fi
676689

690+
# Consolidate all vGPU instances (from PF and VFs)
691+
if [ ${#vlist[@]} -gt 0 ]; then
692+
VGPU_ARRAY="[$(
693+
IFS=,
694+
echo "${vlist[*]}"
695+
)]"
696+
fi
697+
677698
# === full_passthrough block ===
678699
# If vgpu_instances and vf_instances are empty, we can assume full passthrough
679700
FP_ENABLED=0

‎server/src/main/java/org/apache/cloudstack/gpu/GpuServiceImpl.java

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -795,7 +795,6 @@ public GPUDeviceTO doInTransaction(TransactionStatus status) {
795795
vgpuProfile.getName(), gpuDevice.getBusAddress(), gpuCard.getVendorId(),
796796
gpuCard.getVendorName(), gpuCard.getDeviceId(), gpuCard.getDeviceName());
797797
vgpuInfo.setDisplay(serviceOffering.getGpuDisplay());
798-
799798
if (gpuDevice.getParentGpuDeviceId() != null) {
800799
GpuDeviceVO parentGpuDevice = gpuDeviceDao.findById(gpuDevice.getParentGpuDeviceId());
801800
if (parentGpuDevice != null) {
@@ -891,14 +890,20 @@ public void addGpuDevicesToHost(final Host host, final List<VgpuTypesInfo> newGp
891890
} else {
892891
// Update the device's info
893892
GpuDeviceVO parentGpuDevice = null;
894-
if (existingDevice.getParentGpuDeviceId() == null
895-
&& deviceInfo.getParentBusAddress() != null) {
893+
if (deviceInfo.getParentBusAddress() != null) {
896894
parentGpuDevice = gpuDeviceDao.findByHostIdAndBusAddress(host.getId(),
897895
deviceInfo.getParentBusAddress());
898896
if (parentGpuDevice != null) {
899897
existingDevice.setParentGpuDeviceId(parentGpuDevice.getId());
898+
parentGpuDevice.setType(GpuDevice.DeviceType.VGPUOnly);
899+
gpuDeviceDao.persist(parentGpuDevice);
900900
}
901901
}
902+
if (deviceInfo.isPassthroughEnabled()) {
903+
existingDevice.setType(deviceInfo.getDeviceType());
904+
} else {
905+
existingDevice.setType(GpuDevice.DeviceType.VGPUOnly);
906+
}
902907
if (existingDevice.getPciRoot() == null) {
903908
existingDevice.setPciRoot(deviceInfo.getPciRoot());
904909
}
@@ -913,7 +918,6 @@ public void addGpuDevicesToHost(final Host host, final List<VgpuTypesInfo> newGp
913918
for (final GpuDeviceVO device : gpuDevicesToDisableMap.values()) {
914919
logger.info("Disabling GPU device {} on host {} due to missing address in the new devices on the host.", device, host);
915920
device.setState(GpuDevice.State.Error);
916-
device.setManagedState(GpuDevice.ManagedState.Unmanaged);
917921
gpuDeviceDao.update(device.getId(), device);
918922
checkAndUpdateParentGpuDeviceState(device.getParentGpuDeviceId());
919923
}
@@ -1024,11 +1028,14 @@ private void createAndAddGpuDeviceToHost(VgpuTypesInfo deviceInfo, Host host, Gp
10241028
deviceInfo.getParentBusAddress());
10251029
if (parentGpuDevice != null) {
10261030
parentGpuDeviceId = parentGpuDevice.getId();
1031+
parentGpuDevice.setType(GpuDevice.DeviceType.VGPUOnly);
1032+
gpuDeviceDao.persist(parentGpuDevice);
10271033
}
10281034
}
10291035
GpuDeviceVO gpuDevice = new GpuDeviceVO(card.getId(), vgpuProfile.getId(), deviceInfo.getBusAddress(),
10301036
host.getId(), parentGpuDeviceId, deviceInfo.getNumaNode(), deviceInfo.getPciRoot());
10311037
gpuDevice.setHostId(host.getId());
1038+
gpuDevice.setType(deviceInfo.getDeviceType());
10321039
gpuDevice.setBusAddress(deviceInfo.getBusAddress());
10331040
gpuDevice.setCardId(card.getId());
10341041
setStateAndVmName(deviceInfo, gpuDevice, parentGpuDevice);

‎ui/src/components/view/GPUSummaryTab.vue

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -167,15 +167,7 @@ export default {
167167
Object.values(cardGroups).forEach(cardGroup => {
168168
const profileCount = Object.keys(cardGroup.profiles).length
169169
170-
// Filter devices for card summary calculation
171-
// Exclude passthrough profile devices from aggregates if there are multiple profiles
172-
let cardDevicesForSummary = cardGroup.devices
173-
if (profileCount > 1) {
174-
cardDevicesForSummary = cardGroup.devices.filter(device => !device.vgpuprofilename || device.vgpuprofilename.toLowerCase() !== 'passthrough'
175-
)
176-
}
177-
178-
const cardSummary = this.calculateSummary(cardDevicesForSummary)
170+
const cardSummary = this.calculateSummary(cardGroup.devices)
179171
const cardKey = `card-${cardGroup.gpucardname}`
180172
181173
const cardNode = {
@@ -192,7 +184,6 @@ export default {
192184
expandedKeys.push(cardKey)
193185
194186
cardNode.children = Object.values(cardGroup.profiles)
195-
.filter(profile => profile.vgpuprofilename.toLowerCase() !== 'passthrough')
196187
.map(profile => {
197188
const profileSummary = this.calculateSummary(profile.devices)
198189
return {
@@ -204,7 +195,6 @@ export default {
204195
}
205196
})
206197
}
207-
208198
summaryTree.push(cardNode)
209199
})
210200
@@ -222,6 +212,9 @@ export default {
222212
}
223213
224214
devices.forEach(device => {
215+
if (device.gpudevicetype === 'VGPUOnly') {
216+
return
217+
}
225218
summary.total++
226219
227220
if (device.virtualmachineid) {

0 commit comments

Comments
 (0)