Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions internal/backendclient/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,14 +80,16 @@ type GPUInfo struct {
}

type GPUDevice struct {
UUID string `json:"uuid"`
BusID string `json:"busID"`
SN string `json:"sn"`
MinorID string `json:"minorID"`
BoardID int `json:"boardID"`
VBIOSVersion string `json:"vbiosVersion"`
ChassisSN string `json:"chassisSN"`
GPUIndex string `json:"gpuIndex,omitempty"`
UUID string `json:"uuid"`
BusID string `json:"busID"`
ClusterUUID string `json:"clusterUUID,omitempty"`
CliqueID *uint32 `json:"cliqueID,omitempty"`
SN string `json:"sn"`
MinorID string `json:"minorID"`
BoardID int `json:"boardID"`
VBIOSVersion string `json:"vbiosVersion"`
ChassisSN string `json:"chassisSN"`
GPUIndex string `json:"gpuIndex,omitempty"`
}

type DiskInfo struct {
Expand Down
14 changes: 11 additions & 3 deletions internal/exporter/collector/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ type HealthData struct {
Timestamp time.Time
MachineInfo *machineinfo.MachineInfo
GPUUUIDToIndex map[string]string
EntityCatalog *EntityCatalog
Metrics pkgmetrics.Metrics
Events eventstore.Events
ComponentData map[string]interface{}
Expand Down Expand Up @@ -109,7 +110,8 @@ func New(
}

var provider machineInfoProvider
if cfg != nil && cfg.IncludeMachineInfo && collectorOpts.nvmlInstance != nil {
needsIdentity := cfg != nil && (cfg.IncludeMachineInfo || cfg.IncludeMetrics || cfg.IncludeEvents || cfg.IncludeComponentData)
if needsIdentity && collectorOpts.nvmlInstance != nil {
var machineInfoOpts []machineinfo.MachineInfoOption
if len(dcgmGPUIndexes) > 0 {
machineInfoOpts = append(machineInfoOpts, machineinfo.WithDCGMGPUIndexes(dcgmGPUIndexes))
Expand Down Expand Up @@ -145,11 +147,17 @@ func (c *collector) Collect(ctx context.Context) (*HealthData, error) {
MachineID: c.machineID,
Timestamp: time.Now().UTC(),
GPUUUIDToIndex: cloneStringMap(c.dcgmGPUIndexes),
EntityCatalog: NewEntityCatalog(nil, c.dcgmGPUIndexes),
}

// Collect machine info if enabled. The converter only exports selected fields.
if c.config.IncludeMachineInfo {
// Identity enrichment uses cached machine info independently of whether the
// full machine-info payload is enabled for export.
if c.machineInfoProvider != nil {
c.collectMachineInfo(ctx, data)
data.EntityCatalog = NewEntityCatalog(data.MachineInfo, c.dcgmGPUIndexes)
if !c.config.IncludeMachineInfo {
data.MachineInfo = nil
}
Comment thread
jingxiang-z marked this conversation as resolved.
}

// Collect metrics if enabled
Expand Down
124 changes: 124 additions & 0 deletions internal/exporter/collector/identity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
"strconv"
"strings"
"time"

"github.com/NVIDIA/fleet-intelligence-agent/internal/machineinfo"
)

// GPUIdentity contains stable, non-workload identity for one physical GPU.
type GPUIdentity struct {
UUID string
GPU string
PCIBusID string
Device string
ModelName string
Architecture string
GPUSerial string
VBIOSVersion string
ClusterUUID string
CliqueID string
}

// EntityCatalog is an immutable identity snapshot shared by metric and log conversion.
// MIG identities are intentionally outside this catalog.
type EntityCatalog struct {
Hostname string
GPUDriverVersion string
CUDADriverVersion string
KernelVersion string
BootTime time.Time
GPUsByUUID map[string]GPUIdentity
GPUUUIDByIndex map[string]string
}

// NewEntityCatalog builds a physical-entity identity snapshot from cached inventory.
func NewEntityCatalog(info *machineinfo.MachineInfo, dcgmGPUIndexes map[string]string) *EntityCatalog {
catalog := &EntityCatalog{
GPUsByUUID: make(map[string]GPUIdentity),
GPUUUIDByIndex: make(map[string]string),
}

for uuid, gpu := range dcgmGPUIndexes {
uuid = strings.TrimSpace(uuid)
gpu = strings.TrimSpace(gpu)
if uuid == "" {
continue
}
identity := GPUIdentity{UUID: uuid, GPU: gpu}
catalog.GPUsByUUID[uuid] = identity
if gpu != "" {
catalog.GPUUUIDByIndex[gpu] = uuid
}
}

if info == nil {
return catalog
}
catalog.Hostname = strings.TrimSpace(info.Hostname)
catalog.GPUDriverVersion = strings.TrimSpace(info.GPUDriverVersion)
catalog.CUDADriverVersion = strings.TrimSpace(info.CUDAVersion)
catalog.KernelVersion = strings.TrimSpace(info.KernelVersion)
if !info.Uptime.IsZero() {
catalog.BootTime = info.Uptime.UTC()
}
if info.GPUInfo == nil {
return catalog
}

defaultModelName := strings.TrimSpace(info.GPUInfo.Product)
defaultArchitecture := strings.TrimSpace(info.GPUInfo.Architecture)
for _, gpuInfo := range info.GPUInfo.GPUs {
uuid := strings.TrimSpace(gpuInfo.UUID)
if uuid == "" {
continue
}

identity := catalog.GPUsByUUID[uuid]
identity.UUID = uuid
if identity.GPU == "" {
identity.GPU = strings.TrimSpace(gpuInfo.GPUIndex)
}
identity.PCIBusID = strings.TrimSpace(gpuInfo.BusID)
identity.Architecture = defaultArchitecture
identity.GPUSerial = strings.TrimSpace(gpuInfo.SN)
identity.VBIOSVersion = strings.TrimSpace(gpuInfo.VBIOSVersion)
identity.ModelName = strings.TrimSpace(gpuInfo.ModelName)
if identity.ModelName == "" {
identity.ModelName = defaultModelName
}
identity.ClusterUUID = strings.TrimSpace(gpuInfo.ClusterUUID)
if gpuInfo.CliqueID != nil {
identity.CliqueID = strconv.FormatUint(uint64(*gpuInfo.CliqueID), 10)
}

minorID := strings.TrimSpace(gpuInfo.MinorID)
if minor, err := strconv.Atoi(minorID); err == nil && minor >= 0 {
identity.Device = "nvidia" + strconv.Itoa(minor)
}

catalog.GPUsByUUID[uuid] = identity
if identity.GPU != "" {
catalog.GPUUUIDByIndex[identity.GPU] = uuid
}
}

return catalog
}
88 changes: 88 additions & 0 deletions internal/exporter/collector/identity_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
"testing"
"time"

apiv1 "github.com/NVIDIA/fleet-intelligence-sdk/api/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/NVIDIA/fleet-intelligence-agent/internal/machineinfo"
)

func TestNewEntityCatalog(t *testing.T) {
cliqueID := uint32(0)
bootTime := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
info := &machineinfo.MachineInfo{
Hostname: "gpu-node-01",
GPUDriverVersion: "575.57.08",
CUDAVersion: "12.9",
KernelVersion: "6.14.0-27-generic",
Uptime: metav1.NewTime(bootTime),
GPUInfo: &apiv1.MachineGPUInfo{
Product: "fallback-model",
Architecture: "hopper",
GPUs: []apiv1.MachineGPUInstance{
{
UUID: "GPU-abc",
GPUIndex: "7",
BusID: "0000:01:00.0",
MinorID: "2",
ModelName: "NVIDIA H100",
SN: "GPU-SERIAL-123",
VBIOSVersion: "97.00.82.00.5F",
ClusterUUID: "11111111-2222-3333-4444-555555555555",
CliqueID: &cliqueID,
},
{
UUID: "GPU-def",
GPUIndex: "8",
MinorID: "-1",
},
},
},
}

catalog := NewEntityCatalog(info, map[string]string{"GPU-abc": "0"})
require.NotNil(t, catalog)
assert.Equal(t, "gpu-node-01", catalog.Hostname)
assert.Equal(t, "575.57.08", catalog.GPUDriverVersion)
assert.Equal(t, "12.9", catalog.CUDADriverVersion)
assert.Equal(t, "6.14.0-27-generic", catalog.KernelVersion)
assert.Equal(t, bootTime, catalog.BootTime)
assert.Equal(t, GPUIdentity{
UUID: "GPU-abc",
GPU: "0",
PCIBusID: "0000:01:00.0",
Device: "nvidia2",
ModelName: "NVIDIA H100",
Architecture: "hopper",
GPUSerial: "GPU-SERIAL-123",
VBIOSVersion: "97.00.82.00.5F",
ClusterUUID: "11111111-2222-3333-4444-555555555555",
CliqueID: "0",
}, catalog.GPUsByUUID["GPU-abc"])
assert.Equal(t, "fallback-model", catalog.GPUsByUUID["GPU-def"].ModelName)
assert.Equal(t, "hopper", catalog.GPUsByUUID["GPU-def"].Architecture)
assert.Empty(t, catalog.GPUsByUUID["GPU-def"].GPUSerial)
assert.Empty(t, catalog.GPUsByUUID["GPU-def"].Device)
assert.Equal(t, "GPU-abc", catalog.GPUUUIDByIndex["0"])
assert.Equal(t, "GPU-def", catalog.GPUUUIDByIndex["8"])
}
Loading
Loading