Skip to content

Commit 4384b73

Browse files
harcheharche
authored
Merge pull request #25 from madeelibm/cherrypick-s390x-cpu-topology
Cherry-pick s390x cpu topology
2 parents 483a3b3 + 85f8d74 commit 4384b73

File tree

6 files changed

+197
-33
lines changed

6 files changed

+197
-33
lines changed

‎info/v1/machine.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ type Core struct {
5353
Caches []Cache `json:"caches"`
5454
UncoreCaches []Cache `json:"uncore_caches"`
5555
SocketID int `json:"socket_id"`
56+
BookID string `json:"book_id,omitempty"`
57+
DrawerID string `json:"drawer_id,omitempty"`
5658
}
5759

5860
type Cache struct {

‎machine/machine.go

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ import (
2121
"path"
2222
"regexp"
2323

24-
// s390/s390x changes
25-
"runtime"
2624
"strconv"
2725
"strings"
2826

@@ -223,10 +221,6 @@ func GetMachineSwapCapacity() (uint64, error) {
223221

224222
// GetTopology returns CPU topology reading information from sysfs
225223
func GetTopology(sysFs sysfs.SysFs) ([]info.Node, int, error) {
226-
// s390/s390x changes
227-
if isSystemZ() {
228-
return nil, getNumCores(), nil
229-
}
230224
return sysinfo.GetNodesInfo(sysFs)
231225
}
232226

@@ -290,15 +284,3 @@ func isRiscv64() bool {
290284
func isMips64() bool {
291285
return strings.Contains(machineArch, "mips64")
292286
}
293-
294-
// s390/s390x changes
295-
func getNumCores() int {
296-
maxProcs := runtime.GOMAXPROCS(0)
297-
numCPU := runtime.NumCPU()
298-
299-
if maxProcs < numCPU {
300-
return maxProcs
301-
}
302-
303-
return numCPU
304-
}

‎machine/topology_test.go

Lines changed: 108 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"os"
2020
"path/filepath"
2121
"reflect"
22+
"runtime"
2223
"sort"
2324
"testing"
2425

@@ -426,11 +427,113 @@ func TestTopologyWithNodesWithoutCPU(t *testing.T) {
426427
}
427428

428429
func TestTopologyOnSystemZ(t *testing.T) {
429-
machineArch = "s390" // overwrite package variable
430-
nodes, cores, err := GetTopology(&fakesysfs.FakeSysFs{})
431-
assert.Nil(t, err)
432-
assert.Nil(t, nodes)
433-
assert.NotNil(t, cores)
430+
if runtime.GOARCH != "s390x" {
431+
t.Skip("skipping TestTopologyOnSystemZ due to wrong architecture")
432+
} else {
433+
machineArch = "s390" // overwrite package variable
434+
sysFs := &fakesysfs.FakeSysFs{}
435+
436+
c := sysfs.CacheInfo{
437+
Id: 0,
438+
Size: 128 * 1024,
439+
Type: "Data",
440+
Level: 0,
441+
Cpus: 2,
442+
}
443+
sysFs.SetCacheInfo(c)
444+
445+
nodesPaths := []string{}
446+
sysFs.SetNodesPaths(nodesPaths, nil)
447+
448+
cpusPaths := map[string][]string{
449+
"/sys/devices/system/cpu": {
450+
"/sys/devices/system/cpu/cpu0",
451+
"/sys/devices/system/cpu/cpu1",
452+
"/sys/devices/system/cpu/cpu2",
453+
"/sys/devices/system/cpu/cpu3",
454+
},
455+
}
456+
sysFs.SetCPUsPaths(cpusPaths, nil)
457+
458+
coreThread := map[string]string{
459+
"/sys/devices/system/cpu/cpu0": "0",
460+
"/sys/devices/system/cpu/cpu1": "1",
461+
"/sys/devices/system/cpu/cpu2": "0",
462+
"/sys/devices/system/cpu/cpu3": "1",
463+
}
464+
sysFs.SetCoreThreads(coreThread, nil)
465+
466+
physicalPackageIDs := map[string]string{
467+
"/sys/devices/system/cpu/cpu0": "0",
468+
"/sys/devices/system/cpu/cpu1": "1",
469+
"/sys/devices/system/cpu/cpu2": "0",
470+
"/sys/devices/system/cpu/cpu3": "1",
471+
}
472+
sysFs.SetPhysicalPackageIDs(physicalPackageIDs, nil)
473+
474+
bookIDs := map[string]string{
475+
"/sys/devices/system/cpu/cpu0": "1",
476+
"/sys/devices/system/cpu/cpu1": "1",
477+
"/sys/devices/system/cpu/cpu2": "1",
478+
"/sys/devices/system/cpu/cpu3": "1",
479+
}
480+
sysFs.SetBookIDs(bookIDs, nil)
481+
482+
drawerIDs := map[string]string{
483+
"/sys/devices/system/cpu/cpu0": "0",
484+
"/sys/devices/system/cpu/cpu1": "0",
485+
"/sys/devices/system/cpu/cpu2": "0",
486+
"/sys/devices/system/cpu/cpu3": "0",
487+
}
488+
sysFs.SetDrawerIDs(drawerIDs, nil)
489+
490+
topology, numCores, err := GetTopology(sysFs)
491+
assert.Nil(t, err)
492+
assert.Equal(t, 2, len(topology))
493+
assert.Equal(t, 4, numCores)
494+
495+
topologyJSON1, err := json.Marshal(topology[0])
496+
assert.Nil(t, err)
497+
topologyJSON2, err := json.Marshal(topology[1])
498+
assert.Nil(t, err)
499+
500+
expectedTopology1 := `{"node_id":0,"memory":0,"hugepages":null,"distances":null,"cores":[{"core_id":0,"thread_ids":[0,2],"caches":[{"id":0, "size":131072,"type":"Data","level":0}], "socket_id": 0, "book_id":"1", "drawer_id":"0", "uncore_caches":null}],"caches":null}`
501+
expectedTopology2 := `
502+
{
503+
"node_id":1,
504+
"memory":0,
505+
"hugepages":null,
506+
"distances": null,
507+
"cores":[
508+
{
509+
"core_id":1,
510+
"thread_ids":[
511+
1,
512+
3
513+
],
514+
"caches":[
515+
{
516+
"id": 0,
517+
"size":131072,
518+
"type":"Data",
519+
"level":0
520+
}
521+
],
522+
"socket_id": 1,
523+
"book_id": "1",
524+
"drawer_id": "0",
525+
"uncore_caches": null
526+
}
527+
],
528+
"caches":null
529+
}`
530+
531+
json1 := string(topologyJSON1)
532+
json2 := string(topologyJSON2)
533+
534+
assert.JSONEq(t, expectedTopology1, json1)
535+
assert.JSONEq(t, expectedTopology2, json2)
536+
}
434537
}
435538

436539
func TestMemoryInfo(t *testing.T) {

‎utils/sysfs/fakesysfs/fake.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,12 @@ type FakeSysFs struct {
6767
physicalPackageIDs map[string]string
6868
physicalPackageIDErr map[string]error
6969

70+
bookIDs map[string]string
71+
bookIDErr map[string]error
72+
73+
drawerIDs map[string]string
74+
drawerIDErr map[string]error
75+
7076
memTotal string
7177
memErr error
7278

@@ -98,6 +104,14 @@ func (fs *FakeSysFs) GetCPUPhysicalPackageID(cpuPath string) (string, error) {
98104
return fs.physicalPackageIDs[cpuPath], fs.physicalPackageIDErr[cpuPath]
99105
}
100106

107+
func (fs *FakeSysFs) GetBookID(coreIDPath string) (string, error) {
108+
return fs.bookIDs[coreIDPath], fs.bookIDErr[coreIDPath]
109+
}
110+
111+
func (fs *FakeSysFs) GetDrawerID(coreIDPath string) (string, error) {
112+
return fs.drawerIDs[coreIDPath], fs.drawerIDErr[coreIDPath]
113+
}
114+
101115
func (fs *FakeSysFs) GetMemInfo(nodePath string) (string, error) {
102116
return fs.memTotal, fs.memErr
103117
}
@@ -185,6 +199,16 @@ func (fs *FakeSysFs) SetPhysicalPackageIDs(physicalPackageIDs map[string]string,
185199
fs.physicalPackageIDErr = physicalPackageIDErrors
186200
}
187201

202+
func (fs *FakeSysFs) SetBookIDs(bookIDs map[string]string, bookIDErrors map[string]error) {
203+
fs.bookIDs = bookIDs
204+
fs.bookIDErr = bookIDErrors
205+
}
206+
207+
func (fs *FakeSysFs) SetDrawerIDs(drawerIDs map[string]string, drawerIDErrors map[string]error) {
208+
fs.drawerIDs = drawerIDs
209+
fs.drawerIDErr = drawerIDErrors
210+
}
211+
188212
func (fs *FakeSysFs) SetMemory(memTotal string, err error) {
189213
fs.memTotal = memTotal
190214
fs.memErr = err

‎utils/sysfs/sysfs.go

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"path"
2222
"path/filepath"
2323
"regexp"
24+
"runtime"
2425
"strconv"
2526
"strings"
2627

@@ -50,6 +51,8 @@ const (
5051

5152
coreIDFilePath = "/" + sysFsCPUTopology + "/core_id"
5253
packageIDFilePath = "/" + sysFsCPUTopology + "/physical_package_id"
54+
bookIDFilePath = "/" + sysFsCPUTopology + "/book_id"
55+
drawerIDFilePath = "/" + sysFsCPUTopology + "/drawer_id"
5356

5457
// memory size calculations
5558

@@ -87,6 +90,10 @@ type SysFs interface {
8790
GetCoreID(coreIDFilePath string) (string, error)
8891
// Get physical package id for specified CPU
8992
GetCPUPhysicalPackageID(cpuPath string) (string, error)
93+
// Get book id for specified CPU
94+
GetBookID(cpuPath string) (string, error)
95+
// Get drawer id for specified CPU
96+
GetDrawerID(cpuPath string) (string, error)
9097
// Get total memory for specified NUMA node
9198
GetMemInfo(nodeDir string) (string, error)
9299
// Get hugepages from specified directory
@@ -164,6 +171,24 @@ func (fs *realSysFs) GetCPUPhysicalPackageID(cpuPath string) (string, error) {
164171
return strings.TrimSpace(string(packageID)), err
165172
}
166173

174+
func (fs *realSysFs) GetBookID(cpuPath string) (string, error) {
175+
bookIDFilePath := fmt.Sprintf("%s%s", cpuPath, bookIDFilePath)
176+
bookID, err := os.ReadFile(bookIDFilePath)
177+
if err != nil {
178+
return "", err
179+
}
180+
return strings.TrimSpace(string(bookID)), nil
181+
}
182+
183+
func (fs *realSysFs) GetDrawerID(cpuPath string) (string, error) {
184+
drawerIDFilePath := fmt.Sprintf("%s%s", cpuPath, drawerIDFilePath)
185+
drawerID, err := os.ReadFile(drawerIDFilePath)
186+
if err != nil {
187+
return "", err
188+
}
189+
return strings.TrimSpace(string(drawerID)), nil
190+
}
191+
167192
func (fs *realSysFs) GetMemInfo(nodePath string) (string, error) {
168193
meminfoPath := fmt.Sprintf("%s/%s", nodePath, meminfoFile)
169194
meminfo, err := os.ReadFile(meminfoPath)
@@ -365,22 +390,24 @@ func getCPUCount(cache string) (count int, err error) {
365390

366391
func (fs *realSysFs) GetCacheInfo(cpu int, name string) (CacheInfo, error) {
367392
cachePath := fmt.Sprintf("%s%d/cache/%s", cacheDir, cpu, name)
368-
out, err := os.ReadFile(path.Join(cachePath, "/id"))
369-
if err != nil {
370-
return CacheInfo{}, err
371-
}
372393
var id int
373-
n, err := fmt.Sscanf(string(out), "%d", &id)
374-
if err != nil || n != 1 {
375-
return CacheInfo{}, err
394+
if runtime.GOARCH != "s390x" {
395+
out, err := os.ReadFile(path.Join(cachePath, "/id"))
396+
if err != nil {
397+
return CacheInfo{}, err
398+
}
399+
n, err := fmt.Sscanf(string(out), "%d", &id)
400+
if err != nil || n != 1 {
401+
return CacheInfo{}, err
402+
}
376403
}
377404

378-
out, err = os.ReadFile(path.Join(cachePath, "/size"))
405+
out, err := os.ReadFile(path.Join(cachePath, "/size"))
379406
if err != nil {
380407
return CacheInfo{}, err
381408
}
382409
var size uint64
383-
n, err = fmt.Sscanf(string(out), "%dK", &size)
410+
n, err := fmt.Sscanf(string(out), "%dK", &size)
384411
if err != nil || n != 1 {
385412
return CacheInfo{}, err
386413
}

‎utils/sysinfo/sysinfo.go

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"fmt"
1919
"os"
2020
"regexp"
21+
"runtime"
2122
"strconv"
2223
"strings"
2324

@@ -464,12 +465,35 @@ func getCoresInfo(sysFs sysfs.SysFs, cpuDirs []string) ([]info.Core, error) {
464465
return nil, err
465466
}
466467

468+
var bookID, drawerID string
469+
// s390/s390x additional cpu topology levels
470+
if runtime.GOARCH == "s390x" {
471+
bookID, err = sysFs.GetBookID(cpuDir)
472+
if os.IsNotExist(err) {
473+
klog.Warningf("Cannot read book id for %s, book_id file does not exist, err: %s", cpuDir, err)
474+
continue
475+
} else if err != nil {
476+
return nil, err
477+
}
478+
drawerID, err = sysFs.GetDrawerID(cpuDir)
479+
if os.IsNotExist(err) {
480+
klog.Warningf("Cannot read drawer id for %s, drawer_id file does not exist, err: %s", cpuDir, err)
481+
continue
482+
} else if err != nil {
483+
return nil, err
484+
}
485+
}
486+
467487
coreIDx := -1
468488
for id, core := range cores {
469489
if core.Id == physicalID && core.SocketID == physicalPackageID {
470-
coreIDx = id
490+
// For s390x, we need to check the BookID and DrawerID match as well.
491+
if runtime.GOARCH != "s390x" || (core.BookID == bookID && core.DrawerID == drawerID) {
492+
coreIDx = id
493+
}
471494
}
472495
}
496+
473497
if coreIDx == -1 {
474498
cores = append(cores, info.Core{})
475499
coreIDx = len(cores) - 1
@@ -478,6 +502,8 @@ func getCoresInfo(sysFs sysfs.SysFs, cpuDirs []string) ([]info.Core, error) {
478502

479503
desiredCore.Id = physicalID
480504
desiredCore.SocketID = physicalPackageID
505+
desiredCore.BookID = bookID
506+
desiredCore.DrawerID = drawerID
481507

482508
if len(desiredCore.Threads) == 0 {
483509
desiredCore.Threads = []int{cpuID}

0 commit comments

Comments
 (0)