Add mon groups for resctrl.

"mon_groups" can be created to monitor subsets of tasks in the CTRL_MON
group that is their ancestor.

More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt

Signed-off-by: Paweł Szulik <[email protected]>

Author: Paweł Szulik

events.go

@@ -154,12 +154,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 	}
 
 	if is := ls.IntelRdtStats; is != nil {
-		if intelrdt.IsCatEnabled() {
+		if intelrdt.IsCatEnabled() && is.L3CacheInfo != nil {
 			s.IntelRdt.L3CacheInfo = convertL3CacheInfo(is.L3CacheInfo)
 			s.IntelRdt.L3CacheSchemaRoot = is.L3CacheSchemaRoot
 			s.IntelRdt.L3CacheSchema = is.L3CacheSchema
 		}
-		if intelrdt.IsMbaEnabled() {
+		if intelrdt.IsMbaEnabled() && is.MemBwInfo != nil{
 			s.IntelRdt.MemBwInfo = convertMemBwInfo(is.MemBwInfo)
 			s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot
 			s.IntelRdt.MemBwSchema = is.MemBwSchema

libcontainer/SPEC.md

@@ -158,22 +158,31 @@ init process will block waiting for the parent to finish setup.
 ### IntelRdt
 
 Intel platforms with new Xeon CPU support Resource Director Technology (RDT).
-Cache Allocation Technology (CAT) and Memory Bandwidth Allocation (MBA) are
-two sub-features of RDT.
+Cache Allocation Technology (CAT), Cache Monitoring Technology (CMT),
+Memory Bandwidth Allocation (MBA) and Memory Bandwidth Monitoring (MBM) are
+four sub-features of RDT.
 
 Cache Allocation Technology (CAT) provides a way for the software to restrict
 cache allocation to a defined 'subset' of L3 cache which may be overlapping
 with other 'subsets'. The different subsets are identified by class of
 service (CLOS) and each CLOS has a capacity bitmask (CBM).
 
+Cache Monitoring Technology (CMT) supports monitoring of the last-level cache (LLC) occupancy
+for each running thread simultaneously.
+
 Memory Bandwidth Allocation (MBA) provides indirect and approximate throttle
 over memory bandwidth for the software. A user controls the resource by
-indicating the percentage of maximum memory bandwidth or memory bandwidth limit
-in MBps unit if MBA Software Controller is enabled.
+indicating the percentage of maximum memory bandwidth or memory bandwidth
+limit in MBps unit if MBA Software Controller is enabled.
+
+Memory Bandwidth Monitoring (MBM) supports monitoring of total and local memory bandwidth  
+for each running thread simultaneously.
 
-It can be used to handle L3 cache and memory bandwidth resources allocation
-for containers if hardware and kernel support Intel RDT CAT and MBA features.
+More details about Intel RDT CAT and MBA can be found in the section 17.18 and 17.19, Volume 3
+of Intel Software Developer Manual:
+https://software.intel.com/en-us/articles/intel-sdm
 
+About Intel RDT kernel interface:
 In Linux 4.10 kernel or newer, the interface is defined and exposed via
 "resource control" filesystem, which is a "cgroup-like" interface.
 
@@ -194,22 +203,43 @@ tree /sys/fs/resctrl
 |   |   |-- cbm_mask
 |   |   |-- min_cbm_bits
 |   |   |-- num_closids
+|   |-- L3_MON
+|   |   |-- max_threshold_occupancy
+|   |   |-- mon_features
+|   |   |-- num_rmids
 |   |-- MB
 |       |-- bandwidth_gran
 |       |-- delay_linear
 |       |-- min_bandwidth
 |       |-- num_closids
-|-- ...
+|-- mon_groups
+    |-- <container_id>
+        |-- ...
+        |-- mon_data
+    	       |-- mon_L3_00
+    	           |-- llc_occupancy
+                   |-- mbm_local_bytes
+                   |-- mbm_total_bytes
+               |-- ...
+        |-- tasks
 |-- schemata
 |-- tasks
 |-- <container_id>
     |-- ...
-    |-- schemata
+    |-- mon_data
+        |-- mon_L3_00
+               |-- llc_occupancy
+               |-- mbm_local_bytes
+            |-- mbm_total_bytes
+        |-- ...
     |-- tasks
+    |-- schemata
+|-- ...
 ```
 
 For runc, we can make use of `tasks` and `schemata` configuration for L3
-cache and memory bandwidth resources constraints.
+cache and memory bandwidth resources constraints, `mon_data` directory for
+CMT and MBM statistics.
 
 The file `tasks` has a list of tasks that belongs to this group (e.g.,
 <container_id>" group). Tasks can be added to a group by writing the task ID
@@ -251,7 +281,7 @@ that is allocated is also dependent on the CPU model and can be looked up at
 min_bw + N * bw_gran. Intermediate values are rounded to the next control
 step available on the hardware.
 
-If MBA Software Controller is enabled through mount option "-o mba_MBps"
+If MBA Software Controller is enabled through mount option "-o mba_MBps":
 mount -t resctrl resctrl -o mba_MBps /sys/fs/resctrl
 We could specify memory bandwidth in "MBps" (Mega Bytes per second) unit
 instead of "percentages". The kernel underneath would use a software feedback
@@ -263,11 +293,12 @@ For example, on a two-socket machine, the schema line could be
 "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on socket 0
 and 7000 MBps memory bandwidth limit on socket 1.
 
-For more information about Intel RDT kernel interface:  
+For more information about Intel RDT kernel interface:
 https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
 
-```
+
 An example for runc:
+```
 Consider a two-socket machine with two L3 caches where the default CBM is
 0x7ff and the max CBM length is 11 bits, and minimum memory bandwidth of 10%
 with a memory bandwidth granularity of 10%.
@@ -278,10 +309,18 @@ maximum memory bandwidth of 20% on socket 0 and 70% on socket 1.
 
 "linux": {
     "intelRdt": {
-        "closID": "guaranteed_group",
         "l3CacheSchema": "L3:0=7f0;1=1f",
         "memBwSchema": "MB:0=20;1=70"
-    }
+	}
+}
+```
+Another example:
+```
+We only want to monitor memory bandwidth and llc occupancy.
+"linux": {
+    "intelRdt": {
+        "monitoring": true
+	}
 }
 ```
 

libcontainer/configs/config.go

@@ -192,7 +192,7 @@ type Config struct {
 	NoNewKeyring bool `json:"no_new_keyring"`
 
 	// IntelRdt specifies settings for Intel RDT group that the container is placed into
-	// to limit the resources (e.g., L3 cache, memory bandwidth) the container has available
+	// to limit the resources (e.g., L3 cache, memory bandwidth) the container has available.
 	IntelRdt *IntelRdt `json:"intel_rdt,omitempty"`
 
 	// RootlessEUID is set when the runc was launched with non-zero EUID.

libcontainer/configs/intelrdt.go

@@ -10,4 +10,7 @@ type IntelRdt struct {
 	// The unit of memory bandwidth is specified in "percentages" by
 	// default, and in "MBps" if MBA Software Controller is enabled.
 	MemBwSchema string `json:"memBwSchema,omitempty"`
+
+	// Monitoring flag that's indicated to monitor llc-occupancy or total and local memory bandwidth.
+	Monitoring bool `json:"monitoring,omitempty"`
 }

libcontainer/configs/validate/validator.go

@@ -182,22 +182,17 @@ func (v *ConfigValidator) sysctl(config *configs.Config) error {
 
 func (v *ConfigValidator) intelrdt(config *configs.Config) error {
 	if config.IntelRdt != nil {
-		if !intelrdt.IsCatEnabled() && !intelrdt.IsMbaEnabled() {
-			return errors.New("intelRdt is specified in config, but Intel RDT is not supported or enabled")
+		if config.IntelRdt.L3CacheSchema == "" && config.IntelRdt.MemBwSchema == "" && !config.IntelRdt.Monitoring {
+			return errors.New("intelRdt config is specified but values are empty")
 		}
-
 		if !intelrdt.IsCatEnabled() && config.IntelRdt.L3CacheSchema != "" {
 			return errors.New("intelRdt.l3CacheSchema is specified in config, but Intel RDT/CAT is not enabled")
 		}
 		if !intelrdt.IsMbaEnabled() && config.IntelRdt.MemBwSchema != "" {
 			return errors.New("intelRdt.memBwSchema is specified in config, but Intel RDT/MBA is not enabled")
 		}
-
-		if intelrdt.IsCatEnabled() && config.IntelRdt.L3CacheSchema == "" {
-			return errors.New("Intel RDT/CAT is enabled and intelRdt is specified in config, but intelRdt.l3CacheSchema is empty")
-		}
-		if intelrdt.IsMbaEnabled() && config.IntelRdt.MemBwSchema == "" {
-			return errors.New("Intel RDT/MBA is enabled and intelRdt is specified in config, but intelRdt.memBwSchema is empty")
+		if !intelrdt.IsMBMEnabled() && !intelrdt.IsCMTEnabled() && config.IntelRdt.Monitoring {
+			return errors.New("intelRdt.monitoring is specified in config, but MBM and CMT Intel RDT are not enabled")
 		}
 	}
 

libcontainer/container_linux.go

@@ -1846,16 +1846,17 @@ func (c *linuxContainer) currentState() (*State, error) {
 		startTime           uint64
 		externalDescriptors []string
 		pid                 = -1
+		intelRdtPath        string
 	)
 	if c.initProcess != nil {
 		pid = c.initProcess.pid()
 		startTime, _ = c.initProcess.startTime()
 		externalDescriptors = c.initProcess.externalDescriptors()
 	}
-	intelRdtPath, err := intelrdt.GetIntelRdtPath(c.ID())
-	if err != nil {
-		intelRdtPath = ""
+	if c.intelRdtManager != nil {
+		intelRdtPath = c.intelRdtManager.GetPath()
 	}
+
 	state := &State{
 		BaseState: BaseState{
 			ID:                   c.ID(),

libcontainer/factory_linux.go

@@ -262,18 +262,16 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err
 		return nil, newGenericError(err, SystemError)
 	}
 	c := &linuxContainer{
-		id:            id,
-		root:          containerRoot,
-		config:        config,
-		initPath:      l.InitPath,
-		initArgs:      l.InitArgs,
-		criuPath:      l.CriuPath,
-		newuidmapPath: l.NewuidmapPath,
-		newgidmapPath: l.NewgidmapPath,
-		cgroupManager: l.NewCgroupsManager(config.Cgroups, nil),
-	}
-	if intelrdt.IsCatEnabled() || intelrdt.IsMbaEnabled() {
-		c.intelRdtManager = l.NewIntelRdtManager(config, id, "")
+		id:              id,
+		root:            containerRoot,
+		config:          config,
+		initPath:        l.InitPath,
+		initArgs:        l.InitArgs,
+		criuPath:        l.CriuPath,
+		newuidmapPath:   l.NewuidmapPath,
+		newgidmapPath:   l.NewgidmapPath,
+		cgroupManager:   l.NewCgroupsManager(config.Cgroups, nil),
+		intelRdtManager: l.NewIntelRdtManager(config, id, ""),
 	}
 	c.state = &stoppedState{c: c}
 	return c, nil
@@ -313,14 +311,13 @@ func (l *LinuxFactory) Load(id string) (Container, error) {
 		cgroupManager:        l.NewCgroupsManager(state.Config.Cgroups, state.CgroupPaths),
 		root:                 containerRoot,
 		created:              state.Created,
+		intelRdtManager:      l.NewIntelRdtManager(&state.Config, id, state.IntelRdtPath),
 	}
 	c.state = &loadedState{c: c}
 	if err := c.refreshState(); err != nil {
 		return nil, err
 	}
-	if intelrdt.IsCatEnabled() || intelrdt.IsMbaEnabled() {
-		c.intelRdtManager = l.NewIntelRdtManager(&state.Config, id, state.IntelRdtPath)
-	}
+
 	return c, nil
 }