Add mon groups for resctrl.

"mon_groups" can be created to monitor subsets of tasks in the CTRL_MON
group that is their ancestor.

More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt

Signed-off-by: Paweł Szulik <[email protected]>

Author: Paweł Szulik

libcontainer/intelrdt/intelrdt.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/opencontainers/runc/libcontainer/configs"
 )
 
@@ -167,10 +169,17 @@ type IntelRdtManager struct {
 	Config *configs.Config
 	Id     string
 	Path   string
+	Type   string
 }
 
 const (
-	IntelRdtTasks = "tasks"
+	IntelRdtTasks                = "tasks"
+	MonitoringGroupDirectoryName = "mon_groups"
+
+	// See "Resource alloc and monitor groups" part in:
+	// https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
+	MonitoringGroupType = "MON"
+	ControlGroupType    = "CTRL_MON"
 )
 
 var (
@@ -548,26 +557,47 @@ func GetIntelRdtPath(id string) (string, error) {
 	return path, nil
 }
 
+// Get the 'container_id" path in Intel RDT "monitoring group" filesystem.
+func getMonGroupIntelRdtPath(id string) (string, error) {
+	rootPath, err := getIntelRdtRoot()
+	if err != nil {
+		return "", err
+	}
+
+	path := filepath.Join(rootPath, MonitoringGroupDirectoryName, id)
+	return path, nil
+}
+
 // Applies Intel RDT configuration to the process with the specified pid
 func (m *IntelRdtManager) Apply(pid int) (err error) {
-	// If intelRdt is not specified in config, we do nothing
-	if m.Config.IntelRdt == nil {
+	switch m.Type {
+	case ControlGroupType:
+		// If intelRdt is not specified in config, we do nothing
+		if m.Config.IntelRdt == nil {
+			return nil
+		}
+		rdtData, err := getIntelRdtData(m.Config, pid)
+		if err != nil && !IsNotFound(err) {
+			return err
+		}
+
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		path, err := rdtData.join(m.Id)
+		if err != nil {
+			return err
+		}
+
+		m.Path = path
 		return nil
-	}
-	d, err := getIntelRdtData(m.Config, pid)
-	if err != nil && !IsNotFound(err) {
-		return err
-	}
 
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	path, err := d.join(m.Id)
-	if err != nil {
-		return err
+	case MonitoringGroupType:
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		return WriteIntelRdtTasks(m.Path, pid)
 	}
 
-	m.Path = path
-	return nil
+	return fmt.Errorf("couldn't recognize resctrl type: %v", m.Type)
 }
 
 // Destroys the Intel RDT 'container_id' group
@@ -584,94 +614,124 @@ func (m *IntelRdtManager) Destroy() error {
 // Returns Intel RDT path to save in a state file and to be able to
 // restore the object later
 func (m *IntelRdtManager) GetPath() string {
+	var err error
 	if m.Path == "" {
-		m.Path, _ = GetIntelRdtPath(m.Id)
+		switch m.Type {
+		case ControlGroupType:
+			m.Path, err = GetIntelRdtPath(m.Id)
+			if err != nil {
+				logrus.Errorf("couldn't obtain Resctrl control group path for manager with id %v: %v", m.Id, err)
+			}
+		case MonitoringGroupType:
+			flattedContainerID := strings.Replace(m.Id, "/", "-", -1)
+			m.Path, err = getMonGroupIntelRdtPath(flattedContainerID)
+			if err != nil {
+				logrus.Errorf("couldn't obtain Resctrl monitoring group path for manager with id %v: %v", m.Id, err)
+			}
+		}
 	}
 	return m.Path
 }
 
 // Returns statistics for Intel RDT
 func (m *IntelRdtManager) GetStats() (*Stats, error) {
-	// If intelRdt is not specified in config
-	if m.Config.IntelRdt == nil {
-		return nil, nil
-	}
-
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	stats := NewStats()
 
-	rootPath, err := getIntelRdtRoot()
-	if err != nil {
-		return nil, err
-	}
-	// The read-only L3 cache and memory bandwidth schemata in root
-	tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
-	if err != nil {
-		return nil, err
-	}
-	schemaRootStrings := strings.Split(tmpRootStrings, "\n")
-
-	// The L3 cache and memory bandwidth schemata in 'container_id' group
-	containerPath := m.GetPath()
-	tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
-	if err != nil {
-		return nil, err
-	}
-	schemaStrings := strings.Split(tmpStrings, "\n")
+	switch m.Type {
+	case ControlGroupType:
+		containerPath := m.GetPath()
 
-	if IsCatEnabled() {
-		// The read-only L3 cache information
-		l3CacheInfo, err := getL3CacheInfo()
+		err := getMonitoringStats(containerPath, stats)
 		if err != nil {
 			return nil, err
 		}
-		stats.L3CacheInfo = l3CacheInfo
 
-		// The read-only L3 cache schema in root
-		for _, schemaRoot := range schemaRootStrings {
-			if strings.Contains(schemaRoot, "L3") {
-				stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
+		// If intelRdt is not specified in config
+		if m.Config != nil {
+			if m.Config.IntelRdt == nil {
+				return nil, nil
 			}
-		}
 
-		// The L3 cache schema in 'container_id' group
-		for _, schema := range schemaStrings {
-			if strings.Contains(schema, "L3") {
-				stats.L3CacheSchema = strings.TrimSpace(schema)
+			rootPath, err := getIntelRdtRoot()
+			if err != nil {
+				return nil, err
 			}
-		}
-	}
+			// The read-only L3 cache and memory bandwidth schemata in root
+			tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
+			if err != nil {
+				return nil, err
+			}
+			schemaRootStrings := strings.Split(tmpRootStrings, "\n")
 
-	if IsMbaEnabled() {
-		// The read-only memory bandwidth information
-		memBwInfo, err := getMemBwInfo()
-		if err != nil {
-			return nil, err
-		}
-		stats.MemBwInfo = memBwInfo
+			// The L3 cache and memory bandwidth schemata in 'container_id' group
 
-		// The read-only memory bandwidth information
-		for _, schemaRoot := range schemaRootStrings {
-			if strings.Contains(schemaRoot, "MB") {
-				stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
+			tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
+			if err != nil {
+				return nil, err
 			}
-		}
+			schemaStrings := strings.Split(tmpStrings, "\n")
+
+			if IsCatEnabled() {
+				// The read-only L3 cache information
+				l3CacheInfo, err := getL3CacheInfo()
+				if err != nil {
+					return nil, err
+				}
+				stats.L3CacheInfo = l3CacheInfo
 
-		// The memory bandwidth schema in 'container_id' group
-		for _, schema := range schemaStrings {
-			if strings.Contains(schema, "MB") {
-				stats.MemBwSchema = strings.TrimSpace(schema)
+				// The read-only L3 cache schema in root
+				for _, schemaRoot := range schemaRootStrings {
+					if strings.Contains(schemaRoot, "L3") {
+						stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
+					}
+				}
+
+				// The L3 cache schema in 'container_id' group
+				for _, schema := range schemaStrings {
+					if strings.Contains(schema, "L3") {
+						stats.L3CacheSchema = strings.TrimSpace(schema)
+					}
+				}
+			}
+
+			if IsMbaEnabled() {
+				// The read-only memory bandwidth information
+				memBwInfo, err := getMemBwInfo()
+				if err != nil {
+					return nil, err
+				}
+				stats.MemBwInfo = memBwInfo
+
+				// The read-only memory bandwidth information
+				for _, schemaRoot := range schemaRootStrings {
+					if strings.Contains(schemaRoot, "MB") {
+						stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
+					}
+				}
+
+				// The memory bandwidth schema in 'container_id' group
+				for _, schema := range schemaStrings {
+					if strings.Contains(schema, "MB") {
+						stats.MemBwSchema = strings.TrimSpace(schema)
+					}
+				}
 			}
 		}
-	}
 
-	err = getMonitoringStats(containerPath, stats)
-	if err != nil {
-		return nil, err
-	}
+		return stats, nil
+
+	case MonitoringGroupType:
+		path := m.GetPath()
 
-	return stats, nil
+		err := getMonitoringStats(path, stats)
+		if err != nil {
+			return nil, err
+		}
+		return stats, nil
+	}
+	return nil, fmt.Errorf("couldn't obtain stats from: %q resctrl manager of type: %q", m.Id, m.Type)
 }
 
 // Set Intel RDT "resource control" filesystem as configured.
@@ -721,34 +781,46 @@ func (m *IntelRdtManager) Set(container *configs.Config) error {
 	// For example, on a two-socket machine, the schema line could be
 	// "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on
 	// socket 0 and 7000 MBps memory bandwidth limit on socket 1.
-	if container.IntelRdt != nil {
-		path := m.GetPath()
-		l3CacheSchema := container.IntelRdt.L3CacheSchema
-		memBwSchema := container.IntelRdt.MemBwSchema
+	switch m.Type {
+	case ControlGroupType:
+		if container.IntelRdt != nil {
+			path := m.GetPath()
+			l3CacheSchema := container.IntelRdt.L3CacheSchema
+			memBwSchema := container.IntelRdt.MemBwSchema
+
+			// Write a single joint schema string to schemata file
+			if l3CacheSchema != "" && memBwSchema != "" {
+				if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
+					return NewLastCmdError(err)
+				}
+			}
 
-		// Write a single joint schema string to schemata file
-		if l3CacheSchema != "" && memBwSchema != "" {
-			if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
-				return NewLastCmdError(err)
+			// Write only L3 cache schema string to schemata file
+			if l3CacheSchema != "" && memBwSchema == "" {
+				if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
+					return NewLastCmdError(err)
+				}
 			}
-		}
 
-		// Write only L3 cache schema string to schemata file
-		if l3CacheSchema != "" && memBwSchema == "" {
-			if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
-				return NewLastCmdError(err)
+			// Write only memory bandwidth schema string to schemata file
+			if l3CacheSchema == "" && memBwSchema != "" {
+				if err := writeFile(path, "schemata", memBwSchema); err != nil {
+					return NewLastCmdError(err)
+				}
 			}
 		}
 
-		// Write only memory bandwidth schema string to schemata file
-		if l3CacheSchema == "" && memBwSchema != "" {
-			if err := writeFile(path, "schemata", memBwSchema); err != nil {
-				return NewLastCmdError(err)
-			}
+	case MonitoringGroupType:
+		if _, err := os.Stat(m.GetPath()); err == nil {
+			return nil
+		}
+		if err := os.Mkdir(m.GetPath(), 0755); err != nil {
+			return err
 		}
+		return nil
 	}
 
-	return nil
+	return fmt.Errorf("couldn't set configuration for: %q resctrl manager of type: %q", m.Id, m.Type)
 }
 
 func (raw *intelRdtData) join(id string) (string, error) {

libcontainer/intelrdt/intelrdt_test.go

@@ -3,6 +3,7 @@
 package intelrdt
 
 import (
+	"errors"
 	"strings"
 	"testing"
 )
@@ -120,3 +121,41 @@ func TestIntelRdtSetMemBwScSchema(t *testing.T) {
 		t.Fatal("Got the wrong value, set 'schemata' failed.")
 	}
 }
+
+func TestIntelRdtManager_GetStats_NotSupported_Type(t *testing.T) {
+	manager := IntelRdtManager{
+		Type: "not_supported",
+		Id:   "id",
+	}
+
+	_, err := manager.GetStats()
+
+	expectedError := errors.New("couldn't obtain stats from: \"id\" resctrl manager of type: \"not_supported\"")
+
+	if err == nil {
+		t.Fatalf("Expected error: %v, got nil.", expectedError)
+	}
+
+	if err.Error() != expectedError.Error() {
+		t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
+	}
+}
+
+func Test_IntelRdtManager_Set_NotSupported_Type(t *testing.T) {
+	manager := IntelRdtManager{
+		Type: "not_supported",
+		Id:   "id",
+	}
+
+	err := manager.Set(nil)
+
+	expectedError := errors.New("couldn't set configuration for: \"id\" resctrl manager of type: \"not_supported\"")
+
+	if err == nil {
+		t.Fatalf("Expected error: %v, got nil.", expectedError)
+	}
+
+	if err.Error() != expectedError.Error() {
+		t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
+	}
+}

update.go

@@ -321,6 +321,7 @@ other options are ignored.
 					Config: &config,
 					Id:     container.ID(),
 					Path:   state.IntelRdtPath,
+					Type:   intelrdt.ControlGroupType,
 				}
 				if err := intelRdtManager.Apply(state.InitProcessPid); err != nil {
 					return err