perfprof: add enablement annotation

In order to use the cpumanager policy option `prefer-align-cpus-by-uncorecache`
users need to create an enablement file through a MachineConfig.
This is because the feature is opt-in and supported only on selected
cases. By providing MachineConfigs, users explicitely opt in and this is
pretty evident on support scenarios.

Problem is: the required MachineConfig has error prone components.
We would like to streamline the flow but keep the explicit opt-in step.

A possible improvement is to automate the MachineConfig generation when
the cpumanager option is injected through the
`kubletconfig.experimental` annotation, which is currently the only
supported way to enable this feature.

To keep the opt-in component, users would also need to supply another
annotation:
```
performance.openshift.io/autogenerate-enablement: "true"
```

So the performance profile could look like
```
annotations:
  "kubeletconfig.experimental": "{\"cpuManagerPolicyOptions\":{\"prefer-align-cpus-by-uncorecache\":\"true\"}}"
  "performance.openshift.io/autogenerate-enablement": "true"
```

Signed-off-by: Francesco Romani <[email protected]>

Author: ffromani

pkg/apis/performanceprofile/v2/performanceprofile_types.go

@@ -34,6 +34,13 @@ const PerformanceProfileEnablePhysicalRpsAnnotation = "performance.openshift.io/
 // that ignores the removal of all RPS settings when realtime workload hint is explicitly set to false.
 const PerformanceProfileEnableRpsAnnotation = "performance.openshift.io/enable-rps"
 
+// PerformanceProfileAutogenerateEnablementAnnotation is an experimental annotation which let the controller
+// create the enablement file for the tech-preview `prefer-align-cpus-by-uncorecache` cpumanager policy option.
+// If the option is detected, the generated MachineConfig is automatically augmented with the expected enablement
+// file. Still, users wishing to use the feature need to explicitly opt in. To do so, instead of opting in
+// adding the enablement file, they can opt in adding the annotation and setting its value to "true".
+const PerformanceProfileAutogenerateEnablementAnnotation = "performance.openshift.io/autogenerate-enablement"
+
 // PerformanceProfileSpec defines the desired state of PerformanceProfile.
 type PerformanceProfileSpec struct {
 	// CPU defines a set of CPU related parameters.

pkg/performanceprofile/controller/performanceprofile/components/components.go

@@ -43,6 +43,19 @@ type Options struct {
 type MachineConfigOptions struct {
 	PinningMode      *apiconfigv1.CPUPartitioningMode
 	MixedCPUsEnabled bool
+	LLCFileEnabled   bool
+}
+
+func (mco *MachineConfigOptions) Clone() *MachineConfigOptions {
+	ret := MachineConfigOptions{
+		MixedCPUsEnabled: mco.MixedCPUsEnabled,
+		LLCFileEnabled:   mco.LLCFileEnabled,
+	}
+	if mco.PinningMode != nil {
+		ret.PinningMode = new(apiconfigv1.CPUPartitioningMode)
+		*ret.PinningMode = *mco.PinningMode
+	}
+	return &ret
 }
 
 type KubeletConfigOptions struct {

pkg/performanceprofile/controller/performanceprofile/components/components_test.go

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2025 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package components
+
+import (
+	"testing"
+
+	apiconfigv1 "github.com/openshift/api/config/v1"
+)
+
+func TestMachineConfigOptionsClone(t *testing.T) {
+	mode := apiconfigv1.CPUPartitioningAllNodes
+	mco := &MachineConfigOptions{
+		PinningMode: &mode,
+	}
+
+	mode2 := apiconfigv1.CPUPartitioningNone
+	mco2 := mco.Clone()
+	mco2.PinningMode = &mode2
+	mco2.LLCFileEnabled = true
+
+	// verify changes did not propagate back to the original copy
+	if *mco.PinningMode != apiconfigv1.CPUPartitioningAllNodes {
+		t.Fatalf("mutation of the cloned PinningMode altered back the original")
+	}
+	if mco.LLCFileEnabled {
+		t.Fatalf("mutation of the cloned LLCFileEnabled altered back the original")
+	}
+}

pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig.go

@@ -16,6 +16,10 @@ import (
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
 )
 
+const (
+	CPUManagerPolicyOptionPreferAlignCPUsByUncoreCache = "prefer-align-cpus-by-uncorecache"
+)
+
 const (
 	// experimentalKubeletSnippetAnnotation contains the annotation key that should be used to provide a KubeletConfig snippet with additional
 	// configurations you want to apply on top of the generated KubeletConfig resource.
@@ -46,16 +50,9 @@ const (
 // New returns new KubeletConfig object for performance sensetive workflows
 func New(profile *performancev2.PerformanceProfile, opts *components.KubeletConfigOptions) (*machineconfigv1.KubeletConfig, error) {
 	name := components.GetComponentName(profile.Name, components.ComponentNamePrefix)
-	kubeletConfig := &kubeletconfigv1beta1.KubeletConfiguration{}
-	if v, ok := profile.Annotations[experimentalKubeletSnippetAnnotation]; ok {
-		if err := json.Unmarshal([]byte(v), kubeletConfig); err != nil {
-			return nil, err
-		}
-	}
-
-	kubeletConfig.TypeMeta = metav1.TypeMeta{
-		APIVersion: kubeletconfigv1beta1.SchemeGroupVersion.String(),
-		Kind:       "KubeletConfiguration",
+	kubeletConfig, err := NewFromExperimentalAnnotation(profile)
+	if err != nil {
+		return nil, err
 	}
 
 	kubeletConfig.CPUManagerPolicy = cpuManagerPolicyStatic
@@ -193,3 +190,20 @@ func addStringToQuantity(q *resource.Quantity, value string) error {
 
 	return nil
 }
+
+func NewFromExperimentalAnnotation(profile *performancev2.PerformanceProfile) (*kubeletconfigv1beta1.KubeletConfiguration, error) {
+	kubeletConfig := &kubeletconfigv1beta1.KubeletConfiguration{}
+
+	if v, ok := profile.Annotations[experimentalKubeletSnippetAnnotation]; ok {
+		if err := json.Unmarshal([]byte(v), kubeletConfig); err != nil {
+			return nil, err
+		}
+	}
+
+	// mae sure to setup properly the metadata
+	kubeletConfig.TypeMeta = metav1.TypeMeta{
+		APIVersion: kubeletconfigv1beta1.SchemeGroupVersion.String(),
+		Kind:       "KubeletConfiguration",
+	}
+	return kubeletConfig, nil
+}

pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig_test.go

@@ -195,6 +195,18 @@ var _ = Describe("Kubelet Config", func() {
 			Expect(manifest).To(ContainSubstring("net.core.somaxconn"))
 			Expect(manifest).To(ContainSubstring(`full-pcpus-only: "false"`))
 		})
+	})
 
+	Context("with annotated performance profile", func() {
+		It("should create a kubeletconfig object", func() {
+			profile := testutils.NewPerformanceProfile("kubeletconfig-test")
+			profile.Annotations = map[string]string{
+				experimentalKubeletSnippetAnnotation: `{"allowedUnsafeSysctls": ["net.core.somaxconn"], "cpuManagerPolicyOptions": {"full-pcpus-only": "false"}}`,
+			}
+			kubeletConfig, err := NewFromExperimentalAnnotation(profile)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(kubeletConfig.AllowedUnsafeSysctls).Should(ContainElement("net.core.somaxconn"))
+			Expect(kubeletConfig.CPUManagerPolicyOptions).Should(HaveKeyWithValue("full-pcpus-only", "false"))
+		})
 	})
 })

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go

@@ -74,6 +74,9 @@ const (
 	ovsDynamicPinningTriggerHostFile = "/var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity"
 
 	cpusetConfigure = "cpuset-configure"
+
+	// required until the LLC support code (KEP-4800) goes at least Beta
+	llcEnablementFile = "openshift-llc-alignment"
 )
 
 const (
@@ -392,6 +395,15 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile, opts *componen
 		}
 		addContent(ignitionConfig, content, filepath.Join(kubernetesConfDir, mixedCPUsConfig), ptr.To[int](0644))
 	}
+
+	// required until the LLC support code (KEP-4800) goes at least Beta
+	if opts.LLCFileEnabled {
+		// Note: the content of LLC (enablement) file does not matter. What matters is only the file presence.
+		// If the file is present, the feature is available and can be enabled via the PerformanceProfile object.\
+		// If the file is missing, the feature is forced off regardless of any PerformanceProfile object content.
+		addContent(ignitionConfig, []byte{}, filepath.Join(kubernetesConfDir, llcEnablementFile), ptr.To[int](0644))
+	}
+
 	return ignitionConfig, nil
 }
 

pkg/performanceprofile/controller/performanceprofile/components/manifestset/manifestset.go

@@ -1,13 +1,16 @@
 package manifestset
 
 import (
+	"k8s.io/klog/v2"
+
 	mcov1 "github.com/openshift/api/machineconfiguration/v1"
 	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
 	tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/machineconfig"
 	profilecomponent "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile"
+	profileutil "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/runtimeclass"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/tuned"
 
@@ -53,7 +56,13 @@ func (ms *ManifestResultSet) ToManifestTable() ManifestTable {
 func GetNewComponents(profile *performancev2.PerformanceProfile, opts *components.Options) (*ManifestResultSet, error) {
 	machineConfigPoolSelector := profilecomponent.GetMachineConfigPoolSelector(profile, opts.ProfileMCP)
 
-	mc, err := machineconfig.New(profile, &opts.MachineConfig)
+	autogen := profileutil.HaveEnablementAutogeneration(profile)
+	llcEnabled := profileutil.IsLLCAlignmentEnabled(profile)
+	klog.V(4).InfoS("components manifests", "autogenerateEnablement", autogen, "LLCEnabled", llcEnabled)
+
+	mcOpts := opts.MachineConfig.Clone()
+	mcOpts.LLCFileEnabled = autogen && llcEnabled
+	mc, err := machineconfig.New(profile, mcOpts)
 	if err != nil {
 		return nil, err
 	}

pkg/performanceprofile/controller/performanceprofile/components/manifestset/manifestset_suite_test.go

@@ -0,0 +1,13 @@
+package manifestset
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestManifestSet(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Manifest Set Suite")
+}

pkg/performanceprofile/controller/performanceprofile/components/manifestset/manifestset_test.go

@@ -0,0 +1,95 @@
+package manifestset
+
+import (
+	"encoding/json"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
+
+	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
+	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
+
+	testutils "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/utils/testing"
+)
+
+var _ = Describe("LLC Enablement file", func() {
+	var profile *performancev2.PerformanceProfile
+	var opts *components.Options
+	var expectedPath string
+
+	BeforeEach(func() {
+		profile = testutils.NewPerformanceProfile("test-llc-ann")
+		opts = &components.Options{}
+		expectedPath = "/etc/kubernetes/openshift-llc-alignment"
+	})
+
+	When("the option is configured", func() {
+		It("should not be generated if missing", func() {
+			mfs, err := GetNewComponents(profile, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			ok, err := findFileInIgnition(mfs, expectedPath)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ok).To(BeFalse(), "expected path %q found in ignition", expectedPath)
+		})
+
+		It("should not be generated if missing control annotation", func() {
+			profile.Annotations = map[string]string{
+				"kubeletconfig.experimental": `{"cpuManagerPolicyOptions": { "prefer-align-cpus-by-uncorecache": "true", "full-pcpus-only": "false" }}`,
+			}
+
+			mfs, err := GetNewComponents(profile, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			ok, err := findFileInIgnition(mfs, expectedPath)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ok).To(BeFalse(), "expected path %q found in ignition", expectedPath)
+		})
+
+		It("should not be generated if disabled", func() {
+			profile.Annotations = map[string]string{
+				"performance.openshift.io/autogenerate-enablement": "true",
+				"kubeletconfig.experimental":                       `{"cpuManagerPolicyOptions": { "prefer-align-cpus-by-uncorecache": "false", "full-pcpus-only": "false" }}`,
+			}
+
+			mfs, err := GetNewComponents(profile, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			ok, err := findFileInIgnition(mfs, expectedPath)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ok).To(BeFalse(), "expected path %q found in ignition", expectedPath)
+
+		})
+
+		It("should be generated if enabled", func() {
+			profile.Annotations = map[string]string{
+				"performance.openshift.io/autogenerate-enablement": "true",
+				"kubeletconfig.experimental":                       `{"cpuManagerPolicyOptions": { "prefer-align-cpus-by-uncorecache": "true", "full-pcpus-only": "false" }}`,
+			}
+
+			mfs, err := GetNewComponents(profile, opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			ok, err := findFileInIgnition(mfs, expectedPath)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ok).To(BeTrue(), "expected path %q not found in ignition", expectedPath)
+		})
+	})
+})
+
+func findFileInIgnition(mfs *ManifestResultSet, filePath string) (bool, error) {
+	result := igntypes.Config{}
+	err := json.Unmarshal(mfs.MachineConfig.Spec.Config.Raw, &result)
+	if err != nil {
+		return false, err
+	}
+
+	for _, ignFile := range result.Storage.Files {
+		if ignFile.Path == filePath {
+			return true, nil
+		}
+	}
+	return false, nil
+}

pkg/performanceprofile/controller/performanceprofile/components/profile/profile.go

@@ -1,8 +1,11 @@
 package profile
 
 import (
+	"strconv"
+
 	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
+	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig"
 
 	mcov1 "github.com/openshift/api/machineconfiguration/v1"
 )
@@ -91,3 +94,30 @@ func IsMixedCPUsEnabled(profile *performancev2.PerformanceProfile) bool {
 	}
 	return *profile.Spec.WorkloadHints.MixedCpus
 }
+
+func HaveEnablementAutogeneration(profile *performancev2.PerformanceProfile) bool {
+	if profile.Annotations == nil {
+		return false
+	}
+	val, ok := profile.Annotations[performancev2.PerformanceProfileAutogenerateEnablementAnnotation]
+	if !ok {
+		return false
+	}
+	return val == "true"
+}
+
+func IsLLCAlignmentEnabled(profile *performancev2.PerformanceProfile) bool {
+	kubeletConfig, err := kubeletconfig.NewFromExperimentalAnnotation(profile)
+	if err != nil {
+		return false
+	}
+	val, ok := kubeletConfig.CPUManagerPolicyOptions[kubeletconfig.CPUManagerPolicyOptionPreferAlignCPUsByUncoreCache]
+	if !ok {
+		return false
+	}
+	v, err := strconv.ParseBool(val)
+	if err != nil {
+		return false
+	}
+	return v
+}