Merge pull request kata-containers#7623 from fidencio/topic/runtime-i…

…mprove-vcpu-allocation-on-host-side

runtime: Improve vCPU allocation for the VMMs

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -47,7 +47,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.ctlpath` (R) | `string` | Path to the `acrnctl` binary for the ACRN hypervisor |
 | `io.katacontainers.config.hypervisor.default_max_vcpus` | uint32| the maximum number of vCPUs allocated for the VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.default_memory` | uint32| the memory assigned for a VM by the hypervisor in `MiB` |
-| `io.katacontainers.config.hypervisor.default_vcpus` | uint32| the default vCPUs assigned for a VM by the hypervisor |
+| `io.katacontainers.config.hypervisor.default_vcpus` | float32| the default vCPUs assigned for a VM by the hypervisor |
 | `io.katacontainers.config.hypervisor.disable_block_device_use` | `boolean` | disallow a block device from being used |
 | `io.katacontainers.config.hypervisor.disable_image_nvdimm` | `boolean` | specify if a `nvdimm` device should be used as rootfs for the guest (QEMU) |
 | `io.katacontainers.config.hypervisor.disable_vhost_net` | `boolean` | specify if `vhost-net` is not available on the host |

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -134,7 +134,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.Runti
 		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
 		DisableNewNetNs:      disableNewNetNs,
-		DefaultVCPUCount:     hypConfig.NumVCPUs,
+		DefaultVCPUCount:     hypConfig.NumVCPUs(),
 		DefaultMaxVCPUCount:  hypConfig.DefaultMaxVCPUs,
 		DefaultMemSize:       hypConfig.MemorySize,
 		DefaultMsize9p:       hypConfig.Msize9p,

src/runtime/pkg/katautils/config.go

@@ -133,7 +133,7 @@ type hypervisor struct {
 	MemSlots                       uint32                    `toml:"memory_slots"`
 	DefaultBridges                 uint32                    `toml:"default_bridges"`
 	Msize9p                        uint32                    `toml:"msize_9p"`
-	NumVCPUs                       int32                     `toml:"default_vcpus"`
+	NumVCPUs                       float32                   `toml:"default_vcpus"`
 	BlockDeviceCacheSet            bool                      `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect         bool                      `toml:"block_device_cache_direct"`
 	BlockDeviceCacheNoflush        bool                      `toml:"block_device_cache_noflush"`
@@ -395,17 +395,17 @@ func getCurrentCpuNum() uint32 {
 	return cpu
 }
 
-func (h hypervisor) defaultVCPUs() uint32 {
-	numCPUs := getCurrentCpuNum()
+func (h hypervisor) defaultVCPUs() float32 {
+	numCPUs := float32(getCurrentCpuNum())
 
-	if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
+	if h.NumVCPUs < 0 || h.NumVCPUs > numCPUs {
 		return numCPUs
 	}
 	if h.NumVCPUs == 0 { // or unspecified
-		return defaultVCPUCount
+		return float32(defaultVCPUCount)
 	}
 
-	return uint32(h.NumVCPUs)
+	return h.NumVCPUs
 }
 
 func (h hypervisor) defaultMaxVCPUs() uint32 {
@@ -723,7 +723,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		RootfsType:            rootfsType,
 		FirmwarePath:          firmware,
 		KernelParams:          vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
-		NumVCPUs:              h.defaultVCPUs(),
+		NumVCPUsF:             h.defaultVCPUs(),
 		DefaultMaxVCPUs:       h.defaultMaxVCPUs(),
 		MemorySize:            h.defaultMemSz(),
 		MemSlots:              h.defaultMemSlots(),
@@ -857,7 +857,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		CPUFeatures:             cpuFeatures,
 		KernelParams:            vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
 		HypervisorMachineType:   machineType,
-		NumVCPUs:                h.defaultVCPUs(),
+		NumVCPUsF:               h.defaultVCPUs(),
 		DefaultMaxVCPUs:         h.defaultMaxVCPUs(),
 		MemorySize:              h.defaultMemSz(),
 		MemSlots:                h.defaultMemSlots(),
@@ -968,7 +968,7 @@ func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		HypervisorCtlPathList: h.CtlPathList,
 		FirmwarePath:          firmware,
 		KernelParams:          vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
-		NumVCPUs:              h.defaultVCPUs(),
+		NumVCPUsF:             h.defaultVCPUs(),
 		DefaultMaxVCPUs:       h.defaultMaxVCPUs(),
 		MemorySize:            h.defaultMemSz(),
 		MemSlots:              h.defaultMemSlots(),
@@ -1059,7 +1059,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		MachineAccelerators:            machineAccelerators,
 		KernelParams:                   vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
 		HypervisorMachineType:          machineType,
-		NumVCPUs:                       h.defaultVCPUs(),
+		NumVCPUsF:                      h.defaultVCPUs(),
 		DefaultMaxVCPUs:                h.defaultMaxVCPUs(),
 		MemorySize:                     h.defaultMemSz(),
 		MemSlots:                       h.defaultMemSlots(),
@@ -1132,7 +1132,7 @@ func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		ImagePath:       image,
 		RootfsType:      rootfsType,
 		KernelParams:    vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
-		NumVCPUs:        h.defaultVCPUs(),
+		NumVCPUsF:       h.defaultVCPUs(),
 		DefaultMaxVCPUs: h.defaultMaxVCPUs(),
 		MemorySize:      h.defaultMemSz(),
 		MemSlots:        h.defaultMemSlots(),
@@ -1297,7 +1297,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		MachineAccelerators:      defaultMachineAccelerators,
 		CPUFeatures:              defaultCPUFeatures,
 		HypervisorMachineType:    defaultMachineType,
-		NumVCPUs:                 defaultVCPUCount,
+		NumVCPUsF:                float32(defaultVCPUCount),
 		DefaultMaxVCPUs:          defaultMaxVCPUCount,
 		MemorySize:               defaultMemSize,
 		MemOffset:                defaultMemOffset,

src/runtime/pkg/katautils/config_test.go

@@ -161,7 +161,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		RootfsType:            rootfsType,
 		KernelParams:          vc.DeserializeParams(vc.KernelParamFields(kernelParams)),
 		HypervisorMachineType: machineType,
-		NumVCPUs:              defaultVCPUCount,
+		NumVCPUsF:             float32(defaultVCPUCount),
 		DefaultMaxVCPUs:       getCurrentCpuNum(),
 		MemorySize:            defaultMemSize,
 		DefaultMaxMemorySize:  maxMemory,
@@ -554,7 +554,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		InitrdPath:            defaultInitrdPath,
 		RootfsType:            defaultRootfsType,
 		HypervisorMachineType: defaultMachineType,
-		NumVCPUs:              defaultVCPUCount,
+		NumVCPUsF:             float32(defaultVCPUCount),
 		DefaultMaxVCPUs:       defaultMaxVCPUCount,
 		MemorySize:            defaultMemSize,
 		DisableBlockDeviceUse: defaultDisableBlockDeviceUse,
@@ -926,7 +926,7 @@ func TestHypervisorDefaults(t *testing.T) {
 	h := hypervisor{}
 
 	assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong")
-	assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), float32(defaultVCPUCount), "default vCPU number is wrong")
 	assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong")
 	assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong")
 
@@ -936,13 +936,13 @@ func TestHypervisorDefaults(t *testing.T) {
 
 	// auto inferring
 	h.NumVCPUs = -1
-	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), float32(numCPUs), "default vCPU number is wrong")
 
 	h.NumVCPUs = 2
-	assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong")
+	assert.Equal(h.defaultVCPUs(), float32(2), "default vCPU number is wrong")
 
-	h.NumVCPUs = int32(numCPUs) + 1
-	assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong")
+	h.NumVCPUs = float32(numCPUs + 1)
+	assert.Equal(h.defaultVCPUs(), float32(numCPUs), "default vCPU number is wrong")
 
 	h.DefaultMaxVCPUs = 2
 	assert.Equal(h.defaultMaxVCPUs(), uint32(2), "default max vCPU number is wrong")
@@ -1395,7 +1395,7 @@ func TestDefaultCPUFeatures(t *testing.T) {
 func TestUpdateRuntimeConfigurationVMConfig(t *testing.T) {
 	assert := assert.New(t)
 
-	vcpus := uint(2)
+	vcpus := float32(2)
 	mem := uint32(2048)
 
 	config := oci.RuntimeConfig{}
@@ -1404,7 +1404,7 @@ func TestUpdateRuntimeConfigurationVMConfig(t *testing.T) {
 	tomlConf := tomlConfig{
 		Hypervisor: map[string]hypervisor{
 			qemuHypervisorTableType: {
-				NumVCPUs:       int32(vcpus),
+				NumVCPUs:       vcpus,
 				MemorySize:     mem,
 				Path:           "/",
 				Kernel:         "/",
@@ -1727,7 +1727,7 @@ vfio_mode="vfio"
 	assert.NoError(t, err)
 
 	assert.Equal(t, config.Hypervisor["qemu"].Path, "/usr/bin/qemu-kvm")
-	assert.Equal(t, config.Hypervisor["qemu"].NumVCPUs, int32(2))
+	assert.Equal(t, config.Hypervisor["qemu"].NumVCPUs, float32(2))
 	assert.Equal(t, config.Hypervisor["qemu"].DefaultBridges, uint32(4))
 	assert.Equal(t, config.Hypervisor["qemu"].SharedFS, "virtio-9p")
 	assert.Equal(t, config.Runtime.Debug, true)
@@ -1765,7 +1765,7 @@ func TestUpdateRuntimeConfigHypervisor(t *testing.T) {
 		tomlConf := tomlConfig{
 			Hypervisor: map[string]hypervisor{
 				h.name: {
-					NumVCPUs:       int32(2),
+					NumVCPUs:       float32(2),
 					MemorySize:     uint32(2048),
 					Path:           "/",
 					Kernel:         "/",

src/runtime/pkg/oci/utils.go

@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"path/filepath"
 	"regexp"
 	goruntime "runtime"
@@ -137,7 +138,7 @@ type RuntimeConfig struct {
 
 	// Sandbox sizing information which, if provided, indicates the size of
 	// the sandbox needed for the workload(s)
-	SandboxCPUs  uint32
+	SandboxCPUs  float32
 	SandboxMemMB uint32
 
 	// Determines if we should attempt to size the VM at boot time and skip
@@ -683,11 +684,11 @@ func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig
 func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
 	numCPUs := goruntime.NumCPU()
 
-	if err := newAnnotationConfiguration(ocispec, vcAnnotations.DefaultVCPUs).setUintWithCheck(func(vcpus uint64) error {
-		if uint32(vcpus) > uint32(numCPUs) {
-			return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.DefaultVCPUs).setFloat32WithCheck(func(vcpus float32) error {
+		if vcpus > float32(numCPUs) {
+			return fmt.Errorf("Number of cpus %f specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
 		}
-		sbConfig.HypervisorConfig.NumVCPUs = uint32(vcpus)
+		sbConfig.HypervisorConfig.NumVCPUsF = float32(vcpus)
 		return nil
 	}); err != nil {
 		return err
@@ -1016,10 +1017,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 	// with the base number of CPU/memory (which is equal to the default CPU/memory specified for the runtime
 	// configuration or annotations) as well as any specified workload resources.
 	if sandboxConfig.StaticResourceMgmt {
-		sandboxConfig.SandboxResources.BaseCPUs = sandboxConfig.HypervisorConfig.NumVCPUs
+		sandboxConfig.SandboxResources.BaseCPUs = sandboxConfig.HypervisorConfig.NumVCPUsF
 		sandboxConfig.SandboxResources.BaseMemMB = sandboxConfig.HypervisorConfig.MemorySize
 
-		sandboxConfig.HypervisorConfig.NumVCPUs += sandboxConfig.SandboxResources.WorkloadCPUs
+		sandboxConfig.HypervisorConfig.NumVCPUsF += sandboxConfig.SandboxResources.WorkloadCPUs
 		sandboxConfig.HypervisorConfig.MemorySize += sandboxConfig.SandboxResources.WorkloadMemMB
 
 		ociLog.WithFields(logrus.Fields{
@@ -1140,6 +1141,7 @@ func IsCRIOContainerManager(spec *specs.Spec) bool {
 const (
 	errAnnotationPositiveNumericKey = "Error parsing annotation for %s: Please specify positive numeric value"
 	errAnnotationBoolKey            = "Error parsing annotation for %s: Please specify boolean value 'true|false'"
+	errAnnotationNumericKeyIsTooBig = "Error parsing annotation for %s: The number exceeds the maximum allowed for its type"
 )
 
 type annotationConfiguration struct {
@@ -1183,9 +1185,24 @@ func (a *annotationConfiguration) setUintWithCheck(f func(uint64) error) error {
 	return nil
 }
 
+func (a *annotationConfiguration) setFloat32WithCheck(f func(float32) error) error {
+	if value, ok := a.ocispec.Annotations[a.key]; ok {
+		float64Value, err := strconv.ParseFloat(value, 32)
+		if err != nil || float64Value < 0 {
+			return fmt.Errorf(errAnnotationPositiveNumericKey, a.key)
+		}
+		if float64Value > math.MaxFloat32 {
+			return fmt.Errorf(errAnnotationNumericKeyIsTooBig, a.key)
+		}
+		float32Value := float32(float64Value)
+		return f(float32Value)
+	}
+	return nil
+}
+
 // CalculateSandboxSizing will calculate the number of CPUs and amount of Memory that should
 // be added to the VM if sandbox annotations are provided with this sizing details
-func CalculateSandboxSizing(spec *specs.Spec) (numCPU, memSizeMB uint32) {
+func CalculateSandboxSizing(spec *specs.Spec) (numCPU float32, memSizeMB uint32) {
 	var memory, quota int64
 	var period uint64
 	var err error
@@ -1232,7 +1249,7 @@ func CalculateSandboxSizing(spec *specs.Spec) (numCPU, memSizeMB uint32) {
 
 // CalculateContainerSizing will calculate the number of CPUs and amount of memory that is needed
 // based on the provided LinuxResources
-func CalculateContainerSizing(spec *specs.Spec) (numCPU, memSizeMB uint32) {
+func CalculateContainerSizing(spec *specs.Spec) (numCPU float32, memSizeMB uint32) {
 	var memory, quota int64
 	var period uint64
 
@@ -1254,8 +1271,8 @@ func CalculateContainerSizing(spec *specs.Spec) (numCPU, memSizeMB uint32) {
 	return calculateVMResources(period, quota, memory)
 }
 
-func calculateVMResources(period uint64, quota int64, memory int64) (numCPU, memSizeMB uint32) {
-	numCPU = vcutils.CalculateVCpusFromMilliCpus(vcutils.CalculateMilliCPUs(quota, period))
+func calculateVMResources(period uint64, quota int64, memory int64) (numCPU float32, memSizeMB uint32) {
+	numCPU = vcutils.CalculateCPUsF(quota, period)
 
 	if memory < 0 {
 		// While spec allows for a negative value to indicate unconstrained, we don't

src/runtime/pkg/oci/utils_test.go

@@ -671,7 +671,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	err := addAnnotations(ocispec, &sbConfig, runtimeConfig)
 	assert.NoError(err)
 
-	assert.Equal(sbConfig.HypervisorConfig.NumVCPUs, uint32(1))
+	assert.Equal(sbConfig.HypervisorConfig.NumVCPUsF, float32(1))
 	assert.Equal(sbConfig.HypervisorConfig.DefaultMaxVCPUs, uint32(1))
 	assert.Equal(sbConfig.HypervisorConfig.MemorySize, uint32(1024))
 	assert.Equal(sbConfig.HypervisorConfig.MemSlots, uint32(20))
@@ -1087,7 +1087,7 @@ func TestCalculateContainerSizing(t *testing.T) {
 
 	testCases := []struct {
 		spec        *specs.Spec
-		expectedCPU uint32
+		expectedCPU float32
 		expectedMem uint32
 	}{
 		{
@@ -1152,7 +1152,7 @@ func TestCalculateSandboxSizing(t *testing.T) {
 
 	testCases := []struct {
 		spec        *specs.Spec
-		expectedCPU uint32
+		expectedCPU float32
 		expectedMem uint32
 	}{
 		{

src/runtime/virtcontainers/acrn_test.go

@@ -25,7 +25,7 @@ func newAcrnConfig() HypervisorConfig {
 		ImagePath:         testAcrnImagePath,
 		HypervisorPath:    testAcrnPath,
 		HypervisorCtlPath: testAcrnCtlPath,
-		NumVCPUs:          defaultVCPUs,
+		NumVCPUsF:         defaultVCPUs,
 		MemorySize:        defaultMemSzMiB,
 		BlockDeviceDriver: config.VirtioBlock,
 		DefaultBridges:    defaultBridges,

src/runtime/virtcontainers/clh.go

@@ -520,7 +520,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 		clh.vmconfig.Memory.HotplugSize = func(i int64) *int64 { return &i }(int64((utils.MemUnit(hotplugSize) * utils.MiB).ToBytes()))
 	}
 	// Set initial amount of cpu's for the virtual machine
-	clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs), int32(clh.config.DefaultMaxVCPUs))
+	clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs()), int32(clh.config.DefaultMaxVCPUs))
 
 	params, err := GetKernelRootParams(hypervisorConfig.RootfsType, clh.config.ConfidentialGuest, false)
 	if err != nil {
@@ -855,7 +855,7 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro
 		clhDisk.Direct = &clh.config.BlockDeviceCacheDirect
 	}
 
-	queues := int32(clh.config.NumVCPUs)
+	queues := int32(clh.config.NumVCPUs())
 	queueSize := int32(1024)
 	clhDisk.NumQueues = &queues
 	clhDisk.QueueSize = &queueSize

src/runtime/virtcontainers/clh_test.go

@@ -56,7 +56,7 @@ func newClhConfig() (HypervisorConfig, error) {
 		ImagePath:                     testClhImagePath,
 		RootfsType:                    string(EXT4),
 		HypervisorPath:                testClhPath,
-		NumVCPUs:                      defaultVCPUs,
+		NumVCPUsF:                     defaultVCPUs,
 		BlockDeviceDriver:             config.VirtioBlock,
 		MemorySize:                    defaultMemSzMiB,
 		DefaultBridges:                defaultBridges,

src/runtime/virtcontainers/factory/factory_linux.go

@@ -71,7 +71,7 @@ func NewFactory(ctx context.Context, config Config, fetchOnly bool) (vc.Factory,
 }
 
 func resetHypervisorConfig(config *vc.VMConfig) {
-	config.HypervisorConfig.NumVCPUs = 0
+	config.HypervisorConfig.NumVCPUsF = 0
 	config.HypervisorConfig.MemorySize = 0
 	config.HypervisorConfig.BootToBeTemplate = false
 	config.HypervisorConfig.BootFromTemplate = false
@@ -156,8 +156,8 @@ func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error)
 
 	online := false
 	baseConfig := f.base.Config().HypervisorConfig
-	if baseConfig.NumVCPUs < hypervisorConfig.NumVCPUs {
-		err = vm.AddCPUs(ctx, hypervisorConfig.NumVCPUs-baseConfig.NumVCPUs)
+	if baseConfig.NumVCPUsF < hypervisorConfig.NumVCPUsF {
+		err = vm.AddCPUs(ctx, hypervisorConfig.NumVCPUs()-baseConfig.NumVCPUs())
 		if err != nil {
 			return nil, err
 		}

src/runtime/virtcontainers/factory/factory_test.go

@@ -246,7 +246,7 @@ func TestFactoryGetVM(t *testing.T) {
 	assert.Nil(err)
 
 	// CPU hotplug
-	vmConfig.HypervisorConfig.NumVCPUs++
+	vmConfig.HypervisorConfig.NumVCPUsF++
 	vm, err = f.GetVM(ctx, vmConfig)
 	assert.Nil(err)
 
@@ -278,9 +278,9 @@ func TestDeepCompare(t *testing.T) {
 	bar := vc.VMConfig{}
 	assert.True(utils.DeepCompare(foo, bar))
 
-	foo.HypervisorConfig.NumVCPUs = 1
+	foo.HypervisorConfig.NumVCPUsF = 1
 	assert.False(utils.DeepCompare(foo, bar))
-	bar.HypervisorConfig.NumVCPUs = 1
+	bar.HypervisorConfig.NumVCPUsF = 1
 	assert.True(utils.DeepCompare(foo, bar))
 
 	// slice

src/runtime/virtcontainers/fc.go

@@ -692,7 +692,7 @@ func (fc *firecracker) fcInitConfiguration(ctx context.Context) error {
 	}
 
 	fc.fcSetVMBaseConfig(ctx, int64(fc.config.MemorySize),
-		int64(fc.config.NumVCPUs), false)
+		int64(fc.config.NumVCPUs()), false)
 
 	kernelPath, err := fc.config.KernelAssetPath()
 	if err != nil {