Skip to content

Commit 0ddd782

Browse files
cyclindercyclinder
committed
webhook helps update pod's multus annotations with resourceClaim
1 parent c7b4ceb commit 0ddd782

File tree

7 files changed

+190
-82
lines changed

7 files changed

+190
-82
lines changed

‎cmd/spiderpool-controller/cmd/daemon.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ func initControllerServiceManagers(ctx context.Context) {
265265
}
266266
controllerContext.PodManager = podManager
267267

268-
if controllerContext.Cfg.PodResourceInjectConfig.Enabled {
268+
if controllerContext.Cfg.PodResourceInjectConfig.Enabled || controllerContext.Cfg.PodResourceInjectConfig.EnabledDRAWebhook {
269269
logger.Info("Begin to init Pod MutatingWebhook")
270270
if err := podmanager.InitPodWebhook(controllerContext.CRDManager); err != nil {
271271
logger.Fatal(err.Error())

‎pkg/dra/config.go

Lines changed: 39 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,14 @@ import (
77
"strings"
88
"unicode"
99

10-
"github.com/spidernet-io/spiderpool/pkg/constant"
1110
resourcev1beta1 "k8s.io/api/resource/v1beta1"
1211
)
1312

13+
type ParameterConfig struct {
14+
PodDefaultRouteNic string `json:"podDefaultRouteNic"`
15+
MultusNamaspace string `json:"multusNamespace"`
16+
}
17+
1418
type NetworkConfig struct {
1519
// MultusNamespace is the namespace where the MultusConfig CRs are located
1620
MultusNamespace string `json:"multusNamespace"`
@@ -70,43 +74,42 @@ func ParseNetworkConfig(configs []resourcev1beta1.DeviceClaimConfiguration) (*Ne
7074
return multusConfig, nil
7175
}
7276

73-
func ParseToAnnotations(annotations map[string]string) {
74-
if annotations == nil {
75-
annotations = make(map[string]string)
76-
}
77-
78-
if d.DefaultNic != nil {
79-
annotations[constant.MultusDefaultNetAnnot] = MultusAnnotationValue(d.MultusNamespace, d.DefaultNic.MultusName)
80-
}
81-
82-
if d.SecondaryNics != nil {
83-
for idx, nic := range d.SecondaryNics.StaticNics {
84-
if nic == nil {
85-
continue
86-
}
87-
// by default, the default route is locatee at the first nic of the pod(eth0).
88-
// we can configure the default route to the second nics of the pod via annotations
89-
// e.g.
90-
// annotations:
91-
// ipam.spidernet.io/default-route-nic: net1
92-
// In multus, the multi-nic is formatted as "net1", "net2", etc.
93-
// Note: we expect only one nic to be the default route, if we configure DefaultRoute to
94-
// true for multi-nic, the first nic only be selected.
95-
if nic.DefaultRoute && annotations[constant.AnnoDefaultRouteInterface] == "" {
96-
annotations[constant.AnnoDefaultRouteInterface] = fmt.Sprintf("net%d", idx+1)
97-
}
98-
99-
if idx == 0 {
100-
annotations[constant.MultusNetworkAttachmentAnnot] = MultusAnnotationValue(d.MultusNamespace, nic.MultusName)
101-
continue
102-
}
103-
annotations[constant.MultusNetworkAttachmentAnnot] = annotations[constant.MultusNetworkAttachmentAnnot] + "," + MultusAnnotationValue(d.MultusNamespace, nic.MultusName)
104-
}
105-
}
106-
}
77+
// func ParseToAnnotations(annotations map[string]string) {
78+
// if annotations == nil {
79+
// annotations = make(map[string]string)
80+
// }
81+
82+
// if d.DefaultNic != nil {
83+
// annotations[constant.MultusDefaultNetAnnot] = MultusAnnotationValue(d.MultusNamespace, d.DefaultNic.MultusName)
84+
// }
85+
86+
// if d.SecondaryNics != nil {
87+
// for idx, nic := range d.SecondaryNics.StaticNics {
88+
// if nic == nil {
89+
// continue
90+
// }
91+
// // by default, the default route is locatee at the first nic of the pod(eth0).
92+
// // we can configure the default route to the second nics of the pod via annotations
93+
// // e.g.
94+
// // annotations:
95+
// // ipam.spidernet.io/default-route-nic: net1
96+
// // In multus, the multi-nic is formatted as "net1", "net2", etc.
97+
// // Note: we expect only one nic to be the default route, if we configure DefaultRoute to
98+
// // true for multi-nic, the first nic only be selected.
99+
// if nic.DefaultRoute && annotations[constant.AnnoDefaultRouteInterface] == "" {
100+
// annotations[constant.AnnoDefaultRouteInterface] = fmt.Sprintf("net%d", idx+1)
101+
// }
102+
103+
// if idx == 0 {
104+
// annotations[constant.MultusNetworkAttachmentAnnot] = MultusAnnotationValue(d.MultusNamespace, nic.MultusName)
105+
// continue
106+
// }
107+
// annotations[constant.MultusNetworkAttachmentAnnot] = annotations[constant.MultusNetworkAttachmentAnnot] + "," + MultusAnnotationValue(d.MultusNamespace, nic.MultusName)
108+
// }
109+
// }
110+
// }
107111

108112
func (d *NetworkConfig) GetResourceNames() []string {
109-
110113
return nil
111114
}
112115

‎pkg/dra/device_state.go

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ func (d *DeviceState) GetNetDevices() []resourceapi.Device {
4343
d.logger.Sugar().Debugf("netdev %s is sriov vf, skip to add to resource slices", link.Attrs().Name)
4444
continue
4545
}
46-
4746
devices = append(devices, d.getNetDevice(link))
4847
}
4948
return devices
@@ -58,31 +57,23 @@ func (d *DeviceState) getNetDevice(link netlink.Link) resourceapi.Device {
5857
},
5958
}
6059

61-
linkAttrs := link.Attrs()
6260
// make sure the ifname is an valid dns1123 label, if not normalize it
6361
if len(validation.IsDNS1123Label(link.Attrs().Name)) > 0 {
6462
device.Name = NormalizedDNS1123Label(link.Attrs().Name)
6563
d.logger.Sugar().Debugf("iface %s is invalid DNS1123 label, normalized to %s", link.Attrs().Name, device.Name)
6664
}
6765

6866
d.addBasicAttributesForNetDev(link, device.Basic)
69-
70-
// TODO(@cyclinder): gpu topo attributes
71-
device.Basic.Attributes["PIXAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
72-
device.Basic.Attributes["PHBAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
73-
device.Basic.Attributes["SYSAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
74-
device.Basic.Attributes["NODEAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
75-
67+
d.addGPUAffinityAttributesForNetDev(link.Attrs().Name, device.Basic)
7668
// pci attributes
77-
d.addPciInfoAttributesForNetDev(link.Attrs().Name, device.Basic)
78-
69+
d.addPCIAttributesForNetDev(link.Attrs().Name, device.Basic)
7970
// bandwidth attributes
8071
d.addBandwidthAttributesForNetDev(link.Attrs().Name, device.Basic)
81-
72+
d.addSpiderMultusConfigAttributesForNetDev(link.Attrs().Name, device.Basic)
8273
return device
8374
}
8475

85-
func (d *DeviceState) addPciInfoAttributesForNetDev(iface string, device *resourceapi.BasicDevice) {
76+
func (d *DeviceState) addPCIAttributesForNetDev(iface string, device *resourceapi.BasicDevice) {
8677
// get vendor id, device id and pci address from sysfs
8778
deviceId, err := networking.GetPciDeviceIdForNetDev(iface)
8879
if err != nil {
@@ -178,18 +169,27 @@ func (d *DeviceState) addIPAddressAttributesForNetDev(link netlink.Link, device
178169
}
179170

180171
func (d *DeviceState) addBandwidthAttributesForNetDev(iface string, device *resourceapi.BasicDevice) {
181-
speed, err := networking.GetNetdevBandwidth(iface)
172+
bandwidth, err := networking.GetNetdevBandwidth(iface)
182173
if err != nil {
183174
d.logger.Sugar().Debugf("Failed to get bandwidth for netdev %s: %v", iface, err)
184175
// Set default values if we can't get the real bandwidth
185176
device.Attributes["speed"] = resourceapi.DeviceAttribute{IntValue: ptr.To(int64(0))}
186177
return
187178
}
188179

189-
// Store speed in Mbps
190-
device.Attributes["speed"] = resourceapi.DeviceAttribute{IntValue: ptr.To(int64(speed))}
191-
192180
// Calculate bandwidth based on speed and duplex mode
193-
bandwidth := speed
194181
device.Attributes["bandwidth"] = resourceapi.DeviceAttribute{IntValue: ptr.To(int64(bandwidth))}
195182
}
183+
184+
func (d *DeviceState) addGPUAffinityAttributesForNetDev(iface string, device *resourceapi.BasicDevice) {
185+
// TODO(@cyclinder): gpu topo attributes
186+
device.Attributes["PIXAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
187+
device.Attributes["PHBAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
188+
device.Attributes["SYSAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
189+
device.Attributes["NODEAffinityGpus"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
190+
}
191+
192+
func (d *DeviceState) addSpiderMultusConfigAttributesForNetDev(iface string, device *resourceapi.BasicDevice) {
193+
// TODO(@cyclinder): spider multus config attributes
194+
device.Attributes["multusConfigRefs"] = resourceapi.DeviceAttribute{StringValue: ptr.To("")}
195+
}

‎pkg/dra/driver.go

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package dra
22

33
import (
44
"context"
5-
"encoding/json"
65
"fmt"
76
"os"
87
"time"
@@ -106,22 +105,21 @@ func (d *driver) nodePrepareResource(ctx context.Context, claim *drapb.Claim) (d
106105
// get the current pod
107106
// we expect one resourceClaim is only reserved for one pod
108107
// so we only need to get the first pod
109-
var pod *corev1.Pod
110-
for _, reserved := range resourceClaim.Status.ReservedFor {
111-
if reserved.APIGroup == "" && reserved.Resource == "pods" {
112-
pod, err = d.podManager.GetPodByName(ctx, claim.Namespace, reserved.Name, true)
113-
if err != nil {
114-
return nil, fmt.Errorf("failed to get pod '%s/%s': %v", claim.Namespace, reserved.Name, err)
115-
}
116-
break
117-
}
118-
}
119-
120-
d.prepareMultusConfigs()
108+
// var pod *corev1.Pod
109+
// for _, reserved := range resourceClaim.Status.ReservedFor {
110+
// if reserved.APIGroup == "" && reserved.Resource == "pods" {
111+
// pod, err = d.podManager.GetPodByName(ctx, claim.Namespace, reserved.Name, true)
112+
// if err != nil {
113+
// return nil, fmt.Errorf("failed to get pod '%s/%s': %v", claim.Namespace, reserved.Name, err)
114+
// }
115+
// break
116+
// }
117+
// }
118+
119+
// d.prepareMultusConfigs()
121120

122121
d.prepare()
123122
return devices, nil
124-
125123
}
126124

127125
func (d *driver) prepare() error {
@@ -137,13 +135,6 @@ func (d *driver) prepareMultusConfigs(pod *corev1.Pod, configs []resourcev1beta1
137135
}
138136

139137
if multusConfig.SecondaryNics != nil {
140-
if len(multusConfig.SecondaryNics.StaticNics) > 0 {
141-
// staticNic mode
142-
multusConfig.ParseToAnnotations(pod.Annotations)
143-
} else if multusConfig.SecondaryNics.DynamicNics != nil {
144-
//TODO(@cyclinder): dynamicNic mode
145-
// d.prepareDynamicMultusConfigs(pod, multusConfig.SecondaryNics.DynamicNics)
146-
}
147138
}
148139

149140
return nil

‎pkg/podmanager/pod_webhook.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1414
"k8s.io/apimachinery/pkg/runtime"
1515
ctrl "sigs.k8s.io/controller-runtime"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
1617
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
1718
)
1819

@@ -32,6 +33,7 @@ type PodWebhook interface {
3233

3334
type PWebhook struct {
3435
spiderClient crdclientset.Interface
36+
client client.Client
3537
}
3638

3739
// InitPodWebhook initializes the pod webhook.
@@ -48,6 +50,7 @@ func InitPodWebhook(mgr ctrl.Manager) error {
4850

4951
pw := &PWebhook{
5052
spiderClient: spiderClient,
53+
client: mgr.GetClient(),
5154
}
5255

5356
// setup mutating webhook for pods
@@ -82,11 +85,11 @@ func (pw *PWebhook) Default(ctx context.Context, obj runtime.Object) error {
8285
}
8386
}
8487

85-
if !needInject {
88+
if !needInject && len(pod.Spec.ResourceClaims) == 0 {
8689
return nil
8790
}
8891

89-
err := podNetworkMutatingWebhook(pw.spiderClient, pod)
92+
err := podNetworkMutatingWebhook(pw.spiderClient, pw.client, pod)
9093
if err != nil {
9194
mutateLogger.Sugar().Errorf("Failed to inject network resources for pod %s/%s: %v", pod.Namespace, pod.GenerateName, err)
9295
return err

0 commit comments

Comments
 (0)