From e9f4c26bb4c8c0f26858d60b3787c7a9f312b9fe Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Sat, 8 Feb 2025 03:12:10 +0000 Subject: [PATCH] feature: add support for specifying pod's routes by subnet's .spec.routes Signed-off-by: zhangzujian --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 18 ++++++ dist/images/install.sh | 18 ++++++ pkg/apis/kubeovn/v1/subnet.go | 7 +++ pkg/daemon/handler.go | 22 +++---- test/e2e/kube-ovn/subnet/subnet.go | 69 +++++++++++++++++++++ 5 files changed, 123 insertions(+), 11 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index cab46f09965..4685f7f221b 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -2461,6 +2461,24 @@ spec: type: array items: type: string + routes: + description: | + Routes is a list of route rules for the pods which are in the subnet. + If specified, the routes will be added/replaced to the pod's network namespace. + type: array + items: + type: object + properties: + dst: + type: string + format: cidr + gw: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + required: + - gw gatewayType: type: string allowSubnets: diff --git a/dist/images/install.sh b/dist/images/install.sh index 2db55c7810b..0471920774d 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -2712,6 +2712,24 @@ spec: type: array items: type: string + routes: + description: | + Routes is a list of route rules for the pods which are in the subnet. + If specified, the routes will be added/replaced to the pod's network namespace. + type: array + items: + type: object + properties: + dst: + type: string + format: cidr + gw: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + required: + - gw gatewayType: type: string allowSubnets: diff --git a/pkg/apis/kubeovn/v1/subnet.go b/pkg/apis/kubeovn/v1/subnet.go index 15ab98bdfaf..043f90be165 100644 --- a/pkg/apis/kubeovn/v1/subnet.go +++ b/pkg/apis/kubeovn/v1/subnet.go @@ -7,6 +7,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/request" ) const ( @@ -49,6 +51,11 @@ type SubnetSpec struct { ExcludeIps []string `json:"excludeIps,omitempty"` Provider string `json:"provider,omitempty"` + // Routes is a list of route rules for the pods which are in the subnet. + // If specified, the routes will be added/replaced to the pod's network namespace. + // +optional + Routes []request.Route `json:"routes,omitempty"` + GatewayType string `json:"gatewayType,omitempty"` GatewayNode string `json:"gatewayNode"` NatOutgoing bool `json:"natOutgoing"` diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go index 86c7d0d1458..c59eedd05b9 100644 --- a/pkg/daemon/handler.go +++ b/pkg/daemon/handler.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "slices" "strconv" "strings" "time" @@ -226,19 +227,17 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon return } - var mtu int - routes = append(podRequest.Routes, routes...) - if strings.HasSuffix(podRequest.Provider, util.OvnProvider) && subnet != "" { - podSubnet, err := csh.Controller.subnetsLister.Get(subnet) - if err != nil { - errMsg := fmt.Errorf("failed to get subnet %s: %w", subnet, err) - klog.Error(errMsg) - if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()}); err != nil { - klog.Errorf("failed to write response: %v", err) - } - return + if podSubnet, err = csh.Controller.subnetsLister.Get(subnet); err != nil { + errMsg := fmt.Errorf("failed to get subnet %q: %w", subnet, err) + klog.Error(errMsg) + if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()}); err != nil { + klog.Errorf("failed to write response: %v", err) } + return + } + var mtu int + if strings.HasSuffix(podRequest.Provider, util.OvnProvider) { if podSubnet.Status.U2OInterconnectionIP == "" && podSubnet.Spec.U2OInterconnection { errMsg := fmt.Errorf("failed to generate u2o ip on subnet %s", podSubnet.Name) klog.Error(errMsg) @@ -303,6 +302,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon } } + routes = slices.Concat(podRequest.Routes, podSubnet.Spec.Routes, routes) macAddr = pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podRequest.Provider)] klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, routes) podNicName = ifName diff --git a/test/e2e/kube-ovn/subnet/subnet.go b/test/e2e/kube-ovn/subnet/subnet.go index 991f41bb509..f780a4fd109 100644 --- a/test/e2e/kube-ovn/subnet/subnet.go +++ b/test/e2e/kube-ovn/subnet/subnet.go @@ -13,6 +13,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" @@ -21,6 +22,7 @@ import ( "github.com/onsi/gomega" apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/request" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/test/e2e/framework" "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" @@ -289,6 +291,73 @@ var _ = framework.Describe("[group:subnet]", func() { } }) + framework.ConformanceIt(`should be able to configure pod routes by subnet's ".spec.routes" field`, func() { + subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) + ginkgo.By("Creating subnet " + subnetName + " with cidr " + subnet.Spec.CIDRBlock) + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By(`Constructing specified routes by subnet's ".spec.routes" field`) + var routeDst string + for i := 0; i < 3; i++ { + routeDst = framework.RandomCIDR(f.ClusterIPFamily) + if routeDst != subnet.Spec.CIDRBlock { + break + } + } + framework.ExpectNotEqual(routeDst, subnet.Spec.CIDRBlock) + routeGw := framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1) + ipv4Gateway, ipv6Gateway := util.SplitStringIP(subnet.Spec.Gateway) + ipv4RouteDst, ipv6RouteDst := util.SplitStringIP(routeDst) + ipv4RouteGw, ipv6RouteGw := util.SplitStringIP(routeGw) + routes := make([]request.Route, 0, 4) + if f.HasIPv4() { + routes = append(routes, request.Route{Gateway: ipv4RouteGw}) + routes = append(routes, request.Route{Destination: ipv4RouteDst, Gateway: ipv4Gateway}) + + } + if f.HasIPv6() { + routes = append(routes, request.Route{Gateway: ipv6RouteGw}) + routes = append(routes, request.Route{Destination: ipv6RouteDst, Gateway: ipv6Gateway}) + } + + ginkgo.By("Updating subnet " + subnetName + " with routes " + fmt.Sprintf("%v", routes)) + subnet.Spec.Routes = routes + subnet = subnetClient.Update(subnet, metav1.UpdateOptions{}, 2*time.Second) + + ginkgo.By("Creating pod " + podName) + cmd := []string{"sleep", "infinity"} + annotations := map[string]string{util.LogicalSwitchAnnotation: subnetName} + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) + _ = podClient.CreateSync(pod) + + ginkgo.By("Retrieving pod routes") + podRoutes, err := iproute.RouteShow("", "", func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(namespaceName, podName, cmd...) + }) + framework.ExpectNoError(err) + + ginkgo.By("Validating pod routes") + actualRoutes := make([]request.Route, 0, len(podRoutes)) + for _, r := range podRoutes { + if r.Gateway != "" || r.Dst != "" { + actualRoutes = append(actualRoutes, request.Route{Destination: r.Dst, Gateway: r.Gateway}) + } + } + ipv4CIDR, ipv6CIDR := util.SplitStringIP(subnet.Spec.CIDRBlock) + if f.HasIPv4() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv4Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv4RouteGw}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4RouteDst, Gateway: ipv4Gateway}) + } + if f.HasIPv6() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv6CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv6Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv4RouteGw}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4RouteDst, Gateway: ipv6Gateway}) + } + }) + framework.ConformanceIt("should create subnet with centralized gateway", func() { ginkgo.By("Getting nodes") nodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs)