Skip to content

Implement ServiceLoadBalancerStatus CRD for GKE L4 Load Balancers #908

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Implement ServiceLoadBalancerStatus CRD for GKE L4 Load Balancers #908

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion cmd/cloud-controller-manager/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider-gcp/providers/gce"
_ "k8s.io/cloud-provider-gcp/providers/gce"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/names"
Expand Down Expand Up @@ -67,6 +66,10 @@ var enableDiscretePortForwarding bool
// LoadBalancerClass
var enableRBSDefaultForL4NetLB bool

// enableServiceLBStatusCR is bound to a command-line flag. When true, it enables
// the Service Load Balancer Status CRD support in GCE cloud provider.
var enableServiceLBStatusCR bool

func main() {
rand.Seed(time.Now().UnixNano())

Expand All @@ -85,6 +88,7 @@ func main() {
cloudProviderFS.BoolVar(&enableMultiProject, "enable-multi-project", false, "Enables project selection from Node providerID for GCE API calls. CAUTION: Only enable if Node providerID is configured by a trusted source.")
cloudProviderFS.BoolVar(&enableDiscretePortForwarding, "enable-discrete-port-forwarding", false, "Enables forwarding of individual ports instead of port ranges for GCE external load balancers.")
cloudProviderFS.BoolVar(&enableRBSDefaultForL4NetLB, "enable-rbs-default-l4-netlb", false, "Enables RBS defaulting for GCE L4 NetLB")
cloudProviderFS.BoolVar(&enableServiceLBStatusCR, "enable-service-lb-status-cr", false, "Enables Service LB Status CR for GCE services")

// add new controllers and initializers
nodeIpamController := nodeIPAMController{}
Expand Down Expand Up @@ -172,5 +176,20 @@ func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
gceCloud.SetEnableRBSDefaultForL4NetLB(true)
}

if enableServiceLBStatusCR {
gceCloud, ok := (cloud).(*gce.Cloud)
if !ok {
// Fail-fast: If enableServiceLBStatusCR is set, the cloud
// provider MUST be GCE.
klog.Fatalf("enable-service-lb-status-cr requires GCE cloud provider, but got %T", cloud)
}

err := gceCloud.InitializeServiceLoadBalancerStatusCRD(config.Kubeconfig)
if err != nil {
klog.Fatalf("could not initialize service lb status crd")
}
klog.Info("Service Load Balancer Status CRD support enabled")
}

return cloud
}
4 changes: 3 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ require (
)

require (
github.com/GoogleCloudPlatform/gke-networking-api v0.1.2-0.20240904205008-bc15495fd43f
github.com/GoogleCloudPlatform/gke-networking-api v0.2.1-0.20250318085121-e88f4ed9f50a
github.com/hashicorp/go-multierror v1.1.1
github.com/natefinch/atomic v1.0.1
k8s.io/cloud-provider v0.34.0
Expand Down Expand Up @@ -179,3 +179,5 @@ replace (
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.34.0
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.34.0
)

replace github.com/GoogleCloudPlatform/gke-networking-api => github.com/08volt/gke-networking-api v0.2.1-0.20251009151344-cc8d454f80f8
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy
cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I=
cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg=
github.com/08volt/gke-networking-api v0.2.1-0.20251009151344-cc8d454f80f8 h1:98IauCLpRFPXUq2zCte2aVgZ1MoCOU+kzttkAuLdgWc=
github.com/08volt/gke-networking-api v0.2.1-0.20251009151344-cc8d454f80f8/go.mod h1:OA2rutbmP2GgHGrgeCe/2WYEYRzwFsHy6xDV9Z0g5m8=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/GoogleCloudPlatform/gke-networking-api v0.1.2-0.20240904205008-bc15495fd43f h1:OPE+MYCwdQNms+QAXgHjR03TKCHXoqlurXiuhYDuEdI=
github.com/GoogleCloudPlatform/gke-networking-api v0.1.2-0.20240904205008-bc15495fd43f/go.mod h1:YnoYXo/cwpqFmIXKblHOV5jFEpsSL3PZeo0zaR3oGTI=
github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 h1:lwL1vLWmdBJ5h+StMEN6+GMz1J/Y0yUU3RDv+QBy+Q4=
github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0/go.mod h1:UTfhBnADaj2rybPT049NScSh7Eall3u2ib43wmz3deg=
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
Expand Down
12 changes: 12 additions & 0 deletions providers/gce/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ go_library(
"gce_networks.go",
"gce_routes.go",
"gce_securitypolicy.go",
"gce_serviceloadbalancerstatus.go",
"gce_subnetworks.go",
"gce_targetpool.go",
"gce_targetproxy.go",
Expand All @@ -45,6 +46,8 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//vendor/cloud.google.com/go/compute/metadata",
"//vendor/github.com/GoogleCloudPlatform/gke-networking-api/apis/serviceloadbalancerstatus/v1:serviceloadbalancerstatus",
"//vendor/github.com/GoogleCloudPlatform/gke-networking-api/client/serviceloadbalancerstatus/clientset/versioned",
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud",
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/filter",
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta",
Expand All @@ -61,6 +64,7 @@ go_library(
"//vendor/google.golang.org/api/tpu/v1:tpu",
"//vendor/gopkg.in/gcfg.v1:gcfg_v1",
"//vendor/k8s.io/api/core/v1:core",
"//vendor/k8s.io/apimachinery/pkg/api/errors",
"//vendor/k8s.io/apimachinery/pkg/api/resource",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta",
"//vendor/k8s.io/apimachinery/pkg/fields",
Expand All @@ -78,6 +82,7 @@ go_library(
"//vendor/k8s.io/client-go/kubernetes/scheme",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:core",
"//vendor/k8s.io/client-go/pkg/version",
"//vendor/k8s.io/client-go/rest",
"//vendor/k8s.io/client-go/tools/cache",
"//vendor/k8s.io/client-go/tools/record",
"//vendor/k8s.io/client-go/util/flowcontrol",
Expand Down Expand Up @@ -105,12 +110,14 @@ go_test(
"gce_loadbalancer_metrics_test.go",
"gce_loadbalancer_test.go",
"gce_loadbalancer_utils_test.go",
"gce_serviceloadbalancerstatus_test.go",
"gce_test.go",
"gce_util_test.go",
"metrics_test.go",
],
embed = [":gce"],
deps = [
"//vendor/github.com/GoogleCloudPlatform/gke-networking-api/apis/serviceloadbalancerstatus/v1:serviceloadbalancerstatus",
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud",
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta",
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock",
Expand All @@ -123,15 +130,20 @@ go_test(
"//vendor/google.golang.org/api/compute/v1:compute",
"//vendor/google.golang.org/api/googleapi",
"//vendor/k8s.io/api/core/v1:core",
"//vendor/k8s.io/apimachinery/pkg/api/errors",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta",
"//vendor/k8s.io/apimachinery/pkg/runtime",
"//vendor/k8s.io/apimachinery/pkg/types",
"//vendor/k8s.io/apimachinery/pkg/util/intstr",
"//vendor/k8s.io/apimachinery/pkg/util/json",
"//vendor/k8s.io/apimachinery/pkg/util/sets",
"//vendor/k8s.io/client-go/rest",
"//vendor/k8s.io/client-go/testing",
"//vendor/k8s.io/client-go/tools/record",
"//vendor/k8s.io/cloud-provider",
"//vendor/k8s.io/cloud-provider/service/helpers",
"//vendor/k8s.io/utils/net",
"//vendor/k8s.io/utils/strings/slices",
],
)

Expand Down
12 changes: 12 additions & 0 deletions providers/gce/gce.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"time"

gcfg "gopkg.in/gcfg.v1"
restclient "k8s.io/client-go/rest"

"cloud.google.com/go/compute/metadata"
"golang.org/x/oauth2"
Expand All @@ -43,6 +44,7 @@ import (

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"

svclbstatusclient "github.com/GoogleCloudPlatform/gke-networking-api/client/serviceloadbalancerstatus/clientset/versioned"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/informers"
Expand Down Expand Up @@ -209,6 +211,9 @@ type Cloud struct {

// enableRBSDefaultForL4NetLB disable Service controller from picking up services by default
enableRBSDefaultForL4NetLB bool

// enableServiceLBStatusCR enables the Service Load Balancer Status CRD support in GCE cloud provider
serviceLBStatusClient svclbstatusclient.Interface
}

// ConfigGlobal is the in memory representation of the gce.conf config data
Expand Down Expand Up @@ -870,6 +875,13 @@ func (g *Cloud) SetEnableRBSDefaultForL4NetLB(enabled bool) {
g.enableRBSDefaultForL4NetLB = enabled
}

func (g *Cloud) SetServiceLoadBalancerStatusCRDClient(kubeConfig *restclient.Config) {
client, err := svclbstatusclient.NewForConfig(kubeConfig)
if err != nil {
g.serviceLBStatusClient = client
}
}

// getProjectsBasePath returns the compute API endpoint with the `projects/` element.
// The suffix must be added when generating compute resource urls.
func getProjectsBasePath(basePath string) string {
Expand Down
11 changes: 9 additions & 2 deletions providers/gce/gce_loadbalancer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,16 +197,23 @@ func (g *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, svc
}

var status *v1.LoadBalancerStatus
var resourcesURLs []string
switch desiredScheme {
case cloud.SchemeInternal:
status, err = g.ensureInternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
status, resourcesURLs, err = g.ensureInternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
default:
status, err = g.ensureExternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
status, resourcesURLs, err = g.ensureExternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
}
if err != nil {
klog.Errorf("Failed to EnsureLoadBalancer(%s, %s, %s, %s, %s), err: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, err)
return status, err
}
klog.V(4).Infof("EnsureLoadBalancer(%s, %s, %s, %s, %s): done ensuring loadbalancer resources: %v", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region, resourcesURLs)
err = g.EnsureServiceLoadBalancerStatusCR(svc, resourcesURLs)
if err != nil {
klog.Errorf("Failed to update ServiceLoadBalancerStatus CR, err: %v", err)
return status, err
}
klog.V(4).Infof("EnsureLoadBalancer(%s, %s, %s, %s, %s): done ensuring loadbalancer.", clusterName, svc.Namespace, svc.Name, loadBalancerName, g.region)
return status, err
}
Expand Down
Loading