Skip to content

Commit

Permalink
feat: Webhook for reserved namespace resources part-1 (Azure#500)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Arvindthiru
Arvindthiru authored Sep 1, 2023
1 parent 48556dc commit ea45e68
Show file tree
Hide file tree
Showing 8 changed files with 1,322 additions and 230 deletions.
61 changes: 21 additions & 40 deletions pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (

admissionv1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -31,15 +30,13 @@ const (
)

var (
crdGVK = metav1.GroupVersionKind{Group: v1.SchemeGroupVersion.Group, Version: v1.SchemeGroupVersion.Version, Kind: "CustomResourceDefinition"}
mcGVK = metav1.GroupVersionKind{Group: fleetv1alpha1.GroupVersion.Group, Version: fleetv1alpha1.GroupVersion.Version, Kind: "MemberCluster"}
imcGVK = metav1.GroupVersionKind{Group: fleetv1alpha1.GroupVersion.Group, Version: fleetv1alpha1.GroupVersion.Version, Kind: "InternalMemberCluster"}
roleGVK = metav1.GroupVersionKind{Group: rbacv1.SchemeGroupVersion.Group, Version: rbacv1.SchemeGroupVersion.Version, Kind: "Role"}
roleBindingGVK = metav1.GroupVersionKind{Group: rbacv1.SchemeGroupVersion.Group, Version: rbacv1.SchemeGroupVersion.Version, Kind: "RoleBinding"}
namespaceGVK = metav1.GroupVersionKind{Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, Kind: "Namespace"}
crdGVK = metav1.GroupVersionKind{Group: v1.SchemeGroupVersion.Group, Version: v1.SchemeGroupVersion.Version, Kind: "CustomResourceDefinition"}
mcGVK = metav1.GroupVersionKind{Group: fleetv1alpha1.GroupVersion.Group, Version: fleetv1alpha1.GroupVersion.Version, Kind: "MemberCluster"}
imcGVK = metav1.GroupVersionKind{Group: fleetv1alpha1.GroupVersion.Group, Version: fleetv1alpha1.GroupVersion.Version, Kind: "InternalMemberCluster"}
namespaceGVK = metav1.GroupVersionKind{Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, Kind: "Namespace"}
)

// Add registers the webhook for K8s bulit-in object types.
// Add registers the webhook for K8s built-in object types.
func Add(mgr manager.Manager, whiteListedUsers []string) error {
hookServer := mgr.GetWebhookServer()
hookServer.Register(ValidationPath, &webhook.Admission{Handler: &fleetResourceValidator{client: mgr.GetClient(), whiteListedUsers: whiteListedUsers}})
Expand All @@ -54,28 +51,29 @@ type fleetResourceValidator struct {

// Handle receives the request then allows/denies the request to modify fleet resources.
func (v *fleetResourceValidator) Handle(ctx context.Context, req admission.Request) admission.Response {
// special case for Kind:Namespace resources req.Name and req.Namespace has the same value the ObjectMeta.Name of Namespace.
if req.Kind.Kind == "Namespace" {
req.Namespace = ""
}
namespacedName := types.NamespacedName{Name: req.Name, Namespace: req.Namespace}
var response admission.Response
if req.Operation == admissionv1.Create || req.Operation == admissionv1.Update || req.Operation == admissionv1.Delete {
switch req.Kind {
case crdGVK:
switch {
case req.Kind == crdGVK:
klog.V(2).InfoS("handling CRD resource", "GVK", crdGVK, "namespacedName", namespacedName, "operation", req.Operation)
response = v.handleCRD(req)
case mcGVK:
klog.V(2).InfoS("handling Member cluster resource", "GVK", mcGVK, "namespacedName", namespacedName, "operation", req.Operation)
case req.Kind == mcGVK:
klog.V(2).InfoS("handling member cluster resource", "GVK", mcGVK, "namespacedName", namespacedName, "operation", req.Operation)
response = v.handleMemberCluster(req)
case imcGVK:
klog.V(2).InfoS("handling Internal member cluster resource", "GVK", imcGVK, "namespacedName", namespacedName, "operation", req.Operation)
response = v.handleInternalMemberCluster(ctx, req)
case roleGVK:
klog.V(2).InfoS("handling Role resource", "GVK", roleGVK, "namespacedName", namespacedName, "operation", req.Operation)
response = v.handleRole(req)
case roleBindingGVK:
klog.V(2).InfoS("handling Role binding resource", "GVK", roleBindingGVK, "namespacedName", namespacedName, "operation", req.Operation)
response = v.handleRoleBinding(req)
case namespaceGVK:
case req.Kind == namespaceGVK:
klog.V(2).InfoS("handling namespace resource", "GVK", namespaceGVK, "namespacedName", namespacedName, "operation", req.Operation)
response = v.handleNamespace(req)
case req.Kind == imcGVK:
klog.V(2).InfoS("handling internal member cluster resource", "GVK", imcGVK, "namespacedName", namespacedName, "operation", req.Operation)
response = v.handleInternalMemberCluster(ctx, req)
case req.Namespace != "":
klog.V(2).InfoS(fmt.Sprintf("handling %s resource", req.Kind.Kind), "GVK", req.Kind, "namespacedName", namespacedName, "operation", req.Operation)
response = validation.ValidateUserForResource(req.Kind.Kind, types.NamespacedName{Name: req.Name, Namespace: req.Namespace}, v.whiteListedUsers, req.UserInfo)
default:
klog.V(2).InfoS("resource is not monitored by fleet resource validator webhook", "GVK", req.Kind.String(), "namespacedName", namespacedName, "operation", req.Operation)
response = admission.Allowed(fmt.Sprintf("user: %s in groups: %v is allowed to modify resource with GVK: %s", req.UserInfo.Username, req.UserInfo.Groups, req.Kind.String()))
Expand Down Expand Up @@ -127,24 +125,7 @@ func (v *fleetResourceValidator) handleInternalMemberCluster(ctx context.Context
return validation.ValidateUserForResource(currentIMC.Kind, types.NamespacedName{Name: currentIMC.Name, Namespace: currentIMC.Namespace}, v.whiteListedUsers, req.UserInfo)
}

// handleRole allows/denies the request to modify role after validation.
func (v *fleetResourceValidator) handleRole(req admission.Request) admission.Response {
var role rbacv1.Role
if err := v.decodeRequestObject(req, &role); err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
return validation.ValidateUserForResource(role.Kind, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, v.whiteListedUsers, req.UserInfo)
}

// handleRoleBinding allows/denies the request to modify role after validation.
func (v *fleetResourceValidator) handleRoleBinding(req admission.Request) admission.Response {
var rb rbacv1.RoleBinding
if err := v.decodeRequestObject(req, &rb); err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
return validation.ValidateUserForResource(rb.Kind, types.NamespacedName{Name: rb.Name, Namespace: rb.Namespace}, v.whiteListedUsers, req.UserInfo)
}

// handlerNamespace allows/denies request to modify namespace after validation.
func (v *fleetResourceValidator) handleNamespace(req admission.Request) admission.Response {
var currentNS corev1.Namespace
if err := v.decodeRequestObject(req, &currentNS); err != nil {
Expand Down
32 changes: 26 additions & 6 deletions pkg/webhook/validation/uservalidation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@ import (
)

const (
mastersGroup = "system:masters"
serviceAccountsGroup = "system:serviceaccounts"
serviceAccountFmt = "system:serviceaccount:fleet-system:%s"
mastersGroup = "system:masters"
serviceAccountsGroup = "system:serviceaccounts"
nodeGroup = "system:nodes"
kubeSchedulerUser = "system:kube-scheduler"
kubeControllerManagerUser = "system:kube-controller-manager"
serviceAccountFmt = "system:serviceaccount:fleet-system:%s"

imcStatusUpdateNotAllowedFormat = "user: %s in groups: %v is not allowed to update IMC status: %+v"
imcAllowedGetMCFailed = "user: %s in groups: %v is allowed to update IMC: %+v because we failed to get MC"
Expand All @@ -47,11 +50,11 @@ func ValidateUserForFleetCRD(group string, namespacedName types.NamespacedName,

// ValidateUserForResource checks to see if user is allowed to modify argued resource.
func ValidateUserForResource(resKind string, namespacedName types.NamespacedName, whiteListedUsers []string, userInfo authenticationv1.UserInfo) admission.Response {
if isMasterGroupUserOrWhiteListedUser(whiteListedUsers, userInfo) || isUserAuthenticatedServiceAccount(userInfo) {
klog.V(2).InfoS("user in groups is allowed to modify fleet resource", "user", userInfo.Username, "groups", userInfo.Groups, "kind", resKind, "namespacedName", namespacedName)
if isMasterGroupUserOrWhiteListedUser(whiteListedUsers, userInfo) || isUserAuthenticatedServiceAccount(userInfo) || isUserKubeScheduler(userInfo) || isUserKubeControllerManager(userInfo) || isNodeGroupUser(userInfo) {
klog.V(2).InfoS("user in groups is allowed to modify resource", "user", userInfo.Username, "groups", userInfo.Groups, "kind", resKind, "namespacedName", namespacedName)
return admission.Allowed(fmt.Sprintf(resourceAllowedFormat, userInfo.Username, userInfo.Groups, resKind, namespacedName))
}
klog.V(2).InfoS("user in groups is not allowed to modify fleet resource", "user", userInfo.Username, "groups", userInfo.Groups, "kind", resKind, "namespacedName", namespacedName)
klog.V(2).InfoS("user in groups is not allowed to modify resource", "user", userInfo.Username, "groups", userInfo.Groups, "kind", resKind, "namespacedName", namespacedName)
return admission.Denied(fmt.Sprintf(resourceDeniedFormat, userInfo.Username, userInfo.Groups, resKind, namespacedName))
}

Expand Down Expand Up @@ -112,6 +115,23 @@ func isUserAuthenticatedServiceAccount(userInfo authenticationv1.UserInfo) bool
return slices.Contains(userInfo.Groups, serviceAccountsGroup)
}

// isUserKubeScheduler returns true if user is kube-scheduler.
func isUserKubeScheduler(userInfo authenticationv1.UserInfo) bool {
// system:kube-scheduler user only belongs to system:authenticated group hence comparing username.
return userInfo.Username == kubeSchedulerUser
}

// isUserKubeControllerManager return true if user is kube-controller-manager.
func isUserKubeControllerManager(userInfo authenticationv1.UserInfo) bool {
// system:kube-controller-manager user only belongs to system:authenticated group hence comparing username.
return userInfo.Username == kubeControllerManagerUser
}

// isNodeGroupUser returns true if user belongs to system:nodes group.
func isNodeGroupUser(userInfo authenticationv1.UserInfo) bool {
return slices.Contains(userInfo.Groups, nodeGroup)
}

// isMemberClusterMapFieldUpdated return true if member cluster label is updated.
func isMapFieldUpdated(currentMCLabels, oldMCLabels map[string]string) bool {
return !reflect.DeepEqual(currentMCLabels, oldMCLabels)
Expand Down
18 changes: 18 additions & 0 deletions pkg/webhook/validation/uservalidation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,24 @@ func TestValidateUserForResource(t *testing.T) {
namespacedName: types.NamespacedName{Name: "test-role-binding", Namespace: "test-namespace"},
wantResponse: admission.Allowed(fmt.Sprintf(resourceAllowedFormat, "test-user", []string{serviceAccountsGroup}, "RoleBinding", types.NamespacedName{Name: "test-role-binding", Namespace: "test-namespace"})),
},
"allow user in system:node group": {
userInfo: authenticationv1.UserInfo{
Username: "test-user",
Groups: []string{nodeGroup},
},
resKind: "Pod",
namespacedName: types.NamespacedName{Name: "test-pod", Namespace: "test-namespace"},
wantResponse: admission.Allowed(fmt.Sprintf(resourceAllowedFormat, "test-user", []string{nodeGroup}, "Pod", types.NamespacedName{Name: "test-pod", Namespace: "test-namespace"})),
},
"allow system:kube-scheduler user": {
userInfo: authenticationv1.UserInfo{
Username: "system:kube-scheduler",
Groups: []string{"system:authenticated"},
},
resKind: "Pod",
namespacedName: types.NamespacedName{Name: "test-pod", Namespace: "test-namespace"},
wantResponse: admission.Allowed(fmt.Sprintf(resourceAllowedFormat, "system:kube-scheduler", []string{"system:authenticated"}, "Pod", types.NamespacedName{Name: "test-pod", Namespace: "test-namespace"})),
},
"fail to validate user with invalid username, groups": {
userInfo: authenticationv1.UserInfo{
Username: "test-user",
Expand Down
77 changes: 52 additions & 25 deletions pkg/webhook/webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
admv1beta1 "k8s.io/api/admissionregistration/v1beta1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -48,14 +47,11 @@ const (
FleetWebhookCfgName = "fleet-validating-webhook-configuration"
FleetWebhookSvcName = "fleetwebhook"

crdResourceName = "customresourcedefinitions"
memberClusterResourceName = "memberclusters"
internalMemberClusterResourceName = "internalmemberclusters"
namespaceResouceName = "namespaces"
replicaSetResourceName = "replicasets"
podResourceName = "pods"
roleResourceName = "roles"
roleBindingResourceName = "rolebindings"
crdResourceName = "customresourcedefinitions"
memberClusterResourceName = "memberclusters"
namespaceResouceName = "namespaces"
replicaSetResourceName = "replicasets"
podResourceName = "pods"
)

var (
Expand Down Expand Up @@ -218,7 +214,8 @@ func (w *Config) buildValidatingWebHooks() []admv1.ValidatingWebhook {
}

if w.enableGuardRail {
fleetNamespaceSelector := &metav1.LabelSelector{
// MatchLabels/MatchExpressions values are ANDed to select resources.
fleetMemberNamespaceSelector := &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: fleetv1beta1.FleetResourceLabelKey,
Expand All @@ -227,13 +224,35 @@ func (w *Config) buildValidatingWebHooks() []admv1.ValidatingWebhook {
},
},
}

fleetSystemNamespaceSelector := &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: corev1.LabelMetadataName,
Operator: metav1.LabelSelectorOpIn,
Values: []string{"fleet-system"},
},
},
}
kubeNamespaceSelector := &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: corev1.LabelMetadataName,
Operator: metav1.LabelSelectorOpIn,
Values: []string{"kube-system", "kube-public", "kube-node-lease"},
},
},
}
cudOperations := []admv1.OperationType{
admv1.Create,
admv1.Update,
admv1.Delete,
}

namespacedResourcesRules := []admv1.RuleWithOperations{
{
Operations: cudOperations,
Rule: createRule([]string{"*"}, []string{"*"}, []string{"*/*"}, &namespacedScope),
},
}
guardRailWebhookConfigurations := []admv1.ValidatingWebhook{
{
Name: "fleet.customresourcedefinition.validating",
Expand Down Expand Up @@ -262,23 +281,31 @@ func (w *Config) buildValidatingWebHooks() []admv1.ValidatingWebhook {
},
},
{
Name: "fleet.namespacedresources.validating",
Name: "fleet.fleetmembernamespacedresources.validating",
ClientConfig: w.createClientConfig(fleetresourcehandler.ValidationPath),
FailurePolicy: &failPolicy,
SideEffects: &sideEffortsNone,
AdmissionReviewVersions: admissionReviewVersions,
NamespaceSelector: fleetNamespaceSelector,
Rules: []admv1.RuleWithOperations{
{
Operations: cudOperations,
Rule: createRule([]string{rbacv1.SchemeGroupVersion.Group}, []string{rbacv1.SchemeGroupVersion.Version}, []string{roleResourceName, roleBindingResourceName}, &namespacedScope),
},
{
Operations: cudOperations,
Rule: createRule([]string{fleetv1alpha1.GroupVersion.Group}, []string{fleetv1alpha1.GroupVersion.Version}, []string{internalMemberClusterResourceName, internalMemberClusterResourceName + "/status"}, &namespacedScope),
},
// TODO: (Arvindthiru): Add Rules for pods, services, configmaps, secrets, deployments and replicasets
},
NamespaceSelector: fleetMemberNamespaceSelector,
Rules: namespacedResourcesRules,
},
{
Name: "fleet.fleetsystemnamespacedresources.validating",
ClientConfig: w.createClientConfig(fleetresourcehandler.ValidationPath),
FailurePolicy: &failPolicy,
SideEffects: &sideEffortsNone,
AdmissionReviewVersions: admissionReviewVersions,
NamespaceSelector: fleetSystemNamespaceSelector,
Rules: namespacedResourcesRules,
},
{
Name: "fleet.kubenamespacedresources.validating",
ClientConfig: w.createClientConfig(fleetresourcehandler.ValidationPath),
FailurePolicy: &failPolicy,
SideEffects: &sideEffortsNone,
AdmissionReviewVersions: admissionReviewVersions,
NamespaceSelector: kubeNamespaceSelector,
Rules: namespacedResourcesRules,
},
{
Name: "fleet.namespace.validating",
Expand Down
2 changes: 1 addition & 1 deletion pkg/webhook/webhook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func TestBuildValidatingWebhooks(t *testing.T) {
clientConnectionType: &url,
enableGuardRail: true,
},
wantLength: 7,
wantLength: 9,
},
}

Expand Down
4 changes: 2 additions & 2 deletions test/e2e/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,12 +229,12 @@ var _ = BeforeSuite(func() {
testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

By("create resources for webhook e2e")
testutils.CreateResourcesForWebHookE2E(ctx, HubCluster, memberNamespace.Name)
testutils.CreateResourcesForWebHookE2E(ctx, HubCluster)
})

var _ = AfterSuite(func() {
By("delete resources created for webhook e2e")
testutils.DeleteResourcesForWebHookE2E(ctx, HubCluster, memberNamespace.Name)
testutils.DeleteResourcesForWebHookE2E(ctx, HubCluster)

By("update member cluster in the hub cluster")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName)
Expand Down
Loading

0 comments on commit ea45e68

Please sign in to comment.