Merge pull request #85 from weaveworks/config-generator

Config generator.

Author: bigkevmcd

api/v1alpha1/gitopsset_types.go

@@ -32,6 +32,19 @@ type ClusterGenerator struct {
 	Selector metav1.LabelSelector `json:"selector,omitempty"`
 }
 
+// ConfigGenerator loads a referenced ConfigMap or
+// Secret from the Cluster and makes it available as a resource.
+type ConfigGenerator struct {
+	// Kind of the referent.
+	// +kubebuilder:validation:Enum=ConfigMap;Secret
+	// +required
+	Kind string `json:"kind"`
+
+	// Name of the referent.
+	// +required
+	Name string `json:"name"`
+}
+
 // ListGenerator generates from a hard-coded list.
 type ListGenerator struct {
 	Elements []apiextensionsv1.JSON `json:"elements,omitempty"`
@@ -180,6 +193,7 @@ type GitOpsSetNestedGenerator struct {
 	Cluster       *ClusterGenerator       `json:"cluster,omitempty"`
 	APIClient     *APIClientGenerator     `json:"apiClient,omitempty"`
 	ImagePolicy   *ImagePolicyGenerator   `json:"imagePolicy,omitempty"`
+	Config        *ConfigGenerator        `json:"config,omitempty"`
 }
 
 // ImagePolicyGenerator generates from the ImagePolicy.
@@ -197,6 +211,7 @@ type GitOpsSetGenerator struct {
 	Cluster       *ClusterGenerator       `json:"cluster,omitempty"`
 	APIClient     *APIClientGenerator     `json:"apiClient,omitempty"`
 	ImagePolicy   *ImagePolicyGenerator   `json:"imagePolicy,omitempty"`
+	Config        *ConfigGenerator        `json:"config,omitempty"`
 }
 
 // GitOpsSetSpec defines the desired state of GitOpsSet

api/v1alpha1/zz_generated.deepcopy.go

@@ -166,6 +166,23 @@ spec:
                           type: object
                           x-kubernetes-map-type: atomic
                       type: object
+                    config:
+                      description: ConfigGenerator loads a referenced ConfigMap or
+                        Secret from the Cluster and makes it available as a resource.
+                      properties:
+                        kind:
+                          description: Kind of the referent.
+                          enum:
+                          - ConfigMap
+                          - Secret
+                          type: string
+                        name:
+                          description: Name of the referent.
+                          type: string
+                      required:
+                      - kind
+                      - name
+                      type: object
                     gitRepository:
                       description: GitRepositoryGenerator generates from files in
                         a Flux GitRepository resource.
@@ -357,6 +374,24 @@ spec:
                                     type: object
                                     x-kubernetes-map-type: atomic
                                 type: object
+                              config:
+                                description: ConfigGenerator loads a referenced ConfigMap
+                                  or Secret from the Cluster and makes it available
+                                  as a resource.
+                                properties:
+                                  kind:
+                                    description: Kind of the referent.
+                                    enum:
+                                    - ConfigMap
+                                    - Secret
+                                    type: string
+                                  name:
+                                    description: Name of the referent.
+                                    type: string
+                                required:
+                                - kind
+                                - name
+                                type: object
                               gitRepository:
                                 description: GitRepositoryGenerator generates from
                                   files in a Flux GitRepository resource.

config/crd/bases/templates.weave.works_gitopssets.yaml

@@ -15,13 +15,13 @@ import (
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	"github.com/gitops-tools/pkg/sets"
 	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/cli-utils/pkg/object"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -45,6 +45,8 @@ var accessor = meta.NewAccessor()
 const (
 	gitRepositoryIndexKey string = ".metadata.gitRepository"
 	imagePolicyIndexKey   string = ".metadata.imagePolicy"
+	configMapIndexKey     string = ".metadata.configMap"
+	secretIndexKey        string = ".metadata.secret"
 )
 
 type eventRecorder interface {
@@ -308,12 +310,30 @@ func (r *GitOpsSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return fmt.Errorf("failed setting index fields: %w", err)
 	}
 
+	if err := mgr.GetCache().IndexField(
+		context.TODO(), &templatesv1.GitOpsSet{}, configMapIndexKey, indexConfig("ConfigMap")); err != nil {
+		return fmt.Errorf("failed setting index fields: %w", err)
+	}
+
+	if err := mgr.GetCache().IndexField(
+		context.TODO(), &templatesv1.GitOpsSet{}, secretIndexKey, indexConfig("Secret")); err != nil {
+		return fmt.Errorf("failed setting index fields: %w", err)
+	}
+
 	builder := ctrl.NewControllerManagedBy(mgr).
 		For(&templatesv1.GitOpsSet{}, builder.WithPredicates(
 			predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{}))).
 		Watches(
 			&source.Kind{Type: &sourcev1.GitRepository{}},
 			handler.EnqueueRequestsFromMapFunc(r.gitRepositoryToGitOpsSet),
+		).
+		Watches(
+			&source.Kind{Type: &corev1.ConfigMap{}},
+			handler.EnqueueRequestsFromMapFunc(r.configMapToGitOpsSet),
+		).
+		Watches(
+			&source.Kind{Type: &corev1.Secret{}},
+			handler.EnqueueRequestsFromMapFunc(r.secretToGitOpsSet),
 		)
 
 	// Only watch for GitopsCluster objects if the Cluster generator is enabled.
@@ -360,7 +380,7 @@ func (r *GitOpsSetReconciler) gitOpsClusterToGitOpsSet(o client.Object) []reconc
 	var result []reconcile.Request
 	for _, v := range list.Items {
 		if matchCluster(gitOpsCluster, &v) {
-			result = append(result, reconcile.Request{NamespacedName: types.NamespacedName{Name: v.GetName(), Namespace: v.GetNamespace()}})
+			result = append(result, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&v)})
 		}
 	}
 
@@ -439,41 +459,39 @@ func selectorMatchesCluster(labelSelector metav1.LabelSelector, cluster *cluster
 func (r *GitOpsSetReconciler) gitRepositoryToGitOpsSet(obj client.Object) []reconcile.Request {
 	// TODO: Store the applied version of GitRepositories in the Status, and don't
 	// retrigger if the commit-id isn't different.
-	ctx := context.Background()
-	var list templatesv1.GitOpsSetList
-
-	if err := r.List(ctx, &list, client.MatchingFields{
-		gitRepositoryIndexKey: client.ObjectKeyFromObject(obj).String(),
-	}); err != nil {
-		return nil
-	}
-
-	result := []reconcile.Request{}
-	for _, v := range list.Items {
-		result = append(result, reconcile.Request{NamespacedName: types.NamespacedName{Name: v.GetName(), Namespace: v.GetNamespace()}})
-	}
-
-	return result
+	return r.queryIndexedGitOpsSets(gitRepositoryIndexKey, obj)
 }
 
 func (r *GitOpsSetReconciler) imagePolicyToGitOpsSet(obj client.Object) []reconcile.Request {
+	return r.queryIndexedGitOpsSets(imagePolicyIndexKey, obj)
+}
+
+func (r *GitOpsSetReconciler) queryIndexedGitOpsSets(key string, obj client.Object) []reconcile.Request {
 	ctx := context.Background()
 	var list templatesv1.GitOpsSetList
 
-	if err := r.List(ctx, &list, client.MatchingFields{
-		imagePolicyIndexKey: client.ObjectKeyFromObject(obj).String(),
-	}); err != nil {
+	if err := r.List(ctx, &list,
+		client.MatchingFields{key: client.ObjectKeyFromObject(obj).String()},
+		client.InNamespace(obj.GetNamespace())); err != nil {
 		return nil
 	}
 
 	result := []reconcile.Request{}
-	for _, v := range list.Items {
-		result = append(result, reconcile.Request{NamespacedName: types.NamespacedName{Name: v.GetName(), Namespace: v.GetNamespace()}})
+	for i := range list.Items {
+		result = append(result, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&list.Items[i])})
 	}
 
 	return result
 }
 
+func (r *GitOpsSetReconciler) configMapToGitOpsSet(obj client.Object) []reconcile.Request {
+	return r.queryIndexedGitOpsSets(configMapIndexKey, obj)
+}
+
+func (r *GitOpsSetReconciler) secretToGitOpsSet(obj client.Object) []reconcile.Request {
+	return r.queryIndexedGitOpsSets(secretIndexKey, obj)
+}
+
 func (r *GitOpsSetReconciler) makeImpersonationClient(namespace, serviceAccountName string) (client.Client, error) {
 	copyCfg := rest.CopyConfig(r.Config)
 
@@ -516,6 +534,40 @@ func indexGitRepositories(o client.Object) []string {
 	return referencedNames
 }
 
+func indexConfig(kind string) func(o client.Object) []string {
+	return func(o client.Object) []string {
+		ks, ok := o.(*templatesv1.GitOpsSet)
+		if !ok {
+			panic(fmt.Sprintf("Expected a GitOpsSet, got %T", o))
+		}
+
+		referencedResources := []*templatesv1.ConfigGenerator{}
+		for _, gen := range ks.Spec.Generators {
+			if gen.Config != nil && gen.Config.Kind == kind {
+				referencedResources = append(referencedResources, gen.Config)
+			}
+			if gen.Matrix != nil && gen.Matrix.Generators != nil {
+				for _, matrixGen := range gen.Matrix.Generators {
+					if matrixGen.Config != nil && matrixGen.Config.Kind == kind {
+						referencedResources = append(referencedResources, matrixGen.Config)
+					}
+				}
+			}
+		}
+
+		if len(referencedResources) == 0 {
+			return nil
+		}
+
+		referencedNames := []string{}
+		for _, grg := range referencedResources {
+			referencedNames = append(referencedNames, fmt.Sprintf("%s/%s", ks.GetNamespace(), grg.Name))
+		}
+
+		return referencedNames
+	}
+}
+
 func indexImagePolicies(o client.Object) []string {
 	ks, ok := o.(*templatesv1.GitOpsSet)
 	if !ok {

controllers/gitopsset_controller.go

@@ -0,0 +1,103 @@
+package config
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-logr/logr"
+	templatesv1 "github.com/weaveworks/gitopssets-controller/api/v1alpha1"
+	"github.com/weaveworks/gitopssets-controller/controllers/templates/generators"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// ConfigGenerator generates a single resource from a referenced ConfigMap or
+// Secret.
+type ConfigGenerator struct {
+	client.Client
+	logr.Logger
+}
+
+// GeneratorFactory is a function for creating per-reconciliation generators for
+// the ConfigGenerator.
+func GeneratorFactory(l logr.Logger, c client.Client) generators.Generator {
+	return NewGenerator(l, c)
+}
+
+// NewGenerator creates and returns a new config generator.
+func NewGenerator(l logr.Logger, c client.Client) *ConfigGenerator {
+	return &ConfigGenerator{
+		Client: c,
+		Logger: l,
+	}
+}
+
+func (g *ConfigGenerator) Generate(ctx context.Context, sg *templatesv1.GitOpsSetGenerator, ks *templatesv1.GitOpsSet) ([]map[string]any, error) {
+	if sg == nil {
+		return nil, generators.ErrEmptyGitOpsSet
+	}
+
+	if sg.Config == nil {
+		return nil, nil
+	}
+	g.Logger.Info("generating params from Config generator")
+
+	var paramsList []map[string]any
+
+	switch sg.Config.Kind {
+	case "ConfigMap":
+		data, err := configMapToParams(ctx, g.Client, client.ObjectKey{Name: sg.Config.Name, Namespace: ks.GetNamespace()})
+		if err != nil {
+			return nil, err
+		}
+		paramsList = append(paramsList, data)
+
+	case "Secret":
+		data, err := secretToParams(ctx, g.Client, client.ObjectKey{Name: sg.Config.Name, Namespace: ks.GetNamespace()})
+		if err != nil {
+			return nil, err
+		}
+		paramsList = append(paramsList, data)
+
+	default:
+		return nil, fmt.Errorf("unknown Config Kind %q %q", sg.Config.Kind, sg.Config.Name)
+	}
+
+	return paramsList, nil
+}
+
+// Interval is an implementation of the Generator interface.
+func (g *ConfigGenerator) Interval(sg *templatesv1.GitOpsSetGenerator) time.Duration {
+	return generators.NoRequeueInterval
+}
+
+func configMapToParams(ctx context.Context, k8sClient client.Client, key client.ObjectKey) (map[string]any, error) {
+	var configMap corev1.ConfigMap
+
+	if err := k8sClient.Get(ctx, key, &configMap); err != nil {
+		return nil, err
+	}
+
+	return mapToAnyMap(configMap.Data), nil
+}
+
+func secretToParams(ctx context.Context, k8sClient client.Client, key client.ObjectKey) (map[string]any, error) {
+	var secret corev1.Secret
+
+	if err := k8sClient.Get(ctx, key, &secret); err != nil {
+		return nil, err
+	}
+
+	return mapToAnyMap(secret.Data), nil
+}
+
+func mapToAnyMap[V any](m map[string]V) map[string]any {
+	result := map[string]any{}
+
+	for k, v := range m {
+		result[k] = v
+	}
+
+	return result
+}