code generation changes for resolving referenced resources (#246)

Issue #, if available: aws-controllers-k8s/community#1063

Description of changes:
* Code generation changes for supporting resource reference
* Corresponding runtime PR aws-controllers-k8s/runtime#62
* Generated code can be seen in `pkg/generate/code/resource_reference_test.go` file

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: vijtrip2

pkg/generate/ack/controller.go

@@ -42,6 +42,7 @@ var (
 	controllerIncludePaths = []string{
 		"config/controller/kustomization_def.yaml.tpl",
 		"boilerplate.go.tpl",
+		"pkg/resource/references_read_referenced_resource.go.tpl",
 		"pkg/resource/sdk_find_read_one.go.tpl",
 		"pkg/resource/sdk_find_get_attributes.go.tpl",
 		"pkg/resource/sdk_find_read_many.go.tpl",
@@ -149,6 +150,12 @@ var (
 		"GoCodeIncompleteLateInitialization": func(r *ackmodel.CRD, resVarName string, indentLevel int) string {
 			return code.IncompleteLateInitialization(r.Config(), r, resVarName, indentLevel)
 		},
+		"GoCodeReferencesValidation": func(r *ackmodel.CRD, sourceVarName string, referenceFieldSuffix string, indentLevel int) string {
+			return code.ReferenceFieldsValidation(r, sourceVarName, referenceFieldSuffix, indentLevel)
+		},
+		"GoCodeContainsReferences": func(r *ackmodel.CRD, sourceVarName string) string {
+			return code.ReferenceFieldsPresent(r, sourceVarName)
+		},
 	}
 )
 
@@ -195,6 +202,7 @@ func Controller(
 		"identifiers.go.tpl",
 		"manager.go.tpl",
 		"manager_factory.go.tpl",
+		"references.go.tpl",
 		"resource.go.tpl",
 		"sdk.go.tpl",
 	}

pkg/generate/ack/runtime_test.go

@@ -20,6 +20,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8srt "k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/go-logr/logr"
@@ -121,6 +122,10 @@ func (frm *fakeRM) LateInitialize(context.Context, acktypes.AWSResource) (acktyp
 	return nil, nil
 }
 
+func (frm *fakeRM) ResolveReferences(context.Context, client.Reader, acktypes.AWSResource) (acktypes.AWSResource, error) {
+	return nil, nil
+}
+
 // This test is mostly just a hack to introduce a Go module dependency between
 // the ACK runtime library and the code generator. The code generator doesn't
 // actually depend on Go code in the ACK runtime, but it *produces* templated

pkg/generate/code/compare.go

@@ -128,6 +128,18 @@ func CompareResource(
 			cfg.PrefixConfig.SpecField+"."+specField.Names.Camel, ".",
 		)
 
+		// Use reflect.DeepEqual for comparing Reference fields because
+		// some of reference fields are list of pointer to structs and
+		// DeepEqual is easy way to compare them
+		if specField.IsReference() {
+			out += fmt.Sprintf("%sif !reflect.DeepEqual(%s, %s) {\n",
+				indent, firstResAdaptedVarName, secondResAdaptedVarName)
+			out += fmt.Sprintf("%s\t%s.Add(\"%s\", %s, %s)\n", indent,
+				deltaVarName, fieldPath, firstResAdaptedVarName,
+				secondResAdaptedVarName)
+			out += fmt.Sprintf("%s}\n", indent)
+			continue
+		}
 		memberShapeRef := specField.ShapeRef
 		memberShape := memberShapeRef.Shape
 

pkg/generate/code/resource_reference.go

@@ -0,0 +1,87 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package code
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws-controllers-k8s/code-generator/pkg/model"
+)
+
+// ReferenceFieldsValidation produces the go code to validate reference field and
+// corresponding identifier field.
+// Sample code:
+// if ko.Spec.APIIDRef != nil && ko.Spec.APIID != nil {
+//		return ackerr.ResourceReferenceAndIDNotSupportedFor("APIID", "APIIDRef")
+//	}
+//	if ko.Spec.APIIDRef == nil && ko.Spec.APIID == nil {
+//		return ackerr.ResourceReferenceOrIDRequiredFor("APIID", "APIIDRef")
+//	}
+func ReferenceFieldsValidation(
+	crd *model.CRD,
+	sourceVarName string,
+	referenceFieldSuffix string,
+	indentLevel int,
+) string {
+	out := ""
+	for _, field := range crd.Fields {
+		if field.HasReference() {
+			indent := strings.Repeat("\t", indentLevel)
+			// Validation to make sure both target field and reference are
+			// not present at the same time in desired resource
+			out += fmt.Sprintf("%sif %s.Spec.%s%s != nil"+
+				" && %s.Spec.%s != nil {\n", indent, sourceVarName, field.Names.Camel,
+				referenceFieldSuffix, sourceVarName, field.Names.Camel)
+			out += fmt.Sprintf("%s\treturn "+
+				"ackerr.ResourceReferenceAndIDNotSupportedFor(\"%s\", \"%s%s\")\n",
+				indent,field.Names.Camel, field.Names.Camel, referenceFieldSuffix)
+			out += fmt.Sprintf("%s}\n", indent)
+
+			// If the field is required, make sure either Ref or original
+			// field is present in the resource
+			if field.IsRequired() {
+				out += fmt.Sprintf("%sif %s.Spec.%s%s == nil &&"+
+					" %s.Spec.%s == nil {\n", indent, sourceVarName,
+					field.Names.Camel, referenceFieldSuffix, sourceVarName,
+					field.Names.Camel)
+				out += fmt.Sprintf("%s\treturn "+
+					"ackerr.ResourceReferenceOrIDRequiredFor(\"%s\", \"%s%s\")\n",
+					indent, field.Names.Camel, field.Names.Camel,
+					referenceFieldSuffix)
+				out += fmt.Sprintf("%s}\n", indent)
+			}
+		}
+	}
+	return out
+}
+
+// ReferenceFieldsPresent produces go code(logical condition) for finding whether
+// a non-nil reference field is present in a resource. This checks helps in deciding
+// whether ACK.ReferencesResolved condition should be added to resource status
+// Sample Code:
+// return false || ko.Spec.APIIDRef != nil
+func ReferenceFieldsPresent(
+	crd *model.CRD,
+	sourceVarName string,
+) string {
+	out := "false"
+	for _, field := range crd.Fields {
+		if field.IsReference() {
+			out += fmt.Sprintf(" || %s.Spec.%s != nil", sourceVarName,
+				field.Names.Camel)
+		}
+	}
+	return out
+}

pkg/generate/code/resource_reference_test.go

@@ -0,0 +1,128 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package code_test
+
+import (
+	"testing"
+
+	"github.com/aws-controllers-k8s/code-generator/pkg/generate/code"
+	"github.com/aws-controllers-k8s/code-generator/pkg/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_ReferenceFieldsValidation_NoReferenceConfig(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	g := testutil.NewModelForServiceWithOptions(t, "apigatewayv2",
+		&testutil.TestingModelOptions{
+		GeneratorConfigFile: "generator-with-reference.yaml",
+	})
+
+	crd := testutil.GetCRDByName(t, g, "Api")
+	require.NotNil(crd)
+	expected := ""
+	assert.Equal(expected, code.ReferenceFieldsValidation(crd, "ko", "Ref", 1))
+}
+
+func Test_ReferenceFieldsValidation_SingleReference(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	g := testutil.NewModelForServiceWithOptions(t, "apigatewayv2",
+		&testutil.TestingModelOptions{
+			GeneratorConfigFile: "generator-with-reference.yaml",
+		})
+
+	crd := testutil.GetCRDByName(t, g, "Integration")
+	require.NotNil(crd)
+	expected :=
+		`	if ko.Spec.APIIDRef != nil && ko.Spec.APIID != nil {
+		return ackerr.ResourceReferenceAndIDNotSupportedFor("APIID", "APIIDRef")
+	}
+	if ko.Spec.APIIDRef == nil && ko.Spec.APIID == nil {
+		return ackerr.ResourceReferenceOrIDRequiredFor("APIID", "APIIDRef")
+	}
+`
+	assert.Equal(expected, code.ReferenceFieldsValidation(crd, "ko", "Ref", 1))
+}
+
+func Test_ReferenceFieldsValidation_SliceOfReferences(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	g := testutil.NewModelForServiceWithOptions(t, "apigatewayv2",
+		&testutil.TestingModelOptions{
+			GeneratorConfigFile: "generator-with-reference.yaml",
+		})
+
+	//NOTE: For the moment, we are substituting SecurityGroupId with ApiId
+	// just to test code generation for slices of reference
+	crd := testutil.GetCRDByName(t, g, "VpcLink")
+	require.NotNil(crd)
+	expected :=
+		`	if ko.Spec.SecurityGroupIDsRef != nil && ko.Spec.SecurityGroupIDs != nil {
+		return ackerr.ResourceReferenceAndIDNotSupportedFor("SecurityGroupIDs", "SecurityGroupIDsRef")
+	}
+`
+	assert.Equal(expected, code.ReferenceFieldsValidation(crd, "ko", "Ref", 1))
+}
+
+func Test_ReferenceFieldsPresent_NoReferenceConfig(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	g := testutil.NewModelForServiceWithOptions(t, "apigatewayv2",
+		&testutil.TestingModelOptions{
+			GeneratorConfigFile: "generator-with-reference.yaml",
+		})
+
+	crd := testutil.GetCRDByName(t, g, "Api")
+	require.NotNil(crd)
+	expected := "false"
+	assert.Equal(expected, code.ReferenceFieldsPresent(crd, "ko"))
+}
+
+func Test_ReferenceFieldsPresent_SingleReference(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	g := testutil.NewModelForServiceWithOptions(t, "apigatewayv2",
+		&testutil.TestingModelOptions{
+			GeneratorConfigFile: "generator-with-reference.yaml",
+		})
+
+	crd := testutil.GetCRDByName(t, g, "Integration")
+	require.NotNil(crd)
+	expected := "false || ko.Spec.APIIDRef != nil"
+	assert.Equal(expected, code.ReferenceFieldsPresent(crd, "ko"))
+}
+
+func Test_ReferenceFieldsPresent_SliceOfReferences(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	g := testutil.NewModelForServiceWithOptions(t, "apigatewayv2",
+		&testutil.TestingModelOptions{
+			GeneratorConfigFile: "generator-with-reference.yaml",
+		})
+
+	//NOTE: For the moment, we are substituting SecurityGroupId with ApiId
+	// just to test code generation for slices of reference
+	crd := testutil.GetCRDByName(t, g, "VpcLink")
+	require.NotNil(crd)
+	expected := "false || ko.Spec.SecurityGroupIDsRef != nil"
+	assert.Equal(expected, code.ReferenceFieldsPresent(crd, "ko"))
+}

pkg/generate/config/field.go

@@ -267,6 +267,31 @@ type LateInitializeConfig struct {
 	MaxBackoffSeconds int `json:"max_backoff_seconds"`
 }
 
+// ReferencesConfig contains the instructions for how to add the referenced resource
+// configuration for a field.
+// Example:
+// ```
+// Integration:
+//    fields:
+//      ApiId:
+//        references:
+//          resource: API
+//          path: Status.APIID
+//```
+// The above configuration will result in generation of a new field 'APIIDRef'
+// of type 'AWSResourceReference' for ApiGatewayv2-Integration crd.
+// When 'APIIDRef' field is present in custom resource manifest, reconciler will
+// read the referred 'API' resource and copy the value from 'Status.APIID' in
+// 'Integration' resource's 'APIID' field
+type ReferencesConfig struct {
+	// Resource mentions the K8s resource which is read to resolve the
+	// reference
+	Resource string `json:"resource"`
+	// Path refers to the the path of field which should be copied
+	// to resolve the reference
+	Path string `json:"path"`
+}
+
 // FieldConfig contains instructions to the code generator about how
 // to interpret the value of an Attribute and how to map it to a CRD's Spec or
 // Status field
@@ -325,4 +350,7 @@ type FieldConfig struct {
 	// Late Initialize instructs the code generator how to handle the late initialization
 	// of the field.
 	LateInitialize *LateInitializeConfig `json:"late_initialize,omitempty"`
+	// References instructs the code generator how to refer this field from
+	// other custom resource
+	References     *ReferencesConfig     `json:"references,omitempty"`
 }

pkg/model/crd.go

@@ -144,12 +144,12 @@ func (r *CRD) HasShapeAsMember(toFind string) bool {
 		}
 	}
 	for _, field := range r.SpecFields {
-		if shapeHasMember(field.ShapeRef.Shape, toFind) {
+		if field.ShapeRef != nil && shapeHasMember(field.ShapeRef.Shape, toFind) {
 			return true
 		}
 	}
 	for _, field := range r.StatusFields {
-		if shapeHasMember(field.ShapeRef.Shape, toFind) {
+		if field.ShapeRef != nil && shapeHasMember(field.ShapeRef.Shape, toFind) {
 			return true
 		}
 	}
@@ -757,6 +757,17 @@ func (r *CRD) HasMember(
 	return inSpec, inStatus
 }
 
+// HasReferenceFields returns true if any of the fields in CRD is a reference
+// field. Otherwise returns false
+func (r *CRD) HasReferenceFields() bool {
+	for _, field := range r.Fields {
+		if field.IsReference() {
+			return true
+		}
+	}
+	return false
+}
+
 // NewCRD returns a pointer to a new `ackmodel.CRD` struct that describes a
 // single top-level resource in an AWS service API
 func NewCRD(