Skip to content

Commit

Permalink
feat(RedisInstance): add AWS KCP reconciler (#354)
Browse files Browse the repository at this point in the history
  • Loading branch information
dushanpantic authored Jul 17, 2024
1 parent f550578 commit b660db2
Show file tree
Hide file tree
Showing 26 changed files with 1,042 additions and 25 deletions.
5 changes: 4 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ package main

import (
"flag"
"github.com/fsnotify/fsnotify"
"os"

"github.com/fsnotify/fsnotify"

"github.com/elliotchance/pie/v2"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"

Expand All @@ -35,6 +36,7 @@ import (
"github.com/kyma-project/cloud-manager/pkg/common/abstractions"
awsiprangeclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/iprange/client"
awsnfsinstanceclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/nfsinstance/client"
awsredisinstanceclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/redisinstance/client"
awsvpcpeeringclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/vpcpeering/client"
azureredisclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/azure/redisinstance/client"
azurevpcpeeringclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/azure/vpcpeering/client"
Expand Down Expand Up @@ -249,6 +251,7 @@ func main() {
mgr,
gcpmemorystoreclient.NewMemorystoreClientProvider(),
azureredisclient.NewClientProvider(),
awsredisinstanceclient.NewClientProvider(),
env,
); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "RedisInstance")
Expand Down
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.1.1
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/redis/armredis v1.0.0
github.com/aws/aws-sdk-go-v2 v1.26.1
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/config v1.26.3
github.com/aws/aws-sdk-go-v2/credentials v1.16.14
github.com/aws/aws-sdk-go-v2/service/backup v1.34.2
github.com/aws/aws-sdk-go-v2/service/ec2 v1.141.0
github.com/aws/aws-sdk-go-v2/service/efs v1.26.5
github.com/aws/aws-sdk-go-v2/service/elasticache v1.40.3
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7
github.com/aws/smithy-go v1.20.2
github.com/aws/smithy-go v1.20.3
github.com/elliotchance/pie/v2 v2.8.0
github.com/fatih/color v1.16.0
github.com/fsnotify/fsnotify v1.6.0
Expand Down Expand Up @@ -62,8 +63,8 @@ require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
Expand Down
18 changes: 10 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY=
github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/aws/aws-sdk-go v1.49.21 h1:Rl8KW6HqkwzhATwvXhyr7vD4JFUMi7oXGAw9SrxxIFY=
github.com/aws/aws-sdk-go v1.49.21/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA=
Expand All @@ -60,10 +60,10 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tC
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11 h1:I6lAa3wBWfCz/cKkOpAcumsETRkFAl70sWi8ItcMEsM=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11/go.mod h1:be1NIO30kJA23ORBLqPo1LttEM6tPNSEcjkd1eKzNW0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M=
Expand All @@ -74,6 +74,8 @@ github.com/aws/aws-sdk-go-v2/service/ec2 v1.141.0 h1:cP43vFYAQyREOp972C+6d4+dzpx
github.com/aws/aws-sdk-go-v2/service/ec2 v1.141.0/go.mod h1:qjhtI9zjpUHRc6khtrIM9fb48+ii6+UikL3/b+MKYn0=
github.com/aws/aws-sdk-go-v2/service/efs v1.26.5 h1:N1ezZV2yy7NV2w/bA4s4I/+0n2xpL4DzlmroEg5qFsg=
github.com/aws/aws-sdk-go-v2/service/efs v1.26.5/go.mod h1:PJHqaboMcF/eLy1F/Y9hyls4CQGP5+T5f0iRq6CPXu4=
github.com/aws/aws-sdk-go-v2/service/elasticache v1.40.3 h1:nmEN5lGIAShc0nNFjvUk2/YYlsTSwX2n1XF37Av93Yw=
github.com/aws/aws-sdk-go-v2/service/elasticache v1.40.3/go.mod h1:OcUtpbcNsyMdA/Wv5XenKl8aG3yrqA6HVIOF7ms+Ikc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 h1:L0ai8WICYHozIKK+OtPzVJBugL7culcuM4E4JOpIEm8=
Expand All @@ -90,8 +92,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3W
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
Expand Down
125 changes: 125 additions & 0 deletions internal/controller/cloud-control/redisinstance_aws_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package cloudcontrol

import (
"time"

elasticacheTypes "github.com/aws/aws-sdk-go-v2/service/elasticache/types"
cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/api/cloud-control/v1beta1"
iprangePkg "github.com/kyma-project/cloud-manager/pkg/kcp/iprange"
awsmeta "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/meta"
scopePkg "github.com/kyma-project/cloud-manager/pkg/kcp/scope"
. "github.com/kyma-project/cloud-manager/pkg/testinfra/dsl"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("Feature: KCP RedisInstance", func() {

It("Scenario: KCP AWS RedisInstance is created and deleted", func() {

name := "6e6ff0b2-3edb-4d6e-8ae5-fbd3d3644ce2"
scope := &cloudcontrolv1beta1.Scope{}

By("Given Scope exists", func() {
// Tell Scope reconciler to ignore this kymaName
scopePkg.Ignore.AddName(name)

Eventually(CreateScopeAws).
WithArguments(infra.Ctx(), infra, scope, WithName(name)).
Should(Succeed())
})

kcpIpRangeName := "7017ef87-3814-4dc5-bcd1-966d2f44e285"
kcpIpRange := &cloudcontrolv1beta1.IpRange{}

// Tell IpRange reconciler to ignore this kymaName
iprangePkg.Ignore.AddName(kcpIpRangeName)
By("And Given KCP IPRange exists", func() {
Eventually(CreateKcpIpRange).
WithArguments(
infra.Ctx(), infra.KCP().Client(), kcpIpRange,
WithName(kcpIpRangeName),
WithKcpIpRangeSpecScope(scope.Name),
).
Should(Succeed())
})

By("And Given KCP IpRange has Ready condition", func() {
Eventually(UpdateStatus).
WithArguments(
infra.Ctx(), infra.KCP().Client(), kcpIpRange,
WithKcpIpRangeStatusCidr(kcpIpRange.Spec.Cidr),
WithConditions(KcpReadyCondition()),
).WithTimeout(20*time.Second).WithPolling(200*time.Millisecond).
Should(Succeed(), "Expected KCP IpRange to become ready")
})

redisInstance := &cloudcontrolv1beta1.RedisInstance{}

By("When RedisInstance is created", func() {
Eventually(CreateRedisInstance).
WithArguments(infra.Ctx(), infra.KCP().Client(), redisInstance,
WithName(name),
WithRemoteRef("skr-redis-example-aws"),
WithIpRange(kcpIpRangeName),
WithInstanceScope(name),
WithRedisInstanceAws(),
// todo add specs
).
Should(Succeed(), "failed creating RedisInstance")
})

var awsElastiCacheClusterInstance *elasticacheTypes.CacheCluster
By("Then AWS Redis is created", func() {
Eventually(LoadAndCheck).
WithArguments(infra.Ctx(), infra.KCP().Client(), redisInstance,
NewObjActions(),
HavingRedisInstanceStatusId()).
Should(Succeed(), "expected RedisInstance to get status.id")
awsElastiCacheClusterInstance = infra.AwsMock().GetAwsElastiCacheByName(redisInstance.Status.Id)
})

By("When AWS Redis is Available", func() {
infra.AwsMock().SetAwsElastiCacheLifeCycleState(*awsElastiCacheClusterInstance.CacheClusterId, awsmeta.ElastiCache_AVAILABLE)
})

By("Then RedisInstance has Ready condition", func() {
Eventually(LoadAndCheck).
WithArguments(infra.Ctx(), infra.KCP().Client(), redisInstance,
NewObjActions(),
HavingConditionTrue(cloudcontrolv1beta1.ConditionTypeReady),
).
Should(Succeed(), "expected RedisInstance to has Ready state, but it didn't")
})

By("And Then RedisInstance has .status.primaryEndpoint set", func() {
Expect(len(redisInstance.Status.PrimaryEndpoint) > 0).To(Equal(true))
})
// TODO
// By("And Then RedisInstance has .status.readEndpoint set", func() {
// Expect(len(redisInstance.Status.ReadEndpoint) > 0).To(Equal(true))
// })
// By("And Then RedisInstance has .status.authString set", func() {
// Expect(len(redisInstance.Status.AuthString) > 0).To(Equal(true))
// })

// DELETE

By("When RedisInstance is deleted", func() {
Eventually(Delete).
WithArguments(infra.Ctx(), infra.KCP().Client(), redisInstance).
Should(Succeed(), "failed deleting RedisInstance")
})

By("And When AWS Redis state is deleted", func() {
infra.AwsMock().DeleteAwsElastiCacheByName(*awsElastiCacheClusterInstance.CacheClusterId)
})

By("Then RedisInstance does not exist", func() {
Eventually(IsDeleted, 5*time.Second).
WithArguments(infra.Ctx(), infra.KCP().Client(), redisInstance).
Should(Succeed(), "expected RedisInstance not to exist (be deleted), but it still exists")
})
})

})
5 changes: 5 additions & 0 deletions internal/controller/cloud-control/redisinstance_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ import (
"github.com/kyma-project/cloud-manager/pkg/common/abstractions"
"github.com/kyma-project/cloud-manager/pkg/common/actions/focal"
"github.com/kyma-project/cloud-manager/pkg/composed"
awsclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/client"
awsredisinstance "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/redisinstance"
awsredisinstanceclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/aws/redisinstance/client"
azureclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/azure/client"
azureredisinstance "github.com/kyma-project/cloud-manager/pkg/kcp/provider/azure/redisinstance"
azureredisinstanceclient "github.com/kyma-project/cloud-manager/pkg/kcp/provider/azure/redisinstance/client"
Expand All @@ -40,6 +43,7 @@ func SetupRedisInstanceReconciler(
kcpManager manager.Manager,
gcpFilestoreClientProvider gcpclient.ClientProvider[gcpredisinstanceclient.MemorystoreClient],
azureFilestoreClientProvider azureclient.SkrClientProvider[azureredisinstanceclient.Client],
awsFilestoreClientProvider awsclient.SkrClientProvider[awsredisinstanceclient.ElastiCacheClient],
env abstractions.Environment,
) error {
if env == nil {
Expand All @@ -51,6 +55,7 @@ func SetupRedisInstanceReconciler(
focal.NewStateFactory(),
gcpredisinstance.NewStateFactory(gcpFilestoreClientProvider, env),
azureredisinstance.NewStateFactory(azureFilestoreClientProvider),
awsredisinstance.NewStateFactory(awsFilestoreClientProvider),
),
).SetupWithManager(kcpManager)
}
Expand Down
1 change: 1 addition & 0 deletions internal/controller/cloud-control/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ var _ = BeforeSuite(func() {
infra.KcpManager(),
infra.GcpMock().MemoryStoreProviderFake(),
infra.AzureMock().RedisClientProvider(),
infra.AwsMock().ElastiCacheProviderFake(),
env,
))

Expand Down
41 changes: 33 additions & 8 deletions pkg/kcp/provider/aws/meta/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ package meta
import (
"context"
"errors"
"net/http"

"github.com/aws/aws-sdk-go-v2/aws/retry"
efsTypes "github.com/aws/aws-sdk-go-v2/service/efs/types"
elasticacheTypes "github.com/aws/aws-sdk-go-v2/service/elasticache/types"
"github.com/aws/smithy-go"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/kyma-project/cloud-manager/pkg/composed"
"github.com/kyma-project/cloud-manager/pkg/util"
"net/http"
)

type awsAccountKeyType struct{}
Expand Down Expand Up @@ -47,25 +49,29 @@ func AsApiError(err error) smithy.APIError {
}

var notFoundErrorCodes = map[string]struct{}{
(&efsTypes.FileSystemNotFound{}).ErrorCode(): {},
(&efsTypes.AccessPointNotFound{}).ErrorCode(): {},
(&efsTypes.MountTargetNotFound{}).ErrorCode(): {},
(&efsTypes.PolicyNotFound{}).ErrorCode(): {},
(&efsTypes.FileSystemNotFound{}).ErrorCode(): {},
(&efsTypes.AccessPointNotFound{}).ErrorCode(): {},
(&efsTypes.MountTargetNotFound{}).ErrorCode(): {},
(&efsTypes.PolicyNotFound{}).ErrorCode(): {},
(&elasticacheTypes.CacheSubnetGroupNotFoundFault{}).ErrorCode(): {},
(&elasticacheTypes.CacheClusterNotFoundFault{}).ErrorCode(): {},
}

func IsNotFound(err error) bool {
if err != nil {
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
var smithyhttpErr *smithyhttp.ResponseError
if errors.As(err, &smithyhttpErr) {
return smithyhttpErr.HTTPStatusCode() == http.StatusNotFound
}

_, listed := notFoundErrorCodes[apiErr.ErrorCode()]
if listed {
return true
}

if errors.As(err, &smithyhttpErr) {
return smithyhttpErr.HTTPStatusCode() == http.StatusNotFound
}

}
}
return false
Expand All @@ -92,3 +98,22 @@ func LogErrorAndReturn(err error, msg string, ctx context.Context) (error, conte
result := ErrorToRequeueResponse(err)
return composed.LogErrorAndReturn(err, msg, result, ctx)
}

type ElastiCacheState = string

// github.com/aws/aws-sdk-go-v2/service/[email protected]/types/types.go
// CacheClusterStatus *string
// The current state of this cluster, one of the following values: available ,
// creating , deleted , deleting , incompatible-network , modifying , rebooting
// cluster nodes , restore-failed , or snapshotting .
const (
ElastiCache_AVAILABLE ElastiCacheState = "available"
ElastiCache_CREATING ElastiCacheState = "creating"
ElastiCache_DELETED ElastiCacheState = "deleted"
ElastiCache_DELETING ElastiCacheState = "deleting"
ElastiCache_INCOMPATIBLE_NETWORK ElastiCacheState = "incompatible-network"
ElastiCache_MODIFYING ElastiCacheState = "modifying"
ElastiCache_REBOOTING_CLUSTER_NODES ElastiCacheState = "rebooting cluster nodes"
ElastiCache_RESTORE_FAILED ElastiCacheState = "restore-failed"
ElastiCache_SNAPSHOTTING ElastiCacheState = "snapshotting"
)
Loading

0 comments on commit b660db2

Please sign in to comment.