Skip to content

Commit 377bb7a

Browse files
omerdemirokactions-user
omerdemirok
authored and
actions-user
committed
feat: use explicit blast propagations and revert back to manual linking in the compute instance (#1864)
# Use explicit blast propagator only In dynamic adapters, we decided to use explicit blast propagator. After merging this, we should clean up the old one and the `Link` method that is used for manual adapters. Relevant ticket: [ENG-570](https://linear.app/overmind/issue/ENG-570/use-explicit-blast-propagations-in-dynamic-adapters) GitOrigin-RevId: aa188ea278d60851d7f07550756574cb96aaa346
1 parent e07d06a commit 377bb7a

15 files changed

+315
-595
lines changed

‎sources/gcp/dynamic/adapter-listable.go

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,6 @@ type ListableAdapter struct {
1818

1919
// NewListableAdapter creates a new GCP dynamic adapter.
2020
func NewListableAdapter(listEndpoint string, config *AdapterConfig) discovery.ListableAdapter {
21-
var potentialLinks []string
22-
if blasts, ok := gcpshared.BlastPropagations[config.SDPAssetType]; ok {
23-
for item := range blasts {
24-
potentialLinks = append(potentialLinks, item.String())
25-
}
26-
}
2721

2822
return ListableAdapter{
2923
listEndpoint: listEndpoint,
@@ -39,7 +33,7 @@ func NewListableAdapter(listEndpoint string, config *AdapterConfig) discovery.Li
3933
sdpAdapterCategory: config.SDPAdapterCategory,
4034
terraformMappings: config.TerraformMappings,
4135
linker: config.Linker,
42-
potentialLinks: potentialLinks,
36+
potentialLinks: potentialLinksFromBlasts(config.SDPAssetType, gcpshared.BlastPropagations),
4337
uniqueAttributeKeys: config.UniqueAttributeKeys,
4438
},
4539
}
@@ -70,7 +64,8 @@ func (g ListableAdapter) List(ctx context.Context, scope string, ignoreCache boo
7064
}
7165

7266
var items []*sdp.Item
73-
multiResp, err := externalCallMulti(ctx, g.httpCli, g.httpHeaders, g.listEndpoint)
67+
itemsSelector := g.uniqueAttributeKeys[len(g.uniqueAttributeKeys)-1] // Use the last key as the item selector
68+
multiResp, err := externalCallMulti(ctx, itemsSelector, g.httpCli, g.httpHeaders, g.listEndpoint)
7469
if err != nil {
7570
return nil, fmt.Errorf("failed to list items for %s: %w", g.listEndpoint, err)
7671
}

‎sources/gcp/dynamic/adapter.go

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,6 @@ type Adapter struct {
4242

4343
// NewAdapter creates a new GCP dynamic adapter.
4444
func NewAdapter(config *AdapterConfig) discovery.Adapter {
45-
var potentialLinks []string
46-
if blasts, ok := gcpshared.BlastPropagations[config.SDPAssetType]; ok {
47-
for item := range blasts {
48-
potentialLinks = append(potentialLinks, item.String())
49-
}
50-
}
5145

5246
return Adapter{
5347
projectID: config.ProjectID,
@@ -61,7 +55,7 @@ func NewAdapter(config *AdapterConfig) discovery.Adapter {
6155
sdpAdapterCategory: config.SDPAdapterCategory,
6256
terraformMappings: config.TerraformMappings,
6357
linker: config.Linker,
64-
potentialLinks: potentialLinks,
58+
potentialLinks: potentialLinksFromBlasts(config.SDPAssetType, gcpshared.BlastPropagations),
6559
uniqueAttributeKeys: config.UniqueAttributeKeys,
6660
}
6761
}

‎sources/gcp/dynamic/shared.go

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ func externalCallSingle(ctx context.Context, httpCli *http.Client, httpHeaders h
112112
return result, nil
113113
}
114114

115-
func externalCallMulti(ctx context.Context, httpCli *http.Client, httpHeader http.Header, url string) ([]map[string]interface{}, error) {
115+
func externalCallMulti(ctx context.Context, itemsSelector string, httpCli *http.Client, httpHeader http.Header, url string) ([]map[string]interface{}, error) {
116116
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
117117
if err != nil {
118118
return nil, err
@@ -140,13 +140,16 @@ func externalCallMulti(ctx context.Context, httpCli *http.Client, httpHeader htt
140140
return nil, err
141141
}
142142

143-
items, ok := result["items"].([]any)
143+
items, ok := result[itemsSelector].([]any)
144144
if !ok {
145-
log.WithContext(ctx).WithFields(log.Fields{
146-
"url": url,
147-
}).Warnf("failed to cast resp as a list of items: %v", result)
148-
149-
return nil, nil
145+
// fallback to a generic "items" key if the itemsSelector is not found
146+
items, ok = result["items"].([]any)
147+
if !ok {
148+
log.WithContext(ctx).WithFields(log.Fields{
149+
"url": url,
150+
}).Warnf("failed to cast resp as a list of items: %v", result)
151+
return nil, nil
152+
}
150153
}
151154

152155
var ii []map[string]interface{}
@@ -158,3 +161,17 @@ func externalCallMulti(ctx context.Context, httpCli *http.Client, httpHeader htt
158161

159162
return ii, nil
160163
}
164+
165+
func potentialLinksFromBlasts(itemType shared.ItemType, blasts map[shared.ItemType]map[string]*gcpshared.Impact) []string {
166+
var potentialLinks []string
167+
var potentialLinksMap = make(map[string]bool)
168+
for _, impact := range blasts[itemType] {
169+
potentialLinksMap[impact.ToSDPITemType.String()] = true
170+
}
171+
172+
for it := range potentialLinksMap {
173+
potentialLinks = append(potentialLinks, it)
174+
}
175+
176+
return potentialLinks
177+
}

‎sources/gcp/integration-tests/compute-instance_test.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,7 @@ func TestComputeInstanceIntegration(t *testing.T) {
5050
t.Run("Run", func(t *testing.T) {
5151
log.Printf("Running integration test for Compute Instance in project %s, zone %s", projectID, zone)
5252

53-
// TODO: Use the actual linker with this ticket: https://linear.app/overmind/issue/ENG-523/centralise-integration-tests
54-
instanceWrapper := manual.NewComputeInstance(gcpshared.NewComputeInstanceClient(client), projectID, zone, nil)
53+
instanceWrapper := manual.NewComputeInstance(gcpshared.NewComputeInstanceClient(client), projectID, zone)
5554
scope := instanceWrapper.Scopes()[0]
5655

5756
instanceAdapter := sources.WrapperToAdapter(instanceWrapper)

‎sources/gcp/manual/adapters.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
)
1414

1515
// Adapters returns a slice of discovery.Adapter instances for GCP Source.
16-
func Adapters(ctx context.Context, projectID string, regions []string, zones []string, linker *shared.Linker) ([]discovery.Adapter, error) {
16+
func Adapters(ctx context.Context, projectID string, regions []string, zones []string) ([]discovery.Adapter, error) {
1717
instanceCli, err := compute.NewInstancesRESTClient(ctx)
1818
if err != nil {
1919
return nil, err
@@ -121,7 +121,7 @@ func Adapters(ctx context.Context, projectID string, regions []string, zones []s
121121

122122
for _, zone := range zones {
123123
adapters = append(adapters,
124-
sources.WrapperToAdapter(NewComputeInstance(shared.NewComputeInstanceClient(instanceCli), projectID, zone, linker)),
124+
sources.WrapperToAdapter(NewComputeInstance(shared.NewComputeInstanceClient(instanceCli), projectID, zone)),
125125
sources.WrapperToAdapter(NewComputeAutoscaler(shared.NewComputeAutoscalerClient(autoscalerCli), projectID, zone)),
126126
sources.WrapperToAdapter(NewComputeInstanceGroup(shared.NewComputeInstanceGroupsClient(instanceGroupCli), projectID, zone)),
127127
sources.WrapperToAdapter(NewComputeInstanceGroupManager(shared.NewComputeInstanceGroupManagerClient(instanceGroupManagerCli), projectID, zone)),

‎sources/gcp/manual/compute-instance.go

Lines changed: 61 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package manual
33
import (
44
"context"
55
"errors"
6+
"strings"
67

78
"cloud.google.com/go/compute/apiv1/computepb"
89
"google.golang.org/api/iterator"
@@ -24,16 +25,14 @@ var (
2425

2526
type computeInstanceWrapper struct {
2627
client gcpshared.ComputeInstanceClient
27-
linker *gcpshared.Linker
2828

2929
*gcpshared.ZoneBase
3030
}
3131

3232
// NewComputeInstance creates a new computeInstanceWrapper instance
33-
func NewComputeInstance(client gcpshared.ComputeInstanceClient, projectID, zone string, linker *gcpshared.Linker) sources.ListableWrapper {
33+
func NewComputeInstance(client gcpshared.ComputeInstanceClient, projectID, zone string) sources.ListableWrapper {
3434
return &computeInstanceWrapper{
3535
client: client,
36-
linker: linker,
3736
ZoneBase: gcpshared.NewZoneBase(
3837
projectID,
3938
zone,
@@ -87,7 +86,7 @@ func (c computeInstanceWrapper) Get(ctx context.Context, queryParts ...string) (
8786

8887
var sdpErr *sdp.QueryError
8988
var item *sdp.Item
90-
item, sdpErr = c.gcpComputeInstanceToSDPItem(ctx, instance)
89+
item, sdpErr = c.gcpComputeInstanceToSDPItem(instance)
9190
if sdpErr != nil {
9291
return nil, sdpErr
9392
}
@@ -114,7 +113,7 @@ func (c computeInstanceWrapper) List(ctx context.Context) ([]*sdp.Item, *sdp.Que
114113

115114
var sdpErr *sdp.QueryError
116115
var item *sdp.Item
117-
item, sdpErr = c.gcpComputeInstanceToSDPItem(ctx, instance)
116+
item, sdpErr = c.gcpComputeInstanceToSDPItem(instance)
118117
if sdpErr != nil {
119118
return nil, sdpErr
120119
}
@@ -125,7 +124,7 @@ func (c computeInstanceWrapper) List(ctx context.Context) ([]*sdp.Item, *sdp.Que
125124
return items, nil
126125
}
127126

128-
func (c computeInstanceWrapper) gcpComputeInstanceToSDPItem(ctx context.Context, instance *computepb.Instance) (*sdp.Item, *sdp.QueryError) {
127+
func (c computeInstanceWrapper) gcpComputeInstanceToSDPItem(instance *computepb.Instance) (*sdp.Item, *sdp.QueryError) {
129128
attributes, err := shared.ToAttributesWithExclude(instance, "labels")
130129
if err != nil {
131130
return nil, &sdp.QueryError{
@@ -147,7 +146,25 @@ func (c computeInstanceWrapper) gcpComputeInstanceToSDPItem(ctx context.Context,
147146
// Specifies a valid partial or full URL to an existing Persistent Disk resource.
148147
// "source": "https://www.googleapis.com/compute/v1/projects/project-test/zones/us-central1-c/disks/integration-test-instance"
149148
// last part is the disk name
150-
c.linker.Link(ctx, c.ProjectID(), sdpItem, ComputeInstance, disk.GetSource(), ComputeDisk)
149+
if strings.Contains(disk.GetSource(), "/") {
150+
diskNameParts := strings.Split(disk.GetSource(), "/")
151+
diskName := diskNameParts[len(diskNameParts)-1]
152+
zone := gcpshared.ExtractPathParam("zones", disk.GetSource())
153+
if zone != "" {
154+
sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{
155+
Query: &sdp.Query{
156+
Type: ComputeDisk.String(),
157+
Method: sdp.QueryMethod_GET,
158+
Query: diskName,
159+
Scope: gcpshared.ZonalScope(c.ProjectID(), zone),
160+
},
161+
BlastPropagation: &sdp.BlastPropagation{
162+
In: true,
163+
Out: true,
164+
},
165+
})
166+
}
167+
}
151168
}
152169
}
153170

@@ -194,7 +211,26 @@ func (c computeInstanceWrapper) gcpComputeInstanceToSDPItem(ctx context.Context,
194211
// - regions/region/subnetworks/subnetwork
195212
// "subnetwork": "https://www.googleapis.com/compute/v1/projects/project-test/regions/us-central1/subnetworks/default"
196213
// last part is the subnetwork name
197-
c.linker.Link(ctx, c.ProjectID(), sdpItem, ComputeInstance, subnetwork, ComputeSubnetwork)
214+
if strings.Contains(subnetwork, "/") {
215+
subnetworkNameParts := strings.Split(subnetwork, "/")
216+
subnetworkName := subnetworkNameParts[len(subnetworkNameParts)-1]
217+
region := gcpshared.ExtractPathParam("regions", subnetwork)
218+
if region != "" {
219+
sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{
220+
Query: &sdp.Query{
221+
Type: ComputeSubnetwork.String(),
222+
Method: sdp.QueryMethod_GET,
223+
Query: subnetworkName,
224+
// This is a regional resource
225+
Scope: gcpshared.RegionalScope(c.ProjectID(), region),
226+
},
227+
BlastPropagation: &sdp.BlastPropagation{
228+
In: true,
229+
Out: false,
230+
},
231+
})
232+
}
233+
}
198234
}
199235

200236
if network := networkInterface.GetNetwork(); network != "" {
@@ -210,7 +246,23 @@ func (c computeInstanceWrapper) gcpComputeInstanceToSDPItem(ctx context.Context,
210246
// - global/networks/default
211247
//
212248
// "network": "https://www.googleapis.com/compute/v1/projects/project-test/global/networks/default"
213-
c.linker.Link(ctx, c.ProjectID(), sdpItem, ComputeInstance, network, ComputeNetwork)
249+
if strings.Contains(network, "/") {
250+
networkNameParts := strings.Split(network, "/")
251+
networkName := networkNameParts[len(networkNameParts)-1]
252+
sdpItem.LinkedItemQueries = append(sdpItem.LinkedItemQueries, &sdp.LinkedItemQuery{
253+
Query: &sdp.Query{
254+
Type: ComputeNetwork.String(),
255+
Method: sdp.QueryMethod_GET,
256+
Query: networkName,
257+
// This is a global resource
258+
Scope: c.ProjectID(),
259+
},
260+
BlastPropagation: &sdp.BlastPropagation{
261+
In: true,
262+
Out: false,
263+
},
264+
})
265+
}
214266
}
215267
}
216268
}

‎sources/gcp/manual/compute-instance_test.go

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212
"github.com/overmindtech/cli/sdp-go"
1313
"github.com/overmindtech/cli/sources"
1414
"github.com/overmindtech/cli/sources/gcp/manual"
15-
gcpshared "github.com/overmindtech/cli/sources/gcp/shared"
1615
"github.com/overmindtech/cli/sources/gcp/shared/mocks"
1716
"github.com/overmindtech/cli/sources/shared"
1817
"github.com/overmindtech/cli/sources/stdlib"
@@ -27,10 +26,8 @@ func TestComputeInstance(t *testing.T) {
2726
projectID := "test-project-id"
2827
zone := "us-central1-a"
2928

30-
linker := gcpshared.NewLinker()
31-
3229
t.Run("Get", func(t *testing.T) {
33-
wrapper := manual.NewComputeInstance(mockClient, projectID, zone, linker)
30+
wrapper := manual.NewComputeInstance(mockClient, projectID, zone)
3431

3532
mockClient.EXPECT().Get(ctx, gomock.Any()).Return(createComputeInstance("test-instance", computepb.Instance_RUNNING), nil)
3633

@@ -161,7 +158,7 @@ func TestComputeInstance(t *testing.T) {
161158

162159
for _, tc := range testCases {
163160
t.Run(tc.name, func(t *testing.T) {
164-
wrapper := manual.NewComputeInstance(mockClient, projectID, zone, linker)
161+
wrapper := manual.NewComputeInstance(mockClient, projectID, zone)
165162
adapter := sources.WrapperToAdapter(wrapper)
166163

167164
mockClient.EXPECT().Get(ctx, gomock.Any()).Return(createComputeInstance("test-instance", tc.input), nil)
@@ -179,7 +176,7 @@ func TestComputeInstance(t *testing.T) {
179176
})
180177

181178
t.Run("List", func(t *testing.T) {
182-
wrapper := manual.NewComputeInstance(mockClient, projectID, zone, linker)
179+
wrapper := manual.NewComputeInstance(mockClient, projectID, zone)
183180

184181
adapter := sources.WrapperToAdapter(wrapper)
185182

‎sources/gcp/proc/proc.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func Initialize(ctx context.Context, ec *discovery.EngineConfig) (*discovery.Eng
3737

3838
linker := gcpshared.NewLinker()
3939

40-
manualAdapters, err := manual.Adapters(ctx, cfg.ProjectID, cfg.Regions, cfg.Zones, linker)
40+
manualAdapters, err := manual.Adapters(ctx, cfg.ProjectID, cfg.Regions, cfg.Zones)
4141
if err != nil {
4242
return nil, fmt.Errorf("error creating manual adapters: %w", err)
4343
}

‎sources/gcp/shared/adapter-meta.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,8 +261,9 @@ var SDPAssetTypeToAdapterMeta = map[shared.ItemType]AdapterMeta{
261261
Scope: ScopeProject,
262262
// Format: projects/{project}/locations/{location}/pipelineJobs/{pipelineJob}
263263
GetEndpointBaseURLFunc: projectLevelEndpointFuncWithTwoQueries("https://aiplatform.googleapis.com/v1/projects/%s/locations/%s/pipelineJobs/%s"),
264-
SearchEndpointFunc: projectLevelEndpointFuncWithSingleQuery("https://aiplatform.googleapis.com/v1/projects/%s/locations/%s/pipelineJobs"),
265-
UniqueAttributeKeys: []string{"locations", "pipelineJobs"},
264+
// Reference: https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.pipelineJobs/list
265+
SearchEndpointFunc: projectLevelEndpointFuncWithSingleQuery("https://aiplatform.googleapis.com/v1/projects/%s/locations/%s/pipelineJobs"),
266+
UniqueAttributeKeys: []string{"locations", "pipelineJobs"},
266267
},
267268
ArtifactRegistryDockerImage: {
268269
SDPAdapterCategory: sdp.AdapterCategory_ADAPTER_CATEGORY_STORAGE,
@@ -312,6 +313,7 @@ var SDPAssetTypeToAdapterMeta = map[shared.ItemType]AdapterMeta{
312313
Scope: ScopeProject,
313314
// https://bigquery.googleapis.com/bigquery/v2/projects/{projectId}/datasets/{datasetId}
314315
GetEndpointBaseURLFunc: projectLevelEndpointFuncWithSingleQuery("https://bigquery.googleapis.com/bigquery/v2/projects/%s/datasets/%s"),
316+
// Reference: https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/list
315317
// https://bigquery.googleapis.com/bigquery/v2/projects/{projectId}/datasets
316318
ListEndpointFunc: projectLevelListFunc("https://bigquery.googleapis.com/bigquery/v2/projects/%s/datasets"),
317319
UniqueAttributeKeys: []string{"datasets"},
@@ -321,6 +323,7 @@ var SDPAssetTypeToAdapterMeta = map[shared.ItemType]AdapterMeta{
321323
Scope: ScopeProject,
322324
// https://bigquery.googleapis.com/bigquery/v2/projects/{projectId}/datasets/{datasetId}/tables/{tableId}
323325
GetEndpointBaseURLFunc: projectLevelEndpointFuncWithTwoQueries("https://bigquery.googleapis.com/bigquery/v2/projects/%s/datasets/%s/tables/%s"),
326+
// Reference: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list
324327
// https://bigquery.googleapis.com/bigquery/v2/projects/{projectId}/datasets/{datasetId}/tables
325328
// TODO: Update this for => https://linear.app/overmind/issue/ENG-580/handle-terraform-mappings-in-search-method
326329
// id => projects/{{project}}/datasets/{{dataset}}/tables/{{table}}
@@ -421,6 +424,7 @@ var SDPAssetTypeToAdapterMeta = map[shared.ItemType]AdapterMeta{
421424
Scope: ScopeProject,
422425
// https://compute.googleapis.com/compute/v1/projects/{project}/global/firewalls/{firewall}
423426
GetEndpointBaseURLFunc: projectLevelEndpointFuncWithSingleQuery("https://compute.googleapis.com/compute/v1/projects/%s/global/firewalls/%s"),
427+
// Reference: https://cloud.google.com/compute/docs/reference/rest/v1/firewalls/list
424428
// https://compute.googleapis.com/compute/v1/projects/{project}/global/firewalls
425429
ListEndpointFunc: projectLevelListFunc("https://compute.googleapis.com/compute/v1/projects/%s/global/firewalls"),
426430
UniqueAttributeKeys: []string{"firewalls"},
@@ -674,6 +678,7 @@ var SDPAssetTypeToAdapterMeta = map[shared.ItemType]AdapterMeta{
674678
Scope: ScopeProject,
675679
// https://pubsub.googleapis.com/v1/projects/{project}/subscriptions/{subscription}
676680
GetEndpointBaseURLFunc: projectLevelEndpointFuncWithSingleQuery("https://pubsub.googleapis.com/v1/projects/%s/subscriptions/%s"),
681+
// Reference: https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/list?rep_location=global
677682
// https://pubsub.googleapis.com/v1/projects/{project}/subscriptions
678683
ListEndpointFunc: projectLevelListFunc("https://pubsub.googleapis.com/v1/projects/%s/subscriptions"),
679684
UniqueAttributeKeys: []string{"subscriptions"},
@@ -748,6 +753,7 @@ var SDPAssetTypeToAdapterMeta = map[shared.ItemType]AdapterMeta{
748753
// Reference: https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/Backups/GetBackup
749754
// GET https://sqladmin.googleapis.com/v1/{name=projects/*/backups/*}
750755
GetEndpointBaseURLFunc: projectLevelEndpointFuncWithSingleQuery("https://sqladmin.googleapis.com/v1/projects/%s/backups/%s"),
756+
// Reference: https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/Backups/ListBackups
751757
// GET https://sqladmin.googleapis.com/v1/{parent=projects/*}/backups
752758
ListEndpointFunc: projectLevelListFunc("https://sqladmin.googleapis.com/v1/projects/%s/backups"),
753759
UniqueAttributeKeys: []string{"backups"},

0 commit comments

Comments
 (0)