Eng 582 support search in dynamic adapters (#1877)

# Support SEARCH in dynamic adapters

We currently have all the necessary meta data for creating a searchable
adapter.
We only need to:

* Create a searchable adapters
* Initiate them during source startup

It is important to note that, in the SEARCH method we need to check the
query to see if it is actually for a terraform mappings and targeting a
single resource rather than multiple. Simply, if the query contains and
`/`, then it is for a single query, we should use the
`externalCallSingle` and return the single response within a slice of
`items` to comply with the `Search` method signature. Relevant:
[ENG-580](https://linear.app/overmind/issue/ENG-580/handle-terraform-mappings-in-search-method)

GitOrigin-RevId: 13ee3ef6b5e51acc20b787ff9a728523d5a2e96c

Author: omerdemirok

sources/gcp/dynamic/adapter-listable.go

@@ -46,9 +46,9 @@ func (g ListableAdapter) Metadata() *sdp.AdapterMetadata {
 		DescriptiveName: g.sdpAssetType.Readable(),
 		SupportedQueryMethods: &sdp.AdapterSupportedQueryMethods{
 			Get:             true,
-			GetDescription:  fmt.Sprintf("Get a %s by its name i.e: zones/<zone>/instances/<instance-name>", g.sdpAssetType),
+			GetDescription:  getDescription(g.sdpAssetType, g.scope, g.uniqueAttributeKeys),
 			List:            true,
-			ListDescription: fmt.Sprintf("List all %s within its scopes: %v", g.sdpAssetType, g.Scopes()),
+			ListDescription: listDescription(g.sdpAssetType, g.scope),
 		},
 		TerraformMappings: g.terraformMappings,
 		PotentialLinks:    g.potentialLinks,
@@ -67,7 +67,7 @@ func (g ListableAdapter) List(ctx context.Context, scope string, ignoreCache boo
 	itemsSelector := g.uniqueAttributeKeys[len(g.uniqueAttributeKeys)-1] // Use the last key as the item selector
 	multiResp, err := externalCallMulti(ctx, itemsSelector, g.httpCli, g.httpHeaders, g.listEndpoint)
 	if err != nil {
-		return nil, fmt.Errorf("failed to list items for %s: %w", g.listEndpoint, err)
+		return nil, fmt.Errorf("failed to retrieve items for %s: %w", g.listEndpoint, err)
 	}
 
 	for _, resp := range multiResp {

sources/gcp/dynamic/adapter-searchable.go

@@ -0,0 +1,101 @@
+package dynamic
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/overmindtech/cli/discovery"
+	"github.com/overmindtech/cli/sdp-go"
+	gcpshared "github.com/overmindtech/cli/sources/gcp/shared"
+)
+
+// SearchableAdapter implements discovery.SearchableAdapter for GCP dynamic adapters.
+type SearchableAdapter struct {
+	searchURLFunc gcpshared.EndpointFunc
+	Adapter
+}
+
+// NewSearchableAdapter creates a new GCP dynamic adapter.
+func NewSearchableAdapter(searchURLFunc gcpshared.EndpointFunc, config *AdapterConfig) discovery.SearchableAdapter {
+
+	return SearchableAdapter{
+		searchURLFunc: searchURLFunc,
+		Adapter: Adapter{
+			projectID:  config.ProjectID,
+			scope:      config.Scope,
+			httpCli:    config.HTTPClient,
+			getURLFunc: config.GetURLFunc,
+			httpHeaders: http.Header{
+				"Authorization": []string{"Bearer " + config.Token},
+			},
+			sdpAssetType:        config.SDPAssetType,
+			sdpAdapterCategory:  config.SDPAdapterCategory,
+			terraformMappings:   config.TerraformMappings,
+			linker:              config.Linker,
+			potentialLinks:      potentialLinksFromBlasts(config.SDPAssetType, gcpshared.BlastPropagations),
+			uniqueAttributeKeys: config.UniqueAttributeKeys,
+		},
+	}
+}
+
+func (g SearchableAdapter) Metadata() *sdp.AdapterMetadata {
+	return &sdp.AdapterMetadata{
+		Type:            g.sdpAssetType.String(),
+		Category:        g.sdpAdapterCategory,
+		DescriptiveName: g.sdpAssetType.Readable(),
+		SupportedQueryMethods: &sdp.AdapterSupportedQueryMethods{
+			Get:               true,
+			GetDescription:    getDescription(g.sdpAssetType, g.scope, g.uniqueAttributeKeys),
+			Search:            true,
+			SearchDescription: searchDescription(g.sdpAssetType, g.scope, g.uniqueAttributeKeys),
+		},
+		TerraformMappings: g.terraformMappings,
+		PotentialLinks:    g.potentialLinks,
+	}
+}
+
+func (g SearchableAdapter) Search(ctx context.Context, scope, query string, ignoreCache bool) ([]*sdp.Item, error) {
+	if scope != g.scope {
+		return nil, &sdp.QueryError{
+			ErrorType:   sdp.QueryError_NOSCOPE,
+			ErrorString: fmt.Sprintf("requested scope %v does not match any adapter scope %v", scope, g.Scopes()),
+		}
+	}
+	searchEndpoint := g.searchURLFunc(query)
+	var items []*sdp.Item
+	itemsSelector := g.uniqueAttributeKeys[len(g.uniqueAttributeKeys)-1] // Use the last key as the item selector
+
+	if strings.HasPrefix(query, "projects/") {
+		// This is a single item query for terraform search method mappings.
+		// See: https://linear.app/overmind/issue/ENG-580/handle-terraform-mappings-in-search-method
+		resp, err := externalCallSingle(ctx, g.httpCli, g.httpHeaders, searchEndpoint)
+		if err != nil {
+			return nil, err
+		}
+
+		item, err := externalToSDP(ctx, g.projectID, g.scope, g.uniqueAttributeKeys, resp, g.sdpAssetType, g.linker)
+		if err != nil {
+			return nil, err
+		}
+
+		return append(items, item), nil
+	}
+
+	multiResp, err := externalCallMulti(ctx, itemsSelector, g.httpCli, g.httpHeaders, searchEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve items for %s: %w", searchEndpoint, err)
+	}
+
+	for _, resp := range multiResp {
+		item, err := externalToSDP(ctx, g.projectID, g.scope, g.uniqueAttributeKeys, resp, g.sdpAssetType, g.linker)
+		if err != nil {
+			return nil, err
+		}
+
+		items = append(items, item)
+	}
+
+	return items, nil
+}

sources/gcp/dynamic/adapter.go

@@ -75,7 +75,7 @@ func (g Adapter) Metadata() *sdp.AdapterMetadata {
 		DescriptiveName: g.sdpAssetType.Readable(),
 		SupportedQueryMethods: &sdp.AdapterSupportedQueryMethods{
 			Get:            true,
-			GetDescription: fmt.Sprintf("Get a %s by its unique name within its scope: %s", g.sdpAssetType, g.scope),
+			GetDescription: getDescription(g.sdpAssetType, g.scope, g.uniqueAttributeKeys),
 		},
 		TerraformMappings: g.terraformMappings,
 		PotentialLinks:    g.potentialLinks,

sources/gcp/dynamic/adapters.go

@@ -63,6 +63,17 @@ func Adapters(projectID string, token string, regions []string, zones []string,
 			continue
 		}
 
+		if meta.SearchEndpointFunc != nil {
+			searchEndpointFunc, err := meta.SearchEndpointFunc(projectID)
+			if err != nil {
+				return nil, err
+			}
+
+			adapters = append(adapters, NewSearchableAdapter(searchEndpointFunc, cfg))
+
+			continue
+		}
+
 		adapters = append(adapters, NewAdapter(cfg))
 	}
 
@@ -104,6 +115,17 @@ func Adapters(projectID string, token string, regions []string, zones []string,
 				continue
 			}
 
+			if meta.SearchEndpointFunc != nil {
+				searchEndpointFunc, err := meta.SearchEndpointFunc(projectID, region)
+				if err != nil {
+					return nil, err
+				}
+
+				adapters = append(adapters, NewSearchableAdapter(searchEndpointFunc, cfg))
+
+				continue
+			}
+
 			adapters = append(adapters, NewAdapter(cfg))
 		}
 	}
@@ -135,6 +157,7 @@ func Adapters(projectID string, token string, regions []string, zones []string,
 				HTTPClient:          otelhttp.DefaultClient,
 				UniqueAttributeKeys: meta.UniqueAttributeKeys,
 			}
+
 			if meta.ListEndpointFunc != nil {
 				listEndpoint, err := meta.ListEndpointFunc(projectID, zone)
 				if err != nil {
@@ -145,6 +168,17 @@ func Adapters(projectID string, token string, regions []string, zones []string,
 				continue
 			}
 
+			if meta.SearchEndpointFunc != nil {
+				searchEndpointFunc, err := meta.SearchEndpointFunc(projectID, zone)
+				if err != nil {
+					return nil, err
+				}
+
+				adapters = append(adapters, NewSearchableAdapter(searchEndpointFunc, cfg))
+
+				continue
+			}
+
 			adapters = append(adapters, NewAdapter(cfg))
 		}
 	}

sources/gcp/dynamic/shared.go

@@ -15,6 +15,35 @@ import (
 	"github.com/overmindtech/cli/sources/shared"
 )
 
+var (
+	getDescription = func(sdpAssetType shared.ItemType, scope string, uniqueAttributeKeys []string) string {
+		selector := "{name}"
+		if len(uniqueAttributeKeys) > 1 {
+			// i.e.: {datasets|tables} for bigquery tables
+			selector = "{" + strings.Join(uniqueAttributeKeys, shared.QuerySeparator) + "}"
+		}
+
+		return fmt.Sprintf("Get a %s by its %s within its scope: %s", sdpAssetType, selector, scope)
+	}
+
+	listDescription = func(sdpAssetType shared.ItemType, scope string) string {
+		return fmt.Sprintf("List all %s within its scope: %s", sdpAssetType, scope)
+	}
+
+	searchDescription = func(sdpAssetType shared.ItemType, scope string, uniqueAttributeKeys []string) string {
+		if len(uniqueAttributeKeys) < 2 {
+			panic("searchDescription requires at least two unique attribute keys")
+		}
+		// For service directory endpoint adapter, the uniqueAttributeKeys is: []string{"locations", "namespaces", "services", "endpoints"}
+		// We want to create a selector like:
+		// {locations|namespaces|services}
+		// We remove the last key, because it defines the actual item selector
+		selector := "{" + strings.Join(uniqueAttributeKeys[:len(uniqueAttributeKeys)-1], shared.QuerySeparator) + "}"
+
+		return fmt.Sprintf("Search for %s by its %s within its scope: %s", sdpAssetType, selector, scope)
+	}
+)
+
 func linkItem(ctx context.Context, projectID string, sdpItem *sdp.Item, sdpAssetType shared.ItemType, linker *gcpshared.Linker, resp any, keys []string) {
 	if value, ok := resp.(string); ok {
 		linker.AutoLink(ctx, projectID, sdpItem, sdpAssetType, value, keys)
@@ -62,6 +91,7 @@ func externalToSDP(ctx context.Context, projectID string, scope string, uniqueAt
 	}
 
 	// We need to keep an eye on this.
+	// Name might not exist in the response for all APIs.
 	if name, ok := resp["name"].(string); ok {
 		attrValues := gcpshared.ExtractPathParams(name, uniqueAttrKeys...)
 		uniqueAttrValue := strings.Join(attrValues, shared.QuerySeparator)
@@ -70,7 +100,7 @@ func externalToSDP(ctx context.Context, projectID string, scope string, uniqueAt
 			return nil, err
 		}
 	} else {
-		return nil, fmt.Errorf("unable to determine self link")
+		return nil, fmt.Errorf("unable to determine the name")
 	}
 
 	for k, v := range resp {
@@ -96,6 +126,15 @@ func externalCallSingle(ctx context.Context, httpCli *http.Client, httpHeaders h
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(resp.Body)
+		if err == nil {
+			return nil, fmt.Errorf("failed to make a GET call: %s, HTTP Status: %s, HTTP Body: %s", url, resp.Status, string(body))
+		}
+
+		log.WithContext(ctx).WithFields(log.Fields{
+			"ovm.gcp.dynamic.http.get.url":            url,
+			"ovm.gcp.dynamic.http.get.responseStatus": resp.Status,
+		}).Warnf("failed to read the response body: %v", err)
 		return nil, fmt.Errorf("failed to make call: %s", resp.Status)
 	}
 
@@ -127,7 +166,17 @@ func externalCallMulti(ctx context.Context, itemsSelector string, httpCli *http.
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("failed to make the GET call for the %s URL. HTTP Status: %s", url, resp.Status)
+		// Read the body to provide more context in the error message
+		body, err := io.ReadAll(resp.Body)
+		if err == nil {
+			return nil, fmt.Errorf("failed to make the GET call. HTTP Status: %s, HTTP Body: %s", resp.Status, string(body))
+		}
+
+		log.WithContext(ctx).WithFields(log.Fields{
+			"ovm.gcp.dynamic.http.get.url":            url,
+			"ovm.gcp.dynamic.http.get.responseStatus": resp.Status,
+		}).Warnf("failed to read the response body: %v", err)
+		return nil, fmt.Errorf("failed to make the GET callL. HTTP Status: %s", resp.Status)
 	}
 
 	data, err := io.ReadAll(resp.Body)
@@ -142,12 +191,13 @@ func externalCallMulti(ctx context.Context, itemsSelector string, httpCli *http.
 
 	items, ok := result[itemsSelector].([]any)
 	if !ok {
-		// fallback to a generic "items" key if the itemsSelector is not found
-		items, ok = result["items"].([]any)
+		itemsSelector = "items" // Fallback to a generic "items" key
+		items, ok = result[itemsSelector].([]any)
 		if !ok {
 			log.WithContext(ctx).WithFields(log.Fields{
-				"url": url,
-			}).Warnf("failed to cast resp as a list of items: %v", result)
+				"ovm.gcp.dynamic.http.get.url":           url,
+				"ovm.gcp.dynamic.http.get.itemsSelector": itemsSelector,
+			}).Warnf("failed to cast resp as a list of %s: %v", itemsSelector, result)
 			return nil, nil
 		}
 	}