Skip to content

Commit 106b591

Browse files
pgierpgier
authored
STREAM-152: fix bugs in streaming topic creation (#278)
- send topic requests with application/json header - enable topic tests in CI - refactor to get tests working against dev environment
1 parent 002c4fe commit 106b591

7 files changed

+99
-86
lines changed

‎internal/provider/provider.go

+5
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"net/http"
88
"net/url"
99
"os"
10+
"strings"
1011

1112
astrarestapi "github.com/datastax/astra-client-go/v2/astra-rest-api"
1213
astrastreaming "github.com/datastax/astra-client-go/v2/astra-streaming"
@@ -180,6 +181,9 @@ func configure(providerVersion string, p *schema.Provider) func(context.Context,
180181
providerVersion: providerVersion,
181182
userAgent: userAgent,
182183
}
184+
if strings.Contains(streamingAPIServerURL, "staging") {
185+
clients.streamingTestMode = true
186+
}
183187
return clients, nil
184188
}
185189
}
@@ -223,6 +227,7 @@ type astraClients struct {
223227
stargateClientCache map[string]astrarestapi.Client
224228
providerVersion string
225229
userAgent string
230+
streamingTestMode bool
226231
}
227232

228233
// resourceDataOrDefaultString returns the value in the given ResourceData variable or a default value

‎internal/provider/provider_framework.go

+4
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ type astraClients2 struct {
5959
stargateClientCache map[string]astrarestapi.Client
6060
providerVersion string
6161
userAgent string
62+
streamingTestMode bool
6263
}
6364

6465
// Metadata returns the provider type name.
@@ -196,6 +197,9 @@ func (p *astraProvider) Configure(ctx context.Context, req provider.ConfigureReq
196197
providerVersion: p.Version,
197198
userAgent: userAgent,
198199
}
200+
if strings.Contains(streamingAPIServerURL, "staging") {
201+
clients.streamingTestMode = true
202+
}
199203
resp.ResourceData = clients
200204
resp.DataSourceData = clients
201205
}

‎internal/provider/resource_cdc.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ func prepCDC(ctx context.Context, client *astra.ClientWithResponses, databaseId
416416
cloudProvider := string(*db.Info.CloudProvider)
417417
fmt.Printf("%s", cloudProvider)
418418

419-
pulsarCluster := getPulsarCluster(cloudProvider, *db.Info.Region)
419+
pulsarCluster := getPulsarCluster(cloudProvider, *db.Info.Region, false)
420420
pulsarToken, err := getPulsarToken(ctx, pulsarCluster, token, org, err, streamingClient, tenantName)
421421
return pulsarCluster, pulsarToken, err
422422
}

‎internal/provider/resource_streaming_sink.go

+22-22
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,9 @@ func resourceStreamingSinkDelete(ctx context.Context, resourceData *schema.Resou
122122
return diag.Errorf("\"deletion_protection\" must be explicitly set to \"false\" in order to destroy astra_streaming_sink")
123123
}
124124

125-
streamingClient := meta.(astraClients).astraStreamingClient.(*astrastreaming.ClientWithResponses)
126-
client := meta.(astraClients).astraClient.(*astra.ClientWithResponses)
127-
streamingClientv3 := meta.(astraClients).astraStreamingClientv3
125+
astraClients := meta.(astraClients)
126+
streamingClient := astraClients.astraStreamingClient.(*astrastreaming.ClientWithResponses)
127+
astraClient := astraClients.astraClient.(*astra.ClientWithResponses)
128128

129129
tenantName := resourceData.Get("tenant_name").(string)
130130
sinkName := resourceData.Get("sink_name").(string)
@@ -134,9 +134,9 @@ func resourceStreamingSinkDelete(ctx context.Context, resourceData *schema.Resou
134134
region := strings.ReplaceAll(rawRegion, "-", "")
135135
cloudProvider := resourceData.Get("cloud_provider").(string)
136136

137-
pulsarCluster := getPulsarCluster(cloudProvider, region)
137+
pulsarCluster := getPulsarCluster(cloudProvider, region, astraClients.streamingTestMode)
138138

139-
orgBody, _ := client.GetCurrentOrganization(ctx)
139+
orgBody, _ := astraClient.GetCurrentOrganization(ctx)
140140

141141
var org OrgId
142142
bodyBuffer, err := ioutil.ReadAll(orgBody.Body)
@@ -149,7 +149,7 @@ func resourceStreamingSinkDelete(ctx context.Context, resourceData *schema.Resou
149149
fmt.Println("Can't deserialize", orgBody)
150150
}
151151

152-
token := meta.(astraClients).token
152+
token := astraClients.token
153153
pulsarToken, err := getPulsarToken(ctx, pulsarCluster, token, org, err, streamingClient, tenantName)
154154
if err != nil {
155155
return diag.FromErr(err)
@@ -160,7 +160,7 @@ func resourceStreamingSinkDelete(ctx context.Context, resourceData *schema.Resou
160160
Authorization: fmt.Sprintf("Bearer %s", pulsarToken),
161161
}
162162

163-
deleteSinkResponse, err := streamingClientv3.DeleteSinkWithResponse(ctx, tenantName, namespace, sinkName, &deleteSinkParams)
163+
deleteSinkResponse, err := astraClients.astraStreamingClientv3.DeleteSinkWithResponse(ctx, tenantName, namespace, sinkName, &deleteSinkParams)
164164
if err != nil {
165165
diag.FromErr(err)
166166
}
@@ -210,9 +210,9 @@ type SinkResponse struct {
210210
}
211211

212212
func resourceStreamingSinkRead(ctx context.Context, resourceData *schema.ResourceData, meta interface{}) diag.Diagnostics {
213-
streamingClient := meta.(astraClients).astraStreamingClient.(*astrastreaming.ClientWithResponses)
214-
client := meta.(astraClients).astraClient.(*astra.ClientWithResponses)
215-
streamingClientv3 := meta.(astraClients).astraStreamingClientv3
213+
astraClients := meta.(astraClients)
214+
streamingClient := astraClients.astraStreamingClient.(*astrastreaming.ClientWithResponses)
215+
astraClient := astraClients.astraClient.(*astra.ClientWithResponses)
216216

217217
tenantName := resourceData.Get("tenant_name").(string)
218218
sinkName := resourceData.Get("sink_name").(string)
@@ -223,9 +223,9 @@ func resourceStreamingSinkRead(ctx context.Context, resourceData *schema.Resourc
223223
region := strings.ReplaceAll(rawRegion, "-", "")
224224
cloudProvider := resourceData.Get("cloud_provider").(string)
225225

226-
pulsarCluster := getPulsarCluster(cloudProvider, region)
226+
pulsarCluster := getPulsarCluster(cloudProvider, region, astraClients.streamingTestMode)
227227

228-
orgBody, _ := client.GetCurrentOrganization(ctx)
228+
orgBody, _ := astraClient.GetCurrentOrganization(ctx)
229229

230230
var org OrgId
231231
bodyBuffer, err := ioutil.ReadAll(orgBody.Body)
@@ -235,7 +235,7 @@ func resourceStreamingSinkRead(ctx context.Context, resourceData *schema.Resourc
235235
fmt.Println("Can't deserislize", orgBody)
236236
}
237237

238-
token := meta.(astraClients).token
238+
token := astraClients.token
239239
pulsarToken, err := getPulsarToken(ctx, pulsarCluster, token, org, err, streamingClient, tenantName)
240240
if err != nil {
241241
diag.FromErr(err)
@@ -246,7 +246,7 @@ func resourceStreamingSinkRead(ctx context.Context, resourceData *schema.Resourc
246246
Authorization: fmt.Sprintf("Bearer %s", pulsarToken),
247247
}
248248

249-
getSinkResponse, err := streamingClientv3.GetSinksWithResponse(ctx, tenantName, namespace, sinkName, &getSinksParams)
249+
getSinkResponse, err := astraClients.astraStreamingClientv3.GetSinksWithResponse(ctx, tenantName, namespace, sinkName, &getSinksParams)
250250
if err != nil {
251251
diag.FromErr(err)
252252
}
@@ -263,9 +263,9 @@ func resourceStreamingSinkRead(ctx context.Context, resourceData *schema.Resourc
263263
}
264264

265265
func resourceStreamingSinkCreate(ctx context.Context, resourceData *schema.ResourceData, meta interface{}) diag.Diagnostics {
266-
streamingClient := meta.(astraClients).astraStreamingClient.(*astrastreaming.ClientWithResponses)
267-
client := meta.(astraClients).astraClient.(*astra.ClientWithResponses)
268-
streamingClientv3 := meta.(astraClients).astraStreamingClientv3
266+
astraClients := meta.(astraClients)
267+
streamingClient := astraClients.astraStreamingClient.(*astrastreaming.ClientWithResponses)
268+
astraClient := astraClients.astraClient.(*astra.ClientWithResponses)
269269

270270
rawRegion := resourceData.Get("region").(string)
271271
region := strings.ReplaceAll(rawRegion, "-", "")
@@ -281,7 +281,7 @@ func resourceStreamingSinkCreate(ctx context.Context, resourceData *schema.Resou
281281
topic := resourceData.Get("topic").(string)
282282
autoAck := resourceData.Get("auto_ack").(bool)
283283

284-
orgBody, _ := client.GetCurrentOrganization(ctx)
284+
orgBody, _ := astraClient.GetCurrentOrganization(ctx)
285285

286286
var org OrgId
287287
bodyBuffer, err := ioutil.ReadAll(orgBody.Body)
@@ -311,9 +311,9 @@ func resourceStreamingSinkCreate(ctx context.Context, resourceData *schema.Resou
311311
}
312312
}
313313

314-
pulsarCluster := getPulsarCluster(cloudProvider, region)
314+
pulsarCluster := getPulsarCluster(cloudProvider, region, astraClients.streamingTestMode)
315315

316-
token := meta.(astraClients).token
316+
token := astraClients.token
317317
pulsarToken, err := getPulsarToken(ctx, pulsarCluster, token, org, err, streamingClient, tenantName)
318318
if err != nil {
319319
diag.FromErr(err)
@@ -331,7 +331,7 @@ func resourceStreamingSinkCreate(ctx context.Context, resourceData *schema.Resou
331331
Authorization: pulsarToken,
332332
}
333333

334-
builtinSinksResponse, err := streamingClientv3.GetBuiltInSinks(ctx, &getBuiltinSinkParams)
334+
builtinSinksResponse, err := astraClients.astraStreamingClientv3.GetBuiltInSinks(ctx, &getBuiltinSinkParams)
335335
if err != nil {
336336
diag.FromErr(err)
337337
}
@@ -405,7 +405,7 @@ func resourceStreamingSinkCreate(ctx context.Context, resourceData *schema.Resou
405405
TopicsPattern: nil,
406406
}
407407

408-
sinkCreationResponse, err := streamingClientv3.CreateSinkJSON(ctx, tenantName, namespace, sinkName, &createSinkParams, createSinkBody)
408+
sinkCreationResponse, err := astraClients.astraStreamingClientv3.CreateSinkJSON(ctx, tenantName, namespace, sinkName, &createSinkParams, createSinkBody)
409409
if err != nil {
410410
diag.FromErr(err)
411411
}

‎internal/provider/resource_streaming_topic.go

+53-59
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,8 @@ func resourceStreamingTopicDelete(ctx context.Context, resourceData *schema.Reso
8484
if protectedFromDelete(resourceData) {
8585
return diag.Errorf("\"deletion_protection\" must be explicitly set to \"false\" in order to destroy astra_streaming_topic")
8686
}
87-
streamingClient := meta.(astraClients).astraStreamingClient.(*astrastreaming.ClientWithResponses)
88-
client := meta.(astraClients).astraClient.(*astra.ClientWithResponses)
89-
streamingClientv3 := meta.(astraClients).astraStreamingClientv3
87+
astraClients := meta.(astraClients)
88+
astraClient := astraClients.astraClient.(*astra.ClientWithResponses)
9089

9190
namespace := resourceData.Get("namespace").(string)
9291
topic := resourceData.Get("topic").(string)
@@ -95,47 +94,46 @@ func resourceStreamingTopicDelete(ctx context.Context, resourceData *schema.Reso
9594
cloudProvider := resourceData.Get("cloud_provider").(string)
9695
rawRegion := resourceData.Get("region").(string)
9796

98-
token := meta.(astraClients).token
97+
token := astraClients.token
9998

10099
region := strings.ReplaceAll(rawRegion, "-", "")
101-
pulsarCluster := getPulsarCluster(cloudProvider, region)
102-
orgBody, _ := client.GetCurrentOrganization(ctx)
103-
100+
pulsarCluster := getPulsarCluster(cloudProvider, region, astraClients.streamingTestMode)
101+
orgResp, err := astraClient.GetCurrentOrganization(ctx)
102+
if err != nil {
103+
return diag.Errorf("Failed to read orgnization ID: %v", err)
104+
}
104105
var org OrgId
105-
bodyBuffer, err := ioutil.ReadAll(orgBody.Body)
106-
107-
err = json.Unmarshal(bodyBuffer, &org)
106+
err = json.NewDecoder(orgResp.Body).Decode(&org)
108107
if err != nil {
109-
fmt.Println("Can't deserislize", orgBody)
108+
return diag.Errorf("Failed to read orgnization ID: %v", err)
110109
}
111110

112-
pulsarToken, err := getPulsarToken(ctx, pulsarCluster, token, org, err, streamingClient, tenant)
111+
pulsarToken, err := getLatestPulsarToken(ctx, astraClients.astraStreamingClientv3, token, org.ID, pulsarCluster, tenant)
112+
if err != nil {
113+
return diag.FromErr(err)
114+
}
113115

114116
deleteTopicParams := astrastreaming.DeleteTopicParams{
115117
XDataStaxPulsarCluster: pulsarCluster,
116118
Authorization: fmt.Sprintf("Bearer %s", pulsarToken),
117119
}
118120

119-
deleteTopicResponse, err := streamingClientv3.DeleteTopic(ctx, tenant, namespace, topic, &deleteTopicParams)
121+
deleteTopicResponse, err := astraClients.astraStreamingClientv3.DeleteTopic(ctx, tenant, namespace, topic, &deleteTopicParams)
120122
if err != nil {
121-
diag.FromErr(err)
122-
}
123-
124-
if !strings.HasPrefix(deleteTopicResponse.Status, "2") {
125-
bodyBuffer, err = ioutil.ReadAll(deleteTopicResponse.Body)
123+
return diag.FromErr(err)
124+
} else if deleteTopicResponse.StatusCode > 299 {
125+
bodyBuffer, _ := ioutil.ReadAll(deleteTopicResponse.Body)
126126
return diag.Errorf("Error deleting topic %s", bodyBuffer)
127127
}
128-
bodyBuffer, err = ioutil.ReadAll(deleteTopicResponse.Body)
129128

130129
resourceData.SetId("")
131130

132131
return nil
133132
}
134133

135134
func resourceStreamingTopicRead(ctx context.Context, resourceData *schema.ResourceData, meta interface{}) diag.Diagnostics {
136-
streamingClient := meta.(astraClients).astraStreamingClient.(*astrastreaming.ClientWithResponses)
137-
client := meta.(astraClients).astraClient.(*astra.ClientWithResponses)
138-
streamingClientv3 := meta.(astraClients).astraStreamingClientv3
135+
astraClients := meta.(astraClients)
136+
astraClient := astraClients.astraClient.(*astra.ClientWithResponses)
139137

140138
namespace := resourceData.Get("namespace").(string)
141139
topic := resourceData.Get("topic").(string)
@@ -144,41 +142,37 @@ func resourceStreamingTopicRead(ctx context.Context, resourceData *schema.Resour
144142
cloudProvider := resourceData.Get("cloud_provider").(string)
145143
rawRegion := resourceData.Get("region").(string)
146144

147-
token := meta.(astraClients).token
145+
token := astraClients.token
148146

149147
region := strings.ReplaceAll(rawRegion, "-", "")
150-
pulsarCluster := getPulsarCluster(cloudProvider, region)
151-
orgBody, _ := client.GetCurrentOrganization(ctx)
152-
148+
pulsarCluster := getPulsarCluster(cloudProvider, region, astraClients.streamingTestMode)
149+
orgBody, err := astraClient.GetCurrentOrganization(ctx)
150+
if err != nil {
151+
return diag.Errorf("Failed to get organization ID: %v", err)
152+
}
153153
var org OrgId
154-
bodyBuffer, err := ioutil.ReadAll(orgBody.Body)
154+
err = json.NewDecoder(orgBody.Body).Decode(&org)
155155
if err != nil {
156-
return diag.FromErr(err)
156+
return diag.Errorf("Failed to read organization ID: %v", err)
157157
}
158158

159-
err = json.Unmarshal(bodyBuffer, &org)
159+
pulsarToken, err := getLatestPulsarToken(ctx, astraClients.astraStreamingClientv3, token, org.ID, pulsarCluster, tenant)
160160
if err != nil {
161-
fmt.Println("Can't deserislize", orgBody)
161+
return diag.Errorf("Failed to get pulsar token: %v", err)
162162
}
163163

164-
pulsarToken, err := getPulsarToken(ctx, pulsarCluster, token, org, err, streamingClient, tenant)
165-
166164
getTopicsParams := astrastreaming.GetTopicsParams{
167165
XDataStaxPulsarCluster: &pulsarCluster,
168166
Authorization: fmt.Sprintf("Bearer %s", pulsarToken),
169167
}
170168

171-
createTopicResponse, err := streamingClientv3.GetTopics(ctx, tenant, namespace, &getTopicsParams)
169+
readTopicResponse, err := astraClients.astraStreamingClientv3.GetTopics(ctx, tenant, namespace, &getTopicsParams)
172170
if err != nil {
173-
return diag.FromErr(err)
174-
}
175-
176-
if !strings.HasPrefix(createTopicResponse.Status, "2") {
177-
bodyBuffer, err = ioutil.ReadAll(createTopicResponse.Body)
171+
return diag.Errorf("Failed to get topic list: %v", err)
172+
} else if readTopicResponse.StatusCode > 299 {
173+
bodyBuffer, _ := ioutil.ReadAll(readTopicResponse.Body)
178174
return diag.Errorf("Error reading topic %s", bodyBuffer)
179175
}
180-
bodyBuffer, err = ioutil.ReadAll(createTopicResponse.Body)
181-
182176
//TODO: validate that our topic is there
183177

184178
setStreamingTopicData(resourceData, tenant, topic)
@@ -187,9 +181,8 @@ func resourceStreamingTopicRead(ctx context.Context, resourceData *schema.Resour
187181
}
188182

189183
func resourceStreamingTopicCreate(ctx context.Context, resourceData *schema.ResourceData, meta interface{}) diag.Diagnostics {
190-
streamingClient := meta.(astraClients).astraStreamingClient.(*astrastreaming.ClientWithResponses)
191-
client := meta.(astraClients).astraClient.(*astra.ClientWithResponses)
192-
streamingClientv3 := meta.(astraClients).astraStreamingClientv3
184+
astraClients := meta.(astraClients)
185+
astraClient := astraClients.astraClient.(*astra.ClientWithResponses)
193186

194187
namespace := resourceData.Get("namespace").(string)
195188
topic := resourceData.Get("topic").(string)
@@ -198,38 +191,39 @@ func resourceStreamingTopicCreate(ctx context.Context, resourceData *schema.Reso
198191
cloudProvider := resourceData.Get("cloud_provider").(string)
199192
rawRegion := resourceData.Get("region").(string)
200193

201-
token := meta.(astraClients).token
194+
token := astraClients.token
202195

203196
region := strings.ReplaceAll(rawRegion, "-", "")
204-
pulsarCluster := getPulsarCluster(cloudProvider, region)
205-
orgBody, _ := client.GetCurrentOrganization(ctx)
206-
207-
var org OrgId
208-
bodyBuffer, err := ioutil.ReadAll(orgBody.Body)
197+
pulsarCluster := getPulsarCluster(cloudProvider, region, astraClients.streamingTestMode)
209198

210-
err = json.Unmarshal(bodyBuffer, &org)
199+
orgResp, err := astraClient.GetCurrentOrganization(ctx)
211200
if err != nil {
212-
fmt.Println("Can't deserislize", orgBody)
201+
return diag.Errorf("Failed to get current organization: %v", err)
202+
}
203+
org := OrgId{}
204+
err = json.NewDecoder(orgResp.Body).Decode(&org)
205+
if err != nil {
206+
return diag.Errorf("Failed to read organization: %v", err)
213207
}
214208

215-
pulsarToken, err := getPulsarToken(ctx, pulsarCluster, token, org, err, streamingClient, tenant)
209+
pulsarToken, err := getLatestPulsarToken(ctx, astraClients.astraStreamingClientv3, token, org.ID, pulsarCluster, tenant)
210+
if err != nil {
211+
return diag.FromErr(err)
212+
}
216213

217214
createTopicParams := astrastreaming.CreateTopicParams{
218215
XDataStaxCurrentOrg: &org.ID,
219216
XDataStaxPulsarCluster: pulsarCluster,
220217
Authorization: fmt.Sprintf("Bearer %s", pulsarToken),
221218
}
222219

223-
createTopicResponse, err := streamingClientv3.CreateTopic(ctx, tenant, namespace, topic, &createTopicParams)
220+
createTopicResponse, err := astraClients.astraStreamingClientv3.CreateTopic(ctx, tenant, namespace, topic, &createTopicParams, setContentTypeHeader("application/json"))
224221
if err != nil {
225-
diag.FromErr(err)
226-
}
227-
228-
if !strings.HasPrefix(createTopicResponse.Status, "2") {
229-
bodyBuffer, err = ioutil.ReadAll(createTopicResponse.Body)
222+
return diag.Errorf("Error creating topic: %v", err)
223+
} else if createTopicResponse.StatusCode > 299 {
224+
bodyBuffer, _ := ioutil.ReadAll(createTopicResponse.Body)
230225
return diag.Errorf("Error creating topic %s", bodyBuffer)
231226
}
232-
bodyBuffer, err = ioutil.ReadAll(createTopicResponse.Body)
233227

234228
setStreamingTopicData(resourceData, tenant, topic)
235229

0 commit comments

Comments
 (0)