Skip to content

Commit 84fc3af

Browse files
waleedlatif1waleedlatif1
authored
feat(fivetran): add connector setup, schema config, group, and destination operations (#8988)
1 parent 37af475 commit 84fc3af

40 files changed

Lines changed: 1645 additions & 422 deletions

‎apps/docs/content/docs/integrations/fivetran.mdx‎

Lines changed: 471 additions & 7 deletions
Large diffs are not rendered by default.

‎apps/sim/blocks/blocks/fivetran.ts‎

Lines changed: 255 additions & 11 deletions
Large diffs are not rendered by default.

‎apps/sim/lib/webhooks/providers/fivetran.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,13 @@ export const fivetranHandler: WebhookProviderHandler = {
182182
}
183183
}
184184

185+
// Transformation events describe a dbt job and carry no connection, so the filter skips them.
185186
const connectionId = readString(providerConfig, 'triggerConnectionId')
186-
if (connectionId && payload.connector_id !== connectionId) {
187+
if (
188+
connectionId &&
189+
!event.startsWith('transformation_') &&
190+
payload.connector_id !== connectionId
191+
) {
187192
logger.debug(`[${requestId}] Fivetran event is for another connection. Skipping execution.`, {
188193
webhookId: webhook.id,
189194
})

‎apps/sim/tools/fivetran/cancel_transformation.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import type { FivetranMessageResponse, FivetranTransformationParams } from '@/tools/fivetran/types'
1+
import {
2+
type FivetranMessageResponse,
3+
type FivetranTransformationParams,
4+
MESSAGE_OUTPUTS,
5+
} from '@/tools/fivetran/types'
26
import {
37
FIVETRAN_AUTH_PARAMS,
48
FIVETRAN_ERROR_EXTRACTOR,
@@ -7,7 +11,6 @@ import {
711
fivetranHeaders,
812
fivetranId,
913
fivetranUrl,
10-
MESSAGE_OUTPUTS,
1114
transformMessageResponse,
1215
} from '@/tools/fivetran/utils'
1316
import type { ToolConfig } from '@/tools/types'

‎apps/sim/tools/fivetran/create_connection.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
import type {
2-
FivetranConnectionResponse,
3-
FivetranCreateConnectionParams,
4-
} from '@/tools/fivetran/types'
51
import {
62
CONNECTION_PROPERTIES,
3+
type FivetranConnectionResponse,
4+
type FivetranCreateConnectionParams,
5+
} from '@/tools/fivetran/types'
6+
import {
77
compactBody,
88
envelopeData,
99
FIVETRAN_AUTH_PARAMS,

‎apps/sim/tools/fivetran/create_destination.ts‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import {
2+
DESTINATION_PROPERTIES,
3+
type FivetranCreateDestinationParams,
4+
type FivetranDestinationResponse,
5+
} from '@/tools/fivetran/types'
6+
import {
7+
compactBody,
8+
envelopeData,
9+
FIVETRAN_AUTH_PARAMS,
10+
FIVETRAN_ERROR_EXTRACTOR,
11+
FIVETRAN_OAUTH,
12+
FIVETRAN_TRUST_PARAMS,
13+
fivetranHeaders,
14+
fivetranUrl,
15+
mapDestination,
16+
optionalBoolean,
17+
parseJsonObject,
18+
trimmed,
19+
} from '@/tools/fivetran/utils'
20+
import type { ToolConfig } from '@/tools/types'
21+
22+
export const fivetranCreateDestinationTool: ToolConfig<
23+
FivetranCreateDestinationParams,
24+
FivetranDestinationResponse
25+
> = {
26+
id: 'fivetran_create_destination',
27+
name: 'Fivetran Create Destination',
28+
description:
29+
'Create the destination for a Fivetran group, such as a Databricks workspace. The config object holds the destination-specific connection settings.',
30+
version: '1.0.0',
31+
errorExtractor: FIVETRAN_ERROR_EXTRACTOR,
32+
oauth: FIVETRAN_OAUTH,
33+
34+
params: {
35+
...FIVETRAN_AUTH_PARAMS,
36+
groupId: {
37+
type: 'string',
38+
required: true,
39+
visibility: 'user-or-llm',
40+
description: 'ID of the group the destination belongs to (each group has one destination)',
41+
},
42+
service: {
43+
type: 'string',
44+
required: true,
45+
visibility: 'user-or-llm',
46+
description: 'Destination type, e.g. databricks, snowflake, or big_query',
47+
},
48+
timeZoneOffset: {
49+
type: 'string',
50+
required: true,
51+
visibility: 'user-or-llm',
52+
description: 'UTC offset used for sync scheduling, from -11 to +12, e.g. -5 or 0',
53+
},
54+
region: {
55+
type: 'string',
56+
required: false,
57+
visibility: 'user-or-llm',
58+
description: 'Data processing region, e.g. AWS_US_EAST_1 or GCP_US_EAST4',
59+
},
60+
config: {
61+
type: 'json',
62+
required: false,
63+
visibility: 'user-or-llm',
64+
description:
65+
'Destination-specific settings, e.g. for Databricks {"server_host_name": "...", "http_path": "...", "catalog": "...", "auth_type": "PERSONAL_ACCESS_TOKEN", "personal_access_token": "..."}',
66+
},
67+
daylightSavingTimeEnabled: {
68+
type: 'boolean',
69+
required: false,
70+
visibility: 'user-or-llm',
71+
description: 'Shift the UTC offset with daylight saving time (US only)',
72+
},
73+
runSetupTests: {
74+
type: 'boolean',
75+
required: false,
76+
visibility: 'user-or-llm',
77+
description: 'Run the setup tests after creating the destination',
78+
},
79+
...FIVETRAN_TRUST_PARAMS,
80+
},
81+
82+
request: {
83+
url: () => fivetranUrl('/destinations'),
84+
method: 'POST',
85+
headers: (params) => fivetranHeaders(params),
86+
body: (params) =>
87+
compactBody({
88+
group_id: params.groupId?.trim(),
89+
service: params.service?.trim(),
90+
time_zone_offset: trimmed(params.timeZoneOffset),
91+
region: trimmed(params.region),
92+
config: parseJsonObject(params.config, 'config'),
93+
daylight_saving_time_enabled: optionalBoolean(params.daylightSavingTimeEnabled),
94+
run_setup_tests: optionalBoolean(params.runSetupTests),
95+
trust_certificates: optionalBoolean(params.trustCertificates),
96+
trust_fingerprints: optionalBoolean(params.trustFingerprints),
97+
}),
98+
},
99+
100+
transformResponse: async (response: Response) => {
101+
return {
102+
success: true,
103+
output: { destination: mapDestination(envelopeData(await response.json())) },
104+
}
105+
},
106+
107+
outputs: {
108+
destination: {
109+
type: 'object',
110+
description: 'The created destination, with setup test results when they ran',
111+
properties: DESTINATION_PROPERTIES,
112+
},
113+
},
114+
}

‎apps/sim/tools/fivetran/create_group.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import {
2+
type FivetranCreateGroupParams,
3+
type FivetranGetGroupResponse,
4+
GROUP_PROPERTIES,
5+
} from '@/tools/fivetran/types'
6+
import {
7+
envelopeData,
8+
FIVETRAN_AUTH_PARAMS,
9+
FIVETRAN_ERROR_EXTRACTOR,
10+
FIVETRAN_OAUTH,
11+
fivetranHeaders,
12+
fivetranUrl,
13+
mapGroup,
14+
} from '@/tools/fivetran/utils'
15+
import type { ToolConfig } from '@/tools/types'
16+
17+
export const fivetranCreateGroupTool: ToolConfig<
18+
FivetranCreateGroupParams,
19+
FivetranGetGroupResponse
20+
> = {
21+
id: 'fivetran_create_group',
22+
name: 'Fivetran Create Group',
23+
description:
24+
'Create a Fivetran group. Each group holds one destination, so create the group first, then its destination.',
25+
version: '1.0.0',
26+
errorExtractor: FIVETRAN_ERROR_EXTRACTOR,
27+
oauth: FIVETRAN_OAUTH,
28+
29+
params: {
30+
...FIVETRAN_AUTH_PARAMS,
31+
name: {
32+
type: 'string',
33+
required: true,
34+
visibility: 'user-or-llm',
35+
description: 'Name of the group',
36+
},
37+
},
38+
39+
request: {
40+
url: () => fivetranUrl('/groups'),
41+
method: 'POST',
42+
headers: (params) => fivetranHeaders(params),
43+
body: (params) => ({ name: params.name?.trim() }),
44+
},
45+
46+
transformResponse: async (response: Response) => {
47+
return { success: true, output: { group: mapGroup(envelopeData(await response.json())) } }
48+
},
49+
50+
outputs: {
51+
group: { type: 'object', description: 'The created group', properties: GROUP_PROPERTIES },
52+
},
53+
}

‎apps/sim/tools/fivetran/delete_connection.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import type { FivetranConnectionParams, FivetranMessageResponse } from '@/tools/fivetran/types'
1+
import {
2+
type FivetranConnectionParams,
3+
type FivetranMessageResponse,
4+
MESSAGE_OUTPUTS,
5+
} from '@/tools/fivetran/types'
26
import {
37
FIVETRAN_AUTH_PARAMS,
48
FIVETRAN_CONNECTION_ID_PARAM,
@@ -7,7 +11,6 @@ import {
711
fivetranHeaders,
812
fivetranId,
913
fivetranUrl,
10-
MESSAGE_OUTPUTS,
1114
transformMessageResponse,
1215
} from '@/tools/fivetran/utils'
1316
import type { ToolConfig } from '@/tools/types'

‎apps/sim/tools/fivetran/get_connection.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1-
import type { FivetranConnectionParams, FivetranConnectionResponse } from '@/tools/fivetran/types'
21
import {
32
CONNECTION_PROPERTIES,
3+
type FivetranConnectionParams,
4+
type FivetranConnectionResponse,
5+
} from '@/tools/fivetran/types'
6+
import {
47
envelopeData,
58
FIVETRAN_AUTH_PARAMS,
69
FIVETRAN_CONNECTION_ID_PARAM,

‎apps/sim/tools/fivetran/get_connection_schema_config.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import type { FivetranConnectionParams, FivetranSchemaConfigResponse } from '@/tools/fivetran/types'
1+
import {
2+
type FivetranConnectionParams,
3+
type FivetranSchemaConfigResponse,
4+
SCHEMA_CONFIG_OUTPUTS,
5+
} from '@/tools/fivetran/types'
26
import {
37
envelopeData,
48
FIVETRAN_AUTH_PARAMS,
@@ -9,7 +13,6 @@ import {
913
fivetranId,
1014
fivetranUrl,
1115
mapSchemaConfig,
12-
SCHEMA_CONFIG_OUTPUTS,
1316
} from '@/tools/fivetran/utils'
1417
import type { ToolConfig } from '@/tools/types'
1518

0 commit comments

Comments
 (0)