Skip to content

support adding multiple connectors to same source_kind, using templates #987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 6 commits into
base: rc-v0.5.14
Choose a base branch
from
Draft

support adding multiple connectors to same source_kind, using templates #987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ffa83a7
support passing in custom api connectors for gcp, aws
eschultink Sep 10, 2025
618eb68
rules_file passed in as part of api_connector_spec
eschultink Sep 10, 2025
94130ae
rules_file param in more places
eschultink Sep 10, 2025
595c8b4
support enabling template
eschultink Sep 10, 2025
a042791
piping to enable connectors via templates
eschultink Sep 10, 2025
9e677db
add test, fix bug
eschultink Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions infra/examples-dev/aws/google-workspace.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,14 @@ module "worklytics_connectors_google_workspace" {
google = google.google_workspace
}

environment_id = var.environment_name
enabled_connectors = var.enabled_connectors
gcp_project_id = var.google_workspace_gcp_project_id
google_workspace_example_user = var.google_workspace_example_user
google_workspace_example_admin = var.google_workspace_example_admin
provision_gcp_sa_keys = var.google_workspace_provision_keys
todos_as_local_files = var.todos_as_local_files
environment_id = var.environment_name
enabled_connectors = var.enabled_connectors
enabled_connectors_from_templates = var.enabled_connectors_from_templates
gcp_project_id = var.google_workspace_gcp_project_id
google_workspace_example_user = var.google_workspace_example_user
google_workspace_example_admin = var.google_workspace_example_admin
provision_gcp_sa_keys = var.google_workspace_provision_keys
todos_as_local_files = var.todos_as_local_files
}

output "google_workspace_api_clients" {
Expand Down
2 changes: 2 additions & 0 deletions infra/examples-dev/aws/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ module "worklytics_connectors" {
# source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=rc-v0.5.9"

enabled_connectors = var.enabled_connectors
enabled_connectors_from_templates = var.enabled_connectors_from_templates
chat_gpt_enterprise_example_workspace_id = var.chat_gpt_enterprise_example_workspace_id
confluence_example_cloud_id = var.confluence_example_cloud_id
confluence_example_group_id = var.confluence_example_group_id
Expand Down Expand Up @@ -51,6 +52,7 @@ locals {
module.worklytics_connectors.enabled_api_connectors,
module.worklytics_connectors_google_workspace.enabled_api_connectors,
local.msft_api_connectors_with_auth,
var.custom_api_connectors,
{}
)

Expand Down
1 change: 1 addition & 0 deletions infra/examples-dev/aws/msft-365.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ module "worklytics_connectors_msft_365" {
# source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=rc-v0.5.9"

enabled_connectors = var.enabled_connectors
enabled_connectors_from_templates = var.enabled_connectors_from_templates
environment_id = var.environment_name
msft_tenant_id = var.msft_tenant_id
example_msft_user_guid = var.example_msft_user_guid
Expand Down
42 changes: 42 additions & 0 deletions infra/examples-dev/aws/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,14 @@ variable "enabled_connectors" {
description = "list of ids of connectors to enabled; see modules/worklytics-connector-specs"
}

variable "enabled_connectors_from_templates" {
type = map(object({
template_id = string
}))
Copy link
Member Author

@eschultink eschultink Sep 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. alter connector "specs" to be more like true templates, and have each evaluated in isolation with a "scope" that's the content of this object??
  • problem with that is that I believe Terraform erases/hides object properties not explicitly in variables definition of the module; so we can't have arbitrary content.
  • might also make module structure pretty complicated, with connectors being invoked repeatedly
  1. have a map(string) that is "merged" over top whatever the spec for the template has by default?
    • not sure this is enough
    • undermines value, bc in several cases you won't re-use that much of the template and will have to re-write a bunch of it.
  2. explicit set of properties that can be overridden; target_host, environment_variables, settings_to_provide, and rules_file probably enough ...
    • example_api_requests will remain problematic.

description = "map of connector id --> id of connector template to enable"
default = {}
}

variable "non_production_connectors" {
type = list(string)
description = "connector ids in this list will be in development mode (not for production use)"
Expand All @@ -237,6 +245,40 @@ variable "bulk_sanitized_expiration_days" {
default = 1805 # 5 years; intent is 'forever', but some upperbound in case bucket is forgotten
}

variable "custom_api_connectors" {
type = map(object({
source_kind = string
source_auth_strategy = string
target_host = string
oauth_scopes_needed = optional(list(string), [])
environment_variables = optional(map(string), {})
enable_async_processing = optional(bool, false)
example_api_calls = optional(list(string), [])
example_api_requests = optional(list(object({
method = optional(string, "GET")
path = string
content_type = optional(string, "application/json")
body = optional(string, null)
})), [])
example_api_calls_user_to_impersonate = optional(string)
secured_variables = optional(list(object({
name = string
value = optional(string)
writable = optional(bool, false)
lockable = optional(bool, false)
sensitive = optional(bool, true)
description = optional(string)
value_managed_by_tf = optional(bool, true)
})),
[])
settings_to_provide = optional(map(string), {})
rules_file = optional(string, null)
}))

description = "map of API connectors to provision"
default = {}
}

variable "custom_api_connector_rules" {
type = map(string)
description = "map of connector id --> YAML file with custom rules"
Expand Down
15 changes: 8 additions & 7 deletions infra/examples-dev/gcp/google-workspace.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,14 @@ module "worklytics_connectors_google_workspace" {
google = google.google_workspace
}

environment_id = var.environment_name
enabled_connectors = var.enabled_connectors
gcp_project_id = var.google_workspace_gcp_project_id
google_workspace_example_user = var.google_workspace_example_user
google_workspace_example_admin = var.google_workspace_example_admin
provision_gcp_sa_keys = var.google_workspace_provision_keys
todos_as_local_files = var.todos_as_local_files
environment_id = var.environment_name
enabled_connectors = var.enabled_connectors
enabled_connectors_from_templates = var.enabled_connectors_from_templates
gcp_project_id = var.google_workspace_gcp_project_id
google_workspace_example_user = var.google_workspace_example_user
google_workspace_example_admin = var.google_workspace_example_admin
provision_gcp_sa_keys = var.google_workspace_provision_keys
todos_as_local_files = var.todos_as_local_files
}

output "google_workspace_api_clients" {
Expand Down
1 change: 1 addition & 0 deletions infra/examples-dev/gcp/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ locals {
module.worklytics_connectors.enabled_api_connectors,
module.worklytics_connectors_google_workspace.enabled_api_connectors,
local.msft_api_connectors_with_auth,
var.custom_api_connectors,
{}
)

Expand Down
1 change: 1 addition & 0 deletions infra/examples-dev/gcp/msft-365.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ module "worklytics_connectors_msft_365" {
# source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=rc-v0.5.9"

enabled_connectors = var.enabled_connectors
enabled_connectors_from_templates = var.enabled_connectors_from_templates
environment_id = var.environment_name
msft_tenant_id = var.msft_tenant_id
example_msft_user_guid = var.example_msft_user_guid
Expand Down
43 changes: 43 additions & 0 deletions infra/examples-dev/gcp/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,15 @@ variable "enabled_connectors" {
description = "list of ids of connectors to enabled; see modules/worklytics-connector-specs"
}

variable "enabled_connectors_from_templates" {
type = map(object({
template_id = string
}))
description = "map of connector id --> id of connector template to enable"
default = {}
}


variable "non_production_connectors" {
type = list(string)
description = "connector ids in this list will be in development mode (not for production use"
Expand All @@ -246,6 +255,40 @@ variable "bulk_sanitized_expiration_days" {
default = 1805 # 5 years; intent is 'forever', but some upperbound in case bucket is forgotten
}

variable "custom_api_connectors" {
type = map(object({
source_kind = string
source_auth_strategy = string
target_host = string
oauth_scopes_needed = optional(list(string), [])
environment_variables = optional(map(string), {})
enable_async_processing = optional(bool, false)
example_api_calls = optional(list(string), [])
example_api_requests = optional(list(object({
method = optional(string, "GET")
path = string
content_type = optional(string, "application/json")
body = optional(string, null)
})), [])
example_api_calls_user_to_impersonate = optional(string)
secured_variables = optional(list(object({
name = string
value = optional(string)
writable = optional(bool, false)
lockable = optional(bool, false)
sensitive = optional(bool, true)
value_managed_by_tf = optional(bool, true)
description = optional(string)
})),
[])
settings_to_provide = optional(map(string), {})
rules_file = optional(string, null)
}))

description = "map of custom API connectors to provision"
default = {}
}

variable "custom_api_connector_rules" {
type = map(string)
description = "map of connector id --> YAML file with custom rules"
Expand Down
6 changes: 4 additions & 2 deletions infra/modules/aws-host/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ locals {

has_enabled_webhook_collectors = length(keys(var.webhook_collectors)) > 0
enable_webhook_testing = var.provision_testing_infra && local.has_enabled_webhook_collectors

api_connector_rules_files = merge(var.custom_api_connector_rules, { for k, v in var.api_connectors : k => v if v.rules_file != null })
}

module "psoxy" {
Expand Down Expand Up @@ -218,7 +220,7 @@ module "api_connector" {
{
PSEUDONYMIZE_APP_IDS = tostring(var.pseudonymize_app_ids)
EMAIL_CANONICALIZATION = var.email_canonicalization
CUSTOM_RULES_SHA = try(var.custom_api_connector_rules[each.key], null) != null ? filesha1(var.custom_api_connector_rules[each.key]) : null
CUSTOM_RULES_SHA = try(local.api_connector_rules_files[each.key], null) != null ? filesha1(local.api_connector_rules_files[each.key]) : null
IS_DEVELOPMENT_MODE = contains(var.non_production_connectors, each.key)
}
)
Expand All @@ -229,7 +231,7 @@ module "api_connector" {
module "custom_api_connector_rules" {
source = "../../modules/aws-ssm-rules"

for_each = var.custom_api_connector_rules
for_each = local.api_connector_rules_files

prefix = "${local.instance_ssm_prefix}${replace(upper(each.key), "-", "_")}_"
file_path = each.value
Expand Down
1 change: 1 addition & 0 deletions infra/modules/aws-host/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ variable "api_connectors" {
})),
[])
settings_to_provide = optional(map(string), {})
rules_file = optional(string, null)
}))

description = "map of API connectors to provision"
Expand Down
6 changes: 4 additions & 2 deletions infra/modules/gcp-host/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ locals {
config_parameter_prefix = var.config_parameter_prefix == "" ? local.default_config_parameter_prefix : var.config_parameter_prefix
environment_id_prefix = "${var.environment_name}${length(var.environment_name) > 0 ? "-" : ""}"
environment_id_display_name_qualifier = length(var.environment_name) > 0 ? " ${var.environment_name} " : ""

api_connector_rules_files = merge(var.custom_api_connector_rules, { for k, v in var.api_connectors : k => v if v.rules_file != null })
}

module "psoxy" {
Expand Down Expand Up @@ -196,7 +198,7 @@ module "api_connector" {
BUNDLE_FILENAME = module.psoxy.filename
IS_DEVELOPMENT_MODE = contains(var.non_production_connectors, each.key)
PSEUDONYMIZE_APP_IDS = tostring(var.pseudonymize_app_ids)
CUSTOM_RULES_SHA = try(var.custom_api_connector_rules[each.key], null) != null ? filesha1(var.custom_api_connector_rules[each.key]) : null
CUSTOM_RULES_SHA = try(local.api_connector_rules_files[each.key], null) != null ? filesha1(local.api_connector_rules_files[each.key]) : null
EMAIL_CANONICALIZATION = var.email_canonicalization
}
)
Expand All @@ -209,7 +211,7 @@ module "api_connector" {
}

module "custom_api_connector_rules" {
for_each = var.custom_api_connector_rules
for_each = local.api_connector_rules_files

source = "../../modules/gcp-sm-rules"

Expand Down
1 change: 1 addition & 0 deletions infra/modules/gcp-host/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ variable "api_connectors" {
})),
[])
settings_to_provide = optional(map(string), {})
rules_file = optional(string, null)
}))

description = "map of API connectors to provision"
Expand Down
Loading
Loading