feat: prevent duplicates [CM-2320] (#3282)

Author: ulemons

backend/src/database/migrations/U1753110221__create_segment_repos_table.sql

@@ -0,0 +1,11 @@
+-- 1. Create the table to map repositories to segments
+CREATE TABLE IF NOT EXISTS "segmentRepositories" (
+    "repository" TEXT NOT NULL UNIQUE,
+    "segmentId" UUID NOT NULL REFERENCES "segments"(id) ON DELETE CASCADE,
+    "insightsProjectId" UUID REFERENCES "insightsProjects"(id) ON DELETE CASCADE,
+    "createdAt" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "excluded" BOOLEAN NOT NULL DEFAULT FALSE,
+    "archived" BOOLEAN NOT NULL DEFAULT FALSE,
+
+    PRIMARY KEY ("repository", "segmentId")
+);

backend/src/database/migrations/V1753110221__create_segment_repos_table.sql

@@ -933,6 +933,7 @@ class SegmentRepository extends RepositoryBase<
         "githubRepos" r
        where r."segmentId" = :segmentId
        and r."tenantId" = :tenantId
+       and r."deletedAt" is null
        order by r.url
       `,
       {

backend/src/database/repositories/segmentRepository.ts

@@ -14,6 +14,7 @@ import {
   createInsightsProject,
   deleteCollection,
   deleteInsightsProject,
+  deleteMissingSegmentRepositories,
   disconnectProjectsAndCollections,
   findCollectionProjectConnections,
   queryCollectionById,
@@ -22,6 +23,7 @@ import {
   queryInsightsProjects,
   updateCollection,
   updateInsightsProject,
+  upsertSegmentRepositories,
 } from '@crowd/data-access-layer/src/collections'
 import { fetchIntegrationsForSegment } from '@crowd/data-access-layer/src/integrations'
 import { OrganizationField, findOrgById, queryOrgs } from '@crowd/data-access-layer/src/orgs'
@@ -363,7 +365,17 @@ export class CollectionService extends LoggerBase {
     }
   }
 
-  async updateInsightsProject(id: string, project: Partial<ICreateInsightsProject>) {
+  static normalizeRepositories(
+    repositories?: string[] | { platform: string; url: string }[],
+  ): string[] {
+    if (!repositories || repositories.length === 0) return []
+
+    return typeof repositories[0] === 'string'
+      ? (repositories as string[])
+      : (repositories as { platform: string; url: string }[]).map((r) => r.url)
+  }
+
+  async updateInsightsProject(insightsProjectId: string, project: Partial<ICreateInsightsProject>) {
     return SequelizeRepository.withTx(this.options, async (tx) => {
       const qx = SequelizeRepository.getQueryExecutor(this.options, tx)
 
@@ -373,15 +385,24 @@ export class CollectionService extends LoggerBase {
         project.isLF = segment?.isLF ?? false
       }
 
-      await updateInsightsProject(qx, id, project)
+      const { segmentId } = await updateInsightsProject(qx, insightsProjectId, project)
+
+      const repositories = CollectionService.normalizeRepositories(project.repositories)
+
+      await upsertSegmentRepositories(qx, { insightsProjectId, repositories, segmentId })
+
+      await deleteMissingSegmentRepositories(qx, {
+        repositories,
+        segmentId,
+      })
 
       if (project.collections) {
-        await disconnectProjectsAndCollections(qx, { insightsProjectId: id })
+        await disconnectProjectsAndCollections(qx, { insightsProjectId })
         await connectProjectsAndCollections(
           qx,
           project.collections.map((c) => ({
-            insightsProjectId: id,
             collectionId: c,
+            insightsProjectId,
             starred: project.starred ?? true,
           })),
         )
@@ -391,7 +412,7 @@ export class CollectionService extends LoggerBase {
         ...this.options,
         transaction: tx,
       })
-      return txSvc.findInsightsProjectById(id)
+      return txSvc.findInsightsProjectById(insightsProjectId)
     })
   }
 

backend/src/services/collectionService.ts

@@ -271,13 +271,21 @@ export async function updateInsightsProject(
   id: string,
   project: Partial<ICreateInsightsProject>,
 ) {
-  return updateTableById(
+  const result = await updateTableById(
     qx,
     'insightsProjects',
     id,
     Object.values(InsightsProjectField),
     prepareProject(project),
   )
+
+  const updated = result?.rows?.[0]
+
+  if (!updated) {
+    throw new Error(`Update failed or project with id ${id} not found`)
+  }
+
+  return updated as IInsightsProject
 }
 
 function prepareProject(project: Partial<ICreateInsightsProject>) {
@@ -296,3 +304,55 @@ export async function findBySlug(qx: QueryExecutor, slug: string) {
   })
   return collections
 }
+
+export async function upsertSegmentRepositories(
+  qx: QueryExecutor,
+  {
+    insightsProjectId,
+    repositories,
+    segmentId,
+  }: {
+    insightsProjectId?: string
+    repositories: string[]
+    segmentId: string
+  },
+) {
+  if (repositories.length === 0) {
+    return null
+  }
+
+  const data = repositories.map((repo) => ({
+    insightsProjectId,
+    repository: repo,
+    segmentId,
+  }))
+
+  return qx.result(
+    prepareBulkInsert(
+      'segmentRepositories',
+      ['insightsProjectId', 'repository', 'segmentId'],
+      data,
+      '("repository", "segmentId") DO NOTHING',
+    ),
+  )
+}
+
+export async function deleteMissingSegmentRepositories(
+  qx: QueryExecutor,
+  {
+    repositories,
+    segmentId,
+  }: {
+    repositories: string[]
+    segmentId: string
+  },
+) {
+  return qx.result(
+    `
+    DELETE FROM "segmentRepositories"
+    WHERE "segmentId" = '${segmentId}'
+      AND ${repositories.length > 0 ? `"repository" != ALL(ARRAY[${repositories.map((repo) => `'${repo}'`).join(', ')}])` : 'TRUE'};
+    `,
+    { segmentId, repositories },
+  )
+}