[WIP] Add silo discovery recommendations to tr-pull

README.md

@@ -413,6 +413,8 @@ The API key permissions for this command vary based on the value to the `resourc
 | messages              | Message definitions used across consent, privacy center, email templates and more.                                                   | View Internationalization Messages               | false      | [Privacy Center -> Messages](https://app.transcend.io/privacy-center/messages-internationalization), [Consent Management -> Display Settings -> Messages](https://app.transcend.io/consent-manager/display-settings/messages) |
 | assessments           | Assessment responses.                                                                                                                | View Assessments                                 | false      | [Assessments -> Assessments](https://app.transcend.io/assessments/groups)                                                                                                                                                     |
 | assessmentTemplates   | Assessment template configurations.                                                                                                  | View Assessments                                 | false      | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates)                                                                                                                                                |
+| siloDiscoveryRecommendations   | Pending data silos recommended by Silo Discovery                                                                                                  | View Data Map                                 | false      | [Silo Discovery -> Triage](https://app.transcend.io/data-map/data-inventory/silo-discovery/triage)                                                                                                                                                |
+
 
 _Note: The scopes for tr-push are comprehensive of the scopes for tr-pull_
 

examples/siloDiscoveryRecommendations.yml

@@ -0,0 +1,6 @@
+siloDiscoveryRecommendations:
+  - title: Slack
+    resourceId: 0oa677bq1n5OAyODE5d7
+    lastDiscoveredAt: '2025-01-18T00:05:00.378Z'
+    suggestedCatalog: Slack
+    plugin: Okta

src/codecs.ts

@@ -174,7 +174,7 @@ export const EnricherInput = t.intersection([
      */
     'input-identifier': t.string,
     /**
-     * A regular expression that can be used to match on for cancelation
+     * A regular expression that can be used to match on for cancellation
      */
     testRegex: t.string,
     /**
@@ -1744,6 +1744,54 @@ export const AssessmentInput = t.intersection([
 /** Type override */
 export type AssessmentInput = t.TypeOf<typeof AssessmentInput>;
 
+/**
+ * Input to define a silo discovery recommendation
+ *
+ * @see https://docs.transcend.io/docs/silo-discovery
+ */
+export const SiloDiscoveryRecommendationInput = t.intersection([
+  t.type({
+    /** The unique identifier for the resource */
+    resourceId: t.string,
+    /** Timestamp of the plugin run that found this silo recommendation */
+    lastDiscoveredAt: t.string,
+    /** The plugin that found this recommendation */
+    plugin: t.string, // Assuming Plugin is a string, replace with appropriate type if necessary
+    /** The suggested catalog for this recommendation */
+    suggestedCatalog: t.string,
+  }),
+  /**
+   * TODO: Allow for these to be pulled
+   */
+  t.partial({
+    /** The ISO country code for the AWS Region if applicable */
+    country: t.string,
+    /** The ISO country subdivision code for the AWS Region if applicable */
+    countrySubDivision: t.string,
+    /** The plaintext that we will pass into recommendation */
+    plaintextContext: t.string,
+    /** The plugin configurations for the recommendation */
+    pluginConfigurations: t.string, // Assuming DataSiloPluginConfigurations is a string
+    /** The AWS Region for data silo if applicable */
+    region: t.string,
+    /** The custom title of the data silo recommendation */
+    title: t.string,
+    /** The URL for more information about the recommendation */
+    url: t.string,
+    /** The list of tags associated with the recommendation */
+    tags: t.array(t.string),
+    /** The date the recommendation was created */
+    createdAt: t.string,
+    /** The date the recommendation was last updated */
+    updatedAt: t.string,
+  }),
+]);
+
+/** Type override */
+export type SiloDiscoveryRecommendationInput = t.TypeOf<
+  typeof SiloDiscoveryRecommendationInput
+>;
+
 export const TranscendInput = t.partial({
   /**
    * Action items
@@ -1861,6 +1909,10 @@ export const TranscendInput = t.partial({
    * The full list of assessment results
    */
   assessments: t.array(AssessmentInput),
+  /**
+   * The full list of silo discovery recommendations
+   */
+  siloDiscoveryRecommendations: t.array(SiloDiscoveryRecommendationInput),
 });
 
 /** Type override */

src/constants.ts

@@ -71,6 +71,9 @@ export const TR_PUSH_RESOURCE_SCOPE_MAP: {
   [TranscendPullResource.Policies]: [ScopeName.ManagePolicies],
   [TranscendPullResource.Assessments]: [ScopeName.ManageAssessments],
   [TranscendPullResource.AssessmentTemplates]: [ScopeName.ManageAssessments],
+  [TranscendPullResource.SiloDiscoveryRecommendations]: [
+    ScopeName.ManageDataMap,
+  ],
 };
 
 /**
@@ -114,6 +117,7 @@ export const TR_PULL_RESOURCE_SCOPE_MAP: {
   [TranscendPullResource.Policies]: [ScopeName.ViewPolicies],
   [TranscendPullResource.Assessments]: [ScopeName.ViewAssessments],
   [TranscendPullResource.AssessmentTemplates]: [ScopeName.ViewAssessments],
+  [TranscendPullResource.SiloDiscoveryRecommendations]: [ScopeName.ViewDataMap],
 };
 
 export const TR_YML_RESOURCE_TO_FIELD_NAME: Record<
@@ -150,4 +154,6 @@ export const TR_YML_RESOURCE_TO_FIELD_NAME: Record<
   [TranscendPullResource.Policies]: 'policies',
   [TranscendPullResource.Assessments]: 'assessments',
   [TranscendPullResource.AssessmentTemplates]: 'assessment-templates',
+  [TranscendPullResource.SiloDiscoveryRecommendations]:
+    'siloDiscoveryRecommendations',
 };

src/enums.ts

@@ -47,6 +47,7 @@ export enum TranscendPullResource {
   Messages = 'messages',
   Assessments = 'assessments',
   AssessmentTemplates = 'assessmentTemplates',
+  SiloDiscoveryRecommendations = 'siloDiscoveryRecommendations',
 }
 
 /**

src/graphql/fetchAllSiloDiscoveryRecommendations.ts

@@ -0,0 +1,119 @@
+import { GraphQLClient } from 'graphql-request';
+import { SILO_DISCOVERY_RECOMMENDATIONS } from './gqls';
+import { makeGraphQLRequest } from './makeGraphQLRequest';
+
+export interface SiloDiscoveryRecommendation {
+  /** Title of silo discovery recommendation */
+  title: string;
+  /** Resource ID of silo discovery recommendation */
+  resourceId: string;
+  /** Last discovered at */
+  lastDiscoveredAt: string;
+  /** Suggested catalog */
+  suggestedCatalog: {
+    /** Title */
+    title: string;
+  };
+  /** The plugin that found this recommendation */
+  plugin: {
+    /** The data silo the plugin belongs to */
+    dataSilo: {
+      /** The internal display title */
+      title: string;
+    };
+  };
+}
+
+const PAGE_SIZE = 30;
+
+/**
+ * Fetch all silo discovery recommendations in the organization
+ *
+ * @param client - GraphQL client
+ * @returns All silo discovery recommendations in the organization
+ */
+export async function fetchAllSiloDiscoveryRecommendations(
+  client: GraphQLClient,
+): Promise<SiloDiscoveryRecommendation[]> {
+  const siloDiscoveryRecommendations: SiloDiscoveryRecommendation[] = [];
+  let lastKey = null;
+
+  // Whether to continue looping
+  let shouldContinue = false;
+  do {
+    /**
+     * Input for the GraphQL request
+     */
+    const input: {
+      /** whether to list pending or ignored recommendations */
+      isPending: boolean;
+      /** key for previous page */
+      lastKey?: {
+        /** ID of plugin that found recommendation */
+        pluginId: string;
+        /** unique identifier for the resource */
+        resourceId: string;
+        /** ID of organization resource belongs to */
+        organizationId: string;
+        /** Status of recommendation, concatenated with latest run time */
+        statusLatestRunTime: string;
+      } | null;
+    } = lastKey
+      ? {
+          isPending: true,
+          lastKey,
+        }
+      : {
+          isPending: true,
+        };
+
+    const {
+      siloDiscoveryRecommendations: { nodes, lastKey: newLastKey },
+      // eslint-disable-next-line no-await-in-loop
+    } = await makeGraphQLRequest<{
+      /** Silo Discovery Recommendations */
+      siloDiscoveryRecommendations: {
+        /** List */
+        nodes: SiloDiscoveryRecommendation[];
+        /**
+         * Last key for pagination
+         */
+        lastKey: {
+          /** ID of plugin that found recommendation */
+          pluginId: string;
+          /** unique identifier for the resource */
+          resourceId: string;
+          /** ID of organization resource belongs to */
+          organizationId: string;
+          /** Status of recommendation, concatenated with latest run time */
+          statusLatestRunTime: string;
+        } | null;
+      };
+    }>(client, SILO_DISCOVERY_RECOMMENDATIONS, {
+      first: PAGE_SIZE,
+      input,
+      filterBy: {},
+    });
+
+    /**
+     * TODO: https://transcend.height.app/T-41786
+     * This is a temporary fix to ensure that recommendations without titles are given the title of their suggested catalog.
+     */
+    const titledNodes = nodes.map((node) => {
+      if (
+        node.title === null &&
+        node.suggestedCatalog &&
+        node.suggestedCatalog.title
+      ) {
+        return { ...node, title: node.suggestedCatalog.title };
+      }
+      return node;
+    });
+
+    siloDiscoveryRecommendations.push(...titledNodes);
+    lastKey = newLastKey;
+    shouldContinue = nodes.length === PAGE_SIZE && lastKey !== null;
+  } while (shouldContinue);
+
+  return siloDiscoveryRecommendations;
+}

src/graphql/gqls/index.ts

@@ -46,3 +46,4 @@ export * from './vendor';
 export * from './dataCategory';
 export * from './processingPurpose';
 export * from './sombraVersion';
+export * from './siloDiscoveryRecommendation';

src/graphql/gqls/siloDiscoveryRecommendation.ts

@@ -0,0 +1,30 @@
+import { gql } from 'graphql-request';
+
+export const SILO_DISCOVERY_RECOMMENDATIONS = gql`
+  query TranscendCliSiloDiscoveryRecommendations(
+    $first: Int
+    $input: SiloDiscoveryRecommendationsInput!
+  ) {
+    siloDiscoveryRecommendations(first: $first, input: $input) {
+      nodes {
+        title
+        resourceId
+        lastDiscoveredAt
+        suggestedCatalog {
+          title
+        }
+        plugin {
+          dataSilo {
+            title
+          }
+        }
+      }
+      lastKey {
+        pluginId
+        resourceId
+        organizationId
+        statusLatestRunTime
+      }
+    }
+  }
+`;

src/graphql/index.ts

@@ -83,3 +83,4 @@ export * from './syncAgentFiles';
 export * from './syncVendors';
 export * from './syncDataCategories';
 export * from './syncProcessingPurposes';
+export * from './fetchAllSiloDiscoveryRecommendations';

src/graphql/pullTranscendConfiguration.ts

@@ -32,6 +32,7 @@ import {
   AssessmentSectionInput,
   AssessmentSectionQuestionInput,
   RiskLogicInput,
+  SiloDiscoveryRecommendationInput,
 } from '../codecs';
 import {
   RequestAction,
@@ -89,6 +90,7 @@ import {
   parseAssessmentDisplayLogic,
 } from './parseAssessmentDisplayLogic';
 import { parseAssessmentRiskLogic } from './parseAssessmentRiskLogic';
+import { fetchAllSiloDiscoveryRecommendations } from './fetchAllSiloDiscoveryRecommendations';
 
 export const DEFAULT_TRANSCEND_PULL_RESOURCES = [
   TranscendPullResource.DataSilos,
@@ -180,6 +182,7 @@ export async function pullTranscendConfiguration(
     partitions,
     assessments,
     assessmentTemplates,
+    siloDiscoveryRecommendations,
   ] = await Promise.all([
     // Grab all data subjects in the organization
     resources.includes(TranscendPullResource.DataSilos) ||
@@ -328,6 +331,10 @@ export async function pullTranscendConfiguration(
     resources.includes(TranscendPullResource.AssessmentTemplates)
       ? fetchAllAssessmentTemplates(client)
       : [],
+    // Fetch siloDiscoveryRecommendations
+    resources.includes(TranscendPullResource.SiloDiscoveryRecommendations)
+      ? fetchAllSiloDiscoveryRecommendations(client)
+      : [],
   ]);
 
   const consentManagerTheme =
@@ -762,6 +769,30 @@ export async function pullTranscendConfiguration(
     );
   }
 
+  // Save siloDiscoveryRecommendations
+  if (
+    siloDiscoveryRecommendations.length > 0 &&
+    resources.includes(TranscendPullResource.SiloDiscoveryRecommendations)
+  ) {
+    result.siloDiscoveryRecommendations = siloDiscoveryRecommendations.map(
+      ({
+        title,
+        resourceId,
+        lastDiscoveredAt,
+        suggestedCatalog: { title: suggestedCatalogTitle },
+        plugin: {
+          dataSilo: { title: dataSiloTitle },
+        },
+      }): SiloDiscoveryRecommendationInput => ({
+        title,
+        resourceId,
+        lastDiscoveredAt,
+        suggestedCatalog: suggestedCatalogTitle,
+        plugin: dataSiloTitle,
+      }),
+    );
+  }
+
   // Save prompts
   if (prompts.length > 0 && resources.includes(TranscendPullResource.Prompts)) {
     result.prompts = prompts.map(