Merge branch 'develop' into Traits-extraction

Author: rchinn1

.github/styles/Vocab/Docs/accept.txt

@@ -35,6 +35,7 @@ aws
 background(?:ed|ing)
 Bento
 Bing
+Blackbaud
 Blitzllama
 blocklist
 Bluedot

CODEOWNERS

@@ -21,11 +21,11 @@ CODEOWNERS  @segmentio/segment-doc-team
 
 
 # Libraries owners
-/src/connections/catalog/libraries @stayseesong @markzegarelli
+/src/connections/catalog/libraries @stayseesong
 
 
 # Destinations owners
-# /src/connections/destinations  @stayseesong @markzegarelli
+# /src/connections/destinations  @stayseesong=
 
 # Stratconn
 ## Adobe
@@ -41,7 +41,7 @@ CODEOWNERS  @segmentio/segment-doc-team
 
 
 # Engage
-/src/engage/ @markzegarelli @pwseg @rchinn-segment
+/src/engage/ @pwseg @rchinn-segment
 
 # Unify
 /src/unify @rchinn-segment
@@ -50,4 +50,4 @@ CODEOWNERS  @segmentio/segment-doc-team
 /src/protocols @forstisabella
 
 # Storage owners
-/src/connections/storage @forstisabella
+/src/connections/storage @forstisabella

Makefile

@@ -56,7 +56,7 @@ serve: package
 	@docker run -p 4000:80 segment-docs:latest
 
 catalog:
-	@node scripts/catalog_papi.js
+	@node scripts/catalog/index.js
 
 # make the list of beta connections
 .PHONY: beta
@@ -163,7 +163,7 @@ docker-build:
 
 .PHONY: private-destination
 private_destination:
-	@node scripts/private-destination.js
+	@node scripts/catalog/addPrivateDestination.js
 #.PHONY: docs
 #docs: node_modules
 #	$(BIN)/webpack --mode=production

netlify.toml

@@ -14,7 +14,7 @@
 
 [context.branch-deploy]
   command = "yarn build"
-#  ignore = "./scripts/ignore.sh"
+  # ignore = "./scripts/ignore.sh"
 
 [context.develop]
   command = "yarn develop"

scripts/catalog/addPrivateDestination.js

@@ -0,0 +1,35 @@
+const path = require('path');
+const fs = require('fs');
+const { prompt } = require('enquirer');
+const {
+    getDestinationData,
+    checkExistingStatus
+  } = require('./utilities_private_destinations.js');
+
+const addPrivateDestination = async () => {
+  let ids = await checkExistingStatus();
+  ids.sort();
+
+  const DEST_ID = await prompt({
+    type: 'input',
+    name: 'id',
+    message: 'Enter the destination ID'
+  });
+
+
+    if (ids.includes(DEST_ID.id)) {
+      console.log("This destination is already captured.");
+      return;
+    } else {
+      ids.push(DEST_ID.id);
+      fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/destinations_private.yml`), '');
+    }
+    ids.sort();
+
+    for (const element in ids) {
+      let currentId = ids[element];
+      await getDestinationData(currentId);
+    }
+};
+
+addPrivateDestination();

scripts/catalog/catalog_papi.js

@@ -0,0 +1,95 @@
+const path = require('path');
+const fs = require('fs');
+const yaml = require('js-yaml');
+const {
+  slugify,
+  getCatalog,
+  getConnectionModes,
+  isCatalogItemHidden,
+  sanitize,
+  doesCatalogItemExist
+} = require('./utilities.js');
+
+require('dotenv').config();
+
+const PAPI_URL = "https://api.segmentapis.com";
+
+const regionalSupport = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/regional-support.yml`)));
+
+// This file keeps a list of known test sources that show up in the system. 
+// Because we don't have a status value for sources, they end up showing in our catalog.
+// We use this below to prevent them from being written to yaml.
+const testSources = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/catalog/test_sources.yml`)));
+
+
+
+const updateWarehouses = async () => {
+  let warehouses = [];
+  let nextPageToken = "MA==";
+  let warehousesUpdated = [];
+
+  while (nextPageToken !== undefined) {
+    const res = await getCatalog(`${PAPI_URL}/catalog/warehouses/`, nextPageToken);
+    warehouses = warehouses.concat(res.data.warehousesCatalog);
+    nextPageToken = res.data.pagination.next;
+  }
+
+  warehouses.sort((a, b) => {
+    if (a.name.toLowerCase() < b.name.toLowerCase()) {
+      return -1;
+    }
+    if (a.name.toLowerCase() > b.name.toLowerCase()) {
+      return 1;
+    }
+    return 0;
+  });
+
+  const regionalWarehouseEndpoints = regionalSupport.warehouses.endpoint;
+  const regionalWarehouseRegions = regionalSupport.warehouses.region;
+
+  warehouses.forEach(warehouse => {
+    let slug = slugify(warehouse.slug);
+    let endpoints = ['us'];
+    let regions = ['us'];
+    let url = `connections/storage/catalog/${slug}`;
+
+    if (regionalWarehouseEndpoints.includes(slug)) {
+      endpoints.push('eu');
+    }
+
+    if (regionalWarehouseRegions.includes(slug)) {
+      regions.push('eu');
+    }
+
+    let updatedWarehouse = {
+      id: warehouse.id,
+      display_name: warehouse.name,
+      url,
+      slug,
+      endpoints,
+      regions
+    };
+
+    warehousesUpdated.push(updatedWarehouse);
+  });
+
+  const options = {
+    noArrayIndent: true
+  };
+  const todayDate = new Date().toISOString().slice(0, 10);
+
+  // Create regional support YAML file
+  let output = "# AUTOGENERATED LIST OF CONNECTIONS THAT SUPPORT REGIONAL\n";
+  output += "# Last updated " + todayDate + " \n";
+  output += yaml.dump({
+    warehouses: warehousesUpdated
+  }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/regional-supported.yml`), output);
+
+  console.log("warehouses done");
+};
+
+// Execute the update functions
+updateWarehouses();
+updateSources();
+updateDestinations();

scripts/catalog/index.js

@@ -0,0 +1,17 @@
+const {updateSources} = require('./updateSources.js');
+const {updateDestinations} = require('./updateDestinations.js');
+const {updateWarehouses} = require('./updateWarehouses.js');
+const {updatePrivateDestinations} = require('./updatePrivateDestinations.js');
+
+updateSources();
+updateWarehouses();
+
+// Wait for the main catalog to update before updating the private destinations
+async function destinations() {
+    await updateDestinations()
+    updatePrivateDestinations();
+}
+
+
+
+destinations();