Merge pull request #246 from segmentio/use-ids-for-int-mapping

Use ids for int mapping

Author: markzegarelli

Makefile

@@ -150,6 +150,9 @@ vendor/bundle:
 	@bundle config set --local path 'vendor/bundle'
 	@bundle install
 
+.PHONY: update
+update: 
+	@node scripts/update.js
 
 .PHONY: lint
 lint: node_modules

scripts/update.js

@@ -0,0 +1,158 @@
+// These lines are the required packages we need. These let us do things like make network requests to the Public API
+// and interact with the frontmatter
+const axios = require('axios');
+const path = require('path');
+const fs = require('fs');
+const fm = require('front-matter');
+const yaml = require('js-yaml');
+const {
+  type
+} = require('os');
+
+require('dotenv').config();
+
+// Here, global variables are set
+const PAPI_URL = "https://api.segmentapis.com"
+const slugOverrides = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/catalog/slugs.yml`)))
+
+// This function connects with the Public API. It looks for the endpoint URL and a page token value.
+// The function is called in the updateSources and update Destination functions.
+// Functions let us reuse code easily. Instead of needing to write this out multiple times, I can define it once
+// and pass in the necessary details when I call it.
+const getCatalog = async (url, page_token = "MA==") => {
+  try {
+    const res = await axios.get(url, {
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${process.env.PAPI_TOKEN}`
+      },
+      data: {
+        "pagination": {
+          "count": 200,
+          "cursor": page_token
+        }
+      }
+    });
+
+    return res.data
+  } catch (error) {
+    console.log(error)
+  }
+}
+
+// This function, again called by the two update functions, is what generates the slug values for each integration.
+// It takes the integration's Display Name and converts it to a slug.
+const slugify = (displayName) => {
+  let slug = displayName
+    .toLowerCase()
+    .replace(/\s+/g, '-')
+    .replace('-&-', '-')
+    .replace('/', '-')
+    .replace(/[\(\)]/g, '')
+    .replace('.', '-')
+
+// This is how we handle manual slug overrides right now.
+// If a slug appears in the slugOverrides file, we want to use the 'override' value instead.
+  for (key in slugOverrides) {
+    let original = slugOverrides[key].original
+    let override = slugOverrides[key].override
+
+    if (slug == original) {
+      console.log(original + " -> " + override)
+      slug = override
+    }
+  }
+
+  return slug
+}
+
+// This function does the actual work of adding the id value to the source and destination
+// Notice that the write to file step is commented out. This is to verify that the updated frontmatter
+// is correct before we write a whole bunch of files.
+// Uncomment that line and remove the line above it to run it for real.
+const addIdToExisting = (integration) => {
+  let itemURL = integration.url
+  try {
+    const catalogPath = path.resolve('src', itemURL, 'index.md')
+    if (fs.existsSync(catalogPath)) {
+      const f = fm(fs.readFileSync(catalogPath, 'utf8'));
+      const attr = `---\n${f.frontmatter}\nid: ${integration.id}\n---\n`
+      const body = f.body
+      const content = attr + body
+      console.log(attr)
+      fs.writeFileSync(catalogPath, content)
+    }
+  } catch (e) {
+    console.log(error)
+    return false
+  }
+}
+
+
+// This is just a stripped down version of the updateSources() script from the catalog script.
+// We're retrieving less information overall, because all we care about here is the id.
+const updateSources = async () => {
+
+  let sources = []
+  let nextPageToken = "MA=="
+
+  while (nextPageToken !== null) {
+    const res = await getCatalog(`${PAPI_URL}/catalog/sources/`, nextPageToken)
+    sources = sources.concat(res.data.sourcesCatalog)
+    nextPageToken = res.data.pagination.next
+  }
+
+  const libraryCategories = [
+    'server',
+    'mobile',
+    'ott',
+    'roku',
+    'website'
+  ]
+  sources.forEach(source => {
+    let slug = slugify(source.name)
+    let mainCategory = source.categories[0] ? source.categories[0].toLowerCase() : ''
+
+    if (libraryCategories.includes(mainCategory)) {
+      url = `connections/sources/catalog/libraries/${mainCategory}/${slug}`
+    } else {
+      url = `connections/sources/catalog/cloud-apps/${slug}`
+      mainCategory = 'cloud-app'
+    }
+// So, we retrieve and store only the id and the URL, which is defined in the if statement on line 116.
+    let updatedSource = {
+      id: source.id,
+      url,
+    }
+    addIdToExisting(updatedSource)
+  })
+}
+
+// Similar to the sources script, only for destinations.
+const updateDestinations = async () => {
+  let destinations = []
+  let nextPageToken = "MA=="
+
+  while (nextPageToken !== null) {
+    const res = await getCatalog(`${PAPI_URL}/catalog/destinations/`, nextPageToken)
+    destinations = destinations.concat(res.data.destinationsCatalog)
+    nextPageToken = res.data.pagination.next
+  }
+
+
+  destinations.forEach(destination => {
+    let slug = slugify(destination.name)
+
+    let url = `connections/destinations/catalog/${slug}`
+
+    let updatedDestination = {
+      id: destination.id,
+      url
+    }
+    addIdToExisting(updatedDestination)
+
+  })
+
+}
+updateDestinations()
+updateSources()

src/_includes/content/destination-dossier.html

@@ -1,11 +1,11 @@
 <!-- in the file we're pulling from the API, "name" corresponds with the path to the yml blob for a specific destination.-->
-{% assign thisDestination = page.url | split: "/" | last %}
+{% assign thisDestination = page.id %}
 
 {% assign overrideInfo = site.data.catalog.overrides.items % | where: "slug", thisDestination | first %}
 
 
 
-{% assign destinationInfo = site.data.catalog.destinations.items | where: "slug", thisDestination | first %}
+{% assign destinationInfo = site.data.catalog.destinations.items | where: "destination_id", thisDestination | first %}
 {% comment %}There are probably prettier ways to generate a list of links to these methods, but this was good enough for me.{% endcomment %}
 {% assign destMethods = "" | split: ", " %}
 {% assign methodName = "" | split: " " %}
@@ -78,4 +78,4 @@ <h6>Connection Modes <a href="/docs/connections/destinations/#connection-modes">
 </table>
 </div>
 {% endif %}
-</div>
+</div>

src/connections/destinations/catalog/2mee/index.md

@@ -1,6 +1,7 @@
 ---
 title: 2mee Destination
 rewrite: true
+id: 60b5d0a01f3726b85dc05aab
 ---
 [2mee](https://2mee.com ) is a Human Hologram platform that automatically cuts the person out from the background, removing the visual clutter, and places them in the familiar context of your phone or website so that they dominate the screen.
 

src/connections/destinations/catalog/ab-smartly/index.md

@@ -1,8 +1,7 @@
 ---
 title: AB Smartly Destination
+id: 605dd9d7e5ff0b3873e250a4
 ---
-
-
 [A/B Smartly](https://absmartly.com/?utm_source=segmentio&utm_medium=docs&utm_campaign=partners){:target="_blank"} provides an on-premise, full-stack experimentation platform for engineering and product teams that do continuous experimentation embedded into their development process. A/B Smartly's real-time analytics helps engineering and product teams ensure that new features will improve the customer experience without breaking or degrading performance and/or business metrics.
 
 This destination is maintained by A/B Smartly. For any issues with the destination, [contact A/B Smartly's Support](mailto:[email protected]).

src/connections/destinations/catalog/actions-amplitude/index.md

@@ -2,6 +2,7 @@
 title: Amplitude (Actions) Destination
 hide-boilerplate: true
 hide-dossier: false
+id: 5f7dd6d21ad74f3842b1fc47
 ---
 {% include content/plan-grid.md name="actions" %}
 

src/connections/destinations/catalog/actions-facebook-conversions-api/index.md

@@ -3,8 +3,8 @@ title: Facebook Conversions API (Actions)
 strat: facebook
 hide-boilerplate: true
 hide-dossier: false
+id: 61806e472cd47ea1104885fc
 ---
-
 Facebook Conversions API (Actions) enables advertisers to send events from their servers directly to Facebook. Server-side events link to Facebook Pixel events, and process like browser pixel events. This means that server-side events are used in measurement, reporting, and optimization, just like browser pixel events.
 
 ## Benefits of Facebook Conversions API (Actions) vs Facebook Conversions API Classic

src/connections/destinations/catalog/actions-friendbuy-cloud/index.md

@@ -3,8 +3,8 @@ title: Friendbuy Cloud Mode (Actions) Destination
 hide-boilerplate: true
 hide-dossier: false
 hidden: false
+id: 61dde0dc77eb0db0392649d3
 ---
-
 {% include content/plan-grid.md name="actions" %}
 
 [Friendbuy](https://www.friendbuy.com/){:target='_blank'} powers referral programs for e-commerce merchants of all sizes, providing an easy solution to launch Refer-a-Friend programs and accelerate growth through word of mouth.

src/connections/destinations/catalog/actions-friendbuy/index.md

@@ -3,8 +3,8 @@ title: Friendbuy Web Mode (Actions) Destination
 hide-boilerplate: true
 hide-dossier: false
 hidden: false
+id: 6170a348128093cd0245e0ea
 ---
-
 <!-- The published version of this document is available at https://segment.com/docs/connections/destinations/catalog/actions-friendbuy/ -->
 
 {% include content/plan-grid.md name="actions" %}

src/connections/destinations/catalog/actions-fullstory/index.md

@@ -2,6 +2,7 @@
 title: FullStory (Actions)
 hide-boilerplate: true
 hide-dossier: false
+id: 6141153ee7500f15d3838703
 ---
 {% include content/plan-grid.md name="actions" %}