Skip to content

Commit 59e77ad

Browse files
rchinn1rchinn1
authored
Merge pull request segmentio#2331 from segmentio/develop
Release 22.2.1
2 parents 764792d + a2362a9 commit 59e77ad

File tree

48 files changed

+2090
-984
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+2090
-984
lines changed

‎.github/workflows/check-links-external.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: CheckLinks
2+
3+
# **What it does**: Runs a weekly check for broken external links.
4+
# **Why we have it**: We want to make sure that pages we link to work.
5+
# **Who does it impact**: Everyone
6+
7+
8+
9+
on:
10+
schedule:
11+
- cron: "5 4 * * SUN"
12+
13+
jobs:
14+
checklinks-external:
15+
if: github.repository == 'segmentio/segment-docs'
16+
name: Linux
17+
runs-on: ubuntu-latest
18+
env:
19+
GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_REPO_SCOPE }}
20+
REPORT_AUTHOR: docsbot
21+
REPORT_LABEL: report, automated issue, broken links
22+
REPORT_REPOSITORY: segmentio/segment-docs
23+
strategy:
24+
fail-fast: false
25+
steps:
26+
- uses: actions/checkout@v2
27+
- name: Ruby
28+
uses: ruby/setup-ruby@v1
29+
with:
30+
ruby-version: 2.6 # Not needed with a .ruby-version file
31+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
32+
- name: Yarn install
33+
run: yarn install
34+
- name: Build jekyll website with
35+
run: bundle exec jekyll build
36+
- name: Run Script
37+
run: |
38+
sudo make linkcheck-external > broken_links.md
39+
- if: ${{ failure() }}
40+
name: Create issue from file
41+
id: broken-link-report
42+
uses: peter-evans/create-issue-from-file@b4f9ee0a9d4abbfc6986601d9b1a4f8f8e74c77e
43+
with:
44+
token: ${{ env.GITHUB_TOKEN }}
45+
46+
title: Broken Links Report - External
47+
content-filepath: ./broken_links.md
48+
repository: ${{ env.REPORT_REPOSITORY }}
49+
labels: ${{ env.REPORT_LABEL }}

‎.github/workflows/check-links.yml renamed to ‎.github/workflows/check-links-internal.yml

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,15 @@ on:
1111
- cron: "5 4 * * SAT"
1212

1313
jobs:
14-
checklinks:
14+
checklinks-internal:
1515
if: github.repository == 'segmentio/segment-docs'
1616
name: Linux
1717
runs-on: ubuntu-latest
18+
env:
19+
GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_REPO_SCOPE }}
20+
REPORT_AUTHOR: docsbot
21+
REPORT_LABEL: report, automated issue, broken links
22+
REPORT_REPOSITORY: segmentio/segment-docs
1823
strategy:
1924
fail-fast: false
2025
steps:
@@ -28,18 +33,17 @@ jobs:
2833
run: yarn install
2934
- name: Build jekyll website with
3035
run: bundle exec jekyll build
31-
- name: Link Checker
32-
uses: lycheeverse/[email protected]
36+
- name: Run Script
37+
run: |
38+
sudo make linkcheck-internal > broken_links.md
39+
- if: ${{ failure() }}
40+
name: Create issue from file
41+
id: broken-link-report
42+
uses: peter-evans/create-issue-from-file@b4f9ee0a9d4abbfc6986601d9b1a4f8f8e74c77e
3343
with:
34-
args: --no-progress _site/**/*.html -a 429 -c ./lychee.toml
35-
env:
36-
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
37-
# - name: "exclude non-400 errors"
38-
# run: |
39-
# sudo grep -v "error sending request" ./lychee/out.md > ./lychee/out2.md
40-
- name: Create Issue From File
41-
uses: peter-evans/create-issue-from-file@v3
42-
with:
43-
title: Link Checker Report
44-
content-filepath: ./lychee/out.md
45-
labels: report, automated issue
44+
token: ${{ env.GITHUB_TOKEN }}
45+
46+
title: Broken Links Report - Local
47+
content-filepath: ./broken_links.md
48+
repository: ${{ env.REPORT_REPOSITORY }}
49+
labels: ${{ env.REPORT_LABEL }}

‎Makefile

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,19 @@ intialize-work-dir:
2929

3030
.PHONY: build
3131
build: node_modules vendor/bundle
32-
@echo "Jekyll env: ${JEKYLL_ENV}"
33-
@chown -R jekyll /workdir
34-
@chmod -R 777 /workdir
35-
@echo "env: ${JEKYLL_ENV}"
36-
@$(BIN)/webpack --mode=production
37-
@JEKYLL_ENV=${JEKYLL_ENV} bundle exec jekyll build --trace
38-
@if [ '${BUILDKITE_BRANCH}' == 'staging' ]; then echo "updating sitemap.xml..." && sed -i -r 's/segment.com/segment.build/g' ./_site/sitemap.xml; fi;
32+
@$(BIN)/concurrently --raw --kill-others -n webpack,jekyll \
33+
"$(BIN)/webpack --mode=development --watch" \
34+
"bundle exec jekyll clean && bundle exec jekyll build -V"
35+
36+
# .PHONY: build
37+
# build: node_modules vendor/bundle
38+
# @echo "Jekyll env: ${JEKYLL_ENV}"
39+
# @chown -R jekyll /workdir
40+
# @chmod -R 777 /workdir
41+
# @echo "env: ${JEKYLL_ENV}"
42+
# @$(BIN)/webpack --mode=production
43+
# @JEKYLL_ENV=${JEKYLL_ENV} bundle exec jekyll build --trace
44+
# @if [ '${BUILDKITE_BRANCH}' == 'staging' ]; then echo "updating sitemap.xml..." && sed -i -r 's/segment.com/segment.build/g' ./_site/sitemap.xml; fi;
3945

4046
.PHONY: upload-docs
4147
upload-docs:
@@ -82,6 +88,16 @@ changelog: vendor/bundle
8288
sidenav: vendor/bundle
8389
@node scripts/nav.js
8490

91+
# check internal links
92+
.PHONY: linkcheck-internal
93+
linkcheck-internal:
94+
@node scripts/checklinks-internal.js
95+
96+
# check external links
97+
.PHONY: linkcheck-external
98+
linkcheck-external:
99+
@node scripts/checklinks-external.js
100+
85101
.PHONY: zip-artifacts
86102
zip-artifacts:
87103
@tar czf build_package.tar.gz _site

‎package.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,16 @@
4747
"ajv": "^6.10.2",
4848
"algoliasearch": "^4.10.5",
4949
"ansi-regex": "^6.0.1",
50+
"browser-sync": "^2.27.7",
51+
"check-links": "^1.1.8",
5052
"clipboard": "^2.0.8",
5153
"dotenv": "^10.0.0",
5254
"glightbox": "^3.1.0",
55+
"globby": "11.0.4",
5356
"handlebars": "^4.7.7",
57+
"ora": "5.4.1",
58+
"posthtml": "^0.16.5",
59+
"posthtml-urls": "^1.0.0",
5460
"search-insights": "^2.0.3",
5561
"tap-spot": "^1.1.1",
5662
"tippy.js": "5.2.0",

‎scripts/catalog_papi.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,6 @@ const getConnectionModes = (destination) => {
118118
}
119119
fs.mkdirSync(docsPath)
120120
fs.writeFileSync(`${docsPath}/index.md`, content)
121-
fs.appendFileSync('src/_data/catalog/incompleteDocs.txt', `${docsPath}\n`)
122121
}
123122
}
124123

@@ -167,6 +166,7 @@ const updateSources = async () => {
167166
const hiddenSources = [
168167
'amp',
169168
'factual-engine',
169+
'twilio-event-streams-beta'
170170
]
171171

172172
sources.forEach(source => {
@@ -197,6 +197,7 @@ const updateSources = async () => {
197197

198198
// create the catalog metadata
199199
let updatedSource = {
200+
id: source.id,
200201
display_name: source.name,
201202
slug,
202203
url,

‎scripts/checklinks-external.js

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// A script to check all external links on the docs site
2+
3+
const globby = require('globby')
4+
const posthtml = require('posthtml')
5+
const fs = require('fs')
6+
const server = require('browser-sync').create()
7+
const checkLinks = require('check-links')
8+
const ora = require('ora')
9+
10+
11+
12+
const checkForDeadExternalUrls = async () => {
13+
try {
14+
// Grab all the files from the specified directory, add their paths to a new set
15+
const files = await globby('_site/**/*.html')
16+
const throbber = ora('Link Check Starting').start()
17+
// Use a set here for efficiency, no duplicate values!
18+
const urls = new Set()
19+
20+
// Because we're checking for external URLs only, add every URL that does not start with a `/`,
21+
// or point to 0.0.0.0. Also exclude segmentio GitHub URLs
22+
const ph = posthtml([
23+
require('posthtml-urls')({
24+
eachURL: (url) => {
25+
if (!url.startsWith('http://0') && !url.startsWith('/') && !url.startsWith('https://github.com/segmentio')) {
26+
urls.add(url)
27+
}
28+
},
29+
}),
30+
])
31+
throbber.succeed()
32+
throbber.start('Processing files')
33+
34+
// Run the above logic on each file
35+
files.forEach((file) => {
36+
ph.process(fs.readFileSync(file))
37+
})
38+
39+
40+
// Check all the links collected in the section above
41+
throbber.start('Checking the links')
42+
const results = await checkLinks(
43+
Array.from(urls).map((url) =>
44+
url
45+
),
46+
)
47+
// If a link returns 'dead' (404), add it to an array
48+
const deadUrls = Array.from(urls).filter(
49+
(url) => results[url].status === 'dead',
50+
)
51+
52+
// If there are dead URLs, list them here, along with the count. Exit status 1 to indicate an error.
53+
if (deadUrls.length > 0) {
54+
throbber.fail(`Dead URLS: ${deadUrls.length}\n\n`)
55+
console.log(`Dead URLS: ${deadUrls.length}\n\n${deadUrls.join('\n')}`)
56+
process.exit(1)
57+
}
58+
// Otherwise, claim that all the links work, and exit the process normally.
59+
else {
60+
console.log('All links work!')
61+
process.exit
62+
}
63+
throbber.stop()
64+
server.exit()
65+
} catch (e) {
66+
console.error(e)
67+
server.exit()
68+
}
69+
}
70+
checkForDeadExternalUrls()

‎scripts/checklinks-internal.js

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// A script the check all local links on the docs site
2+
3+
const globby = require('globby')
4+
const posthtml = require('posthtml')
5+
const fs = require('fs')
6+
const server = require('browser-sync').create()
7+
const checkLinks = require('check-links')
8+
const ora = require('ora')
9+
10+
11+
const checkForDeadLocalUrls = async () => {
12+
try {
13+
// Grab all the files from the specified directory, add their paths to a new set
14+
const files = await globby('_site/**/*.html')
15+
const throbber = ora('Link Check Starting').start()
16+
// Use a set here for efficiency, no duplicate values!
17+
const urls = new Set()
18+
19+
// Logic for collecting the list of URLs to check
20+
// If the link starts with `/docs/`, replace that with a localhost:3000 domain, and add it to the list.
21+
const ph = posthtml([
22+
require('posthtml-urls')({
23+
eachURL: (url) => {
24+
if (url.startsWith('/docs/')) {
25+
urls.add(url.replace('/docs/', 'http://localhost:3000/'))
26+
}
27+
},
28+
}),
29+
])
30+
throbber.succeed()
31+
32+
// Using the logic above, iterate through the entire list of files
33+
throbber.start('Processing files')
34+
files.forEach((file) => {
35+
ph.process(fs.readFileSync(file))
36+
})
37+
throbber.succeed()
38+
39+
// Spin up a lightweight browsersync server to check each URL
40+
throbber.start('Starting server')
41+
await new Promise((resolve) => {
42+
server.init({
43+
port: 3000,
44+
server: {
45+
baseDir: '_site',
46+
},
47+
open: false,
48+
logLevel: 'silent',
49+
},
50+
resolve,
51+
)
52+
throbber.succeed()
53+
})
54+
55+
// Check the links against the local browsersync site
56+
const results = await checkLinks(
57+
Array.from(urls).map((url) =>
58+
url
59+
),
60+
)
61+
62+
// If a link returns 'dead' (404), add it to an array
63+
const deadUrls = Array.from(urls).filter(
64+
(url) => results[url].status === 'dead',
65+
)
66+
67+
// For ease of checking, replace the localhost domain with the live domain, add those to a new array.
68+
let broke = []
69+
deadUrls.forEach(url => {
70+
link = url.replace('http://localhost:3000', 'https://segment.com/docs')
71+
if (!link.endsWith('/')){
72+
link = link+'/'
73+
}
74+
broke.push(link)
75+
});
76+
77+
78+
// Sometimes, we redirect urls based on jekyll settings, or a setting an app-nginx.
79+
// For those, we want to remove them from the list of dead links, because they aren't dead.
80+
81+
// app-nginx redirects
82+
const redirects = ['https://segment.com/docs/guides/usage-and-billing/','https://segment.com/docs/connections/sources/catalog/libraries/website/plugins/', 'https://segment.com/docs/assets/docs.bundle.js/']
83+
84+
// Redirects generated by Jekyll
85+
// Pull the redirects json file
86+
const data = require('../_site/redirects.json')
87+
// Grab the 'from' redirect
88+
Object.keys(data).forEach(key => {
89+
if (!key.endsWith('/')){
90+
key = key+'/'
91+
}
92+
redirects.push('https://segment.com/docs'+key.replace('/docs',''))
93+
})
94+
// Remove the redirect urls from the list of broken URLs
95+
broke = broke.filter(val => !redirects.includes(val));
96+
97+
// If there are dead URLs, list them here, along with the count. Exit status 1 to indicate an error.
98+
99+
if (broke.length > 0) {
100+
throbber.fail(`Dead URLS: ${broke.length}\n\n`)
101+
console.log(`Dead URLS: ${broke.length}\n\n${broke.join('\n')}`)
102+
process.exit(1)
103+
}else {
104+
console.log('All links work!')
105+
process.exit
106+
}
107+
throbber.stop()
108+
server.exit()
109+
} catch (e) {
110+
console.error(e)
111+
server.exit()
112+
}
113+
}
114+
115+
checkForDeadLocalUrls()
116+

‎src/_data/catalog/destination_categories.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# AUTOGENERATED FROM PUBLIC API. DO NOT EDIT
2-
# destination categories last updated 2021-12-14
2+
3+
# destination categories last updated 2022-01-10
4+
35
items:
46
- display_name: A/B Testing
57
slug: a-b-testing

0 commit comments

Comments
 (0)