Skip to content

Commit 3da9c77

Browse files
authored
feat: move static pages out of function bundle (#728)
1 parent bfc016f commit 3da9c77

File tree

6 files changed

+173
-13
lines changed

6 files changed

+173
-13
lines changed

demo/next.config.js

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,14 @@ module.exports = {
99
// trailingSlash: true,
1010
// Configurable site features _to_ support:
1111
// basePath: '/docs',
12+
async rewrites() {
13+
return {
14+
beforeFiles: [
15+
{
16+
source: '/old/:path*',
17+
destination: '/:path*',
18+
}
19+
]
20+
}
21+
}
1222
}

package-lock.json

Lines changed: 22 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@
5858
"chalk": "^4.1.2",
5959
"fs-extra": "^10.0.0",
6060
"moize": "^6.1.0",
61+
"node-fetch": "^2.6.5",
6162
"outdent": "^0.8.0",
63+
"p-limit": "^3.1.0",
6264
"pathe": "^0.2.0",
6365
"semver": "^7.3.5",
6466
"slash": "^3.0.0",
@@ -69,6 +71,7 @@
6971
"@babel/preset-env": "^7.15.8",
7072
"@netlify/eslint-config-node": "^3.3.0",
7173
"@testing-library/cypress": "^8.0.1",
74+
"@types/fs-extra": "^9.0.13",
7275
"@types/jest": "^27.0.2",
7376
"@types/mocha": "^9.0.0",
7477
"babel-jest": "^27.2.5",

src/helpers/files.js

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// @ts-check
2+
const { existsSync, readJson, move, cpSync, copy, writeJson } = require('fs-extra')
3+
const pLimit = require('p-limit')
4+
const { join } = require('pathe')
5+
6+
const TEST_ROUTE = /\/\[[^/]+?](?=\/|$)/
7+
8+
const isDynamicRoute = (route) => TEST_ROUTE.test(route)
9+
10+
exports.moveStaticPages = async ({ netlifyConfig, target, i18n, failBuild }) => {
11+
const root = join(netlifyConfig.build.publish, target === 'server' ? 'server' : 'serverless')
12+
const pagesManifestPath = join(root, 'pages-manifest.json')
13+
if (!existsSync(pagesManifestPath)) {
14+
failBuild(`Could not find pages manifest at ${pagesManifestPath}`)
15+
}
16+
const files = []
17+
18+
const moveFile = async (file) => {
19+
const source = join(root, file)
20+
// Trim the initial "pages"
21+
const filePath = file.slice(6)
22+
files.push(filePath)
23+
const dest = join(netlifyConfig.build.publish, filePath)
24+
await move(source, dest)
25+
}
26+
27+
const pagesManifest = await readJson(pagesManifestPath)
28+
// Arbitrary limit of 10 concurrent file moves
29+
const limit = pLimit(10)
30+
const promises = Object.entries(pagesManifest).map(async ([route, filePath]) => {
31+
if (
32+
isDynamicRoute(route) ||
33+
!(filePath.endsWith('.html') || filePath.endsWith('.json')) ||
34+
filePath.endsWith('/404.html') ||
35+
filePath.endsWith('/500.html')
36+
) {
37+
return
38+
}
39+
return limit(moveFile, filePath)
40+
})
41+
await Promise.all(promises)
42+
console.log(`Moved ${files.length} page files`)
43+
44+
// Write the manifest for use in the serverless functions
45+
await writeJson(join(netlifyConfig.build.publish, 'static-manifest.json'), files)
46+
47+
if (i18n?.defaultLocale) {
48+
// Copy the default locale into the root
49+
await copy(join(netlifyConfig.build.publish, i18n.defaultLocale), `${netlifyConfig.build.publish}/`)
50+
}
51+
}
52+
53+
exports.movePublicFiles = async ({ appDir, publish }) => {
54+
const publicDir = join(appDir, 'public')
55+
if (existsSync(publicDir)) {
56+
await copy(publicDir, `${publish}/`)
57+
}
58+
}

src/index.js

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const { copy, existsSync } = require('fs-extra')
66

77
const { restoreCache, saveCache } = require('./helpers/cache')
88
const { getNextConfig, configureHandlerFunctions, generateRedirects } = require('./helpers/config')
9+
const { moveStaticPages, movePublicFiles } = require('./helpers/files')
910
const { generateFunctions, setupImageFunction, generatePagesResolver } = require('./helpers/functions')
1011
const {
1112
verifyNetlifyBuildVersion,
@@ -52,10 +53,12 @@ module.exports = {
5253
await generateFunctions(constants, appDir)
5354
await generatePagesResolver({ netlifyConfig, target, constants })
5455

55-
const publicDir = join(appDir, 'public')
56-
if (existsSync(publicDir)) {
57-
await copy(publicDir, `${publish}/`)
56+
await movePublicFiles({ appDir, publish })
57+
58+
if (process.env.EXPERIMENTAL_MOVE_STATIC_PAGES) {
59+
await moveStaticPages({ target, failBuild, netlifyConfig, i18n })
5860
}
61+
5962
await setupImageFunction({ constants, imageconfig: images, netlifyConfig, basePath })
6063

6164
await generateRedirects({

src/templates/getHandler.js

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,71 @@
1+
const { promises, createWriteStream, existsSync } = require('fs')
12
const { Server } = require('http')
3+
const { tmpdir } = require('os')
24
const path = require('path')
5+
const { promisify } = require('util')
36

7+
const streamPipeline = promisify(require('stream').pipeline)
48
const { Bridge } = require('@vercel/node/dist/bridge')
9+
const fetch = require('node-fetch')
510

611
const makeHandler =
712
() =>
813
// We return a function and then call `toString()` on it to serialise it as the launcher function
9-
(conf, app) => {
10-
// This is just so nft knows about the page entrypoints
14+
(conf, app, pageRoot, staticManifest = []) => {
15+
// This is just so nft knows about the page entrypoints. It's not actually used
1116
try {
1217
// eslint-disable-next-line node/no-missing-require
1318
require.resolve('./pages.js')
1419
} catch {}
1520

21+
// Set during the request as it needs the host header. Hoisted so we can define the function once
22+
let base
23+
24+
// Only do this if we have some static files moved to the CDN
25+
if (staticManifest.length !== 0) {
26+
// These are static page files that have been removed from the function bundle
27+
// In most cases these are served from the CDN, but for rewrites Next may try to read them
28+
// from disk. We need to intercept these and load them from the CDN instead
29+
// Sadly the only way to do this is to monkey-patch fs.promises. Yeah, I know.
30+
const staticFiles = new Set(staticManifest)
31+
32+
// Yes, you can cache stuff locally in a Lambda
33+
const cacheDir = path.join(tmpdir(), 'next-static-cache')
34+
// Grab the real fs.promises.readFile...
35+
const readfileOrig = promises.readFile
36+
// ...then money-patch it to see if it's requesting a CDN file
37+
promises.readFile = async (file, options) => {
38+
// We only care about page files
39+
if (file.startsWith(pageRoot)) {
40+
// We only want the part after `pages/`
41+
const filePath = file.slice(pageRoot.length + 1)
42+
// Is it in the CDN and not local?
43+
if (staticFiles.has(filePath) && !existsSync(file)) {
44+
// This name is safe to use, because it's one that was already created by Next
45+
const cacheFile = path.join(cacheDir, filePath)
46+
// Have we already cached it? We ignore the cache if running locally to avoid staleness
47+
if ((!existsSync(cacheFile) || process.env.NETLIFY_DEV) && base) {
48+
await promises.mkdir(path.dirname(cacheFile), { recursive: true })
49+
50+
// Append the path to our host and we can load it like a regular page
51+
const url = `${base}/${filePath}`
52+
console.log(`Downloading ${url} to ${cacheFile}`)
53+
const response = await fetch(url)
54+
if (!response.ok) {
55+
// Next catches this and returns it as a not found file
56+
throw new Error(`Failed to fetch ${url}`)
57+
}
58+
// Stream it to disk
59+
await streamPipeline(response.body, createWriteStream(cacheFile))
60+
}
61+
// Return the cache file
62+
return readfileOrig(cacheFile, options)
63+
}
64+
}
65+
66+
return readfileOrig(file, options)
67+
}
68+
}
1669
let NextServer
1770
try {
1871
// next >= 11.0.1. Yay breaking changes in patch releases!
@@ -59,7 +112,12 @@ const makeHandler =
59112
// Next expects to be able to parse the query from the URL
60113
const query = new URLSearchParams(event.queryStringParameters).toString()
61114
event.path = query ? `${event.path}?${query}` : event.path
62-
115+
// Only needed if we're intercepting static files
116+
if (staticManifest.length !== 0) {
117+
const { host } = event.headers
118+
const protocol = event.headers['x-forwarded-proto'] || 'http'
119+
base = `${protocol}://${host}`
120+
}
63121
const { headers, ...result } = await bridge.launcher(event, context)
64122
/** @type import("@netlify/functions").HandlerResponse */
65123

@@ -88,15 +146,26 @@ const makeHandler =
88146

89147
const getHandler = ({ isODB = false, publishDir = '../../../.next', appDir = '../../..' }) => `
90148
const { Server } = require("http");
149+
const { tmpdir } = require('os')
150+
const { promises, createWriteStream, existsSync } = require("fs");
151+
const { promisify } = require('util')
152+
const streamPipeline = promisify(require('stream').pipeline)
91153
// We copy the file here rather than requiring from the node module
92154
const { Bridge } = require("./bridge");
155+
const fetch = require('node-fetch')
156+
93157
const { builder } = require("@netlify/functions");
94158
const { config } = require("${publishDir}/required-server-files.json")
159+
let staticManifest
160+
try {
161+
staticManifest = require("${publishDir}/static-manifest.json")
162+
} catch {}
95163
const path = require("path");
164+
const pageRoot = path.resolve(path.join(__dirname, "${publishDir}", config.target === "server" ? "server" : "serverless", "pages"));
96165
exports.handler = ${
97166
isODB
98-
? `builder((${makeHandler().toString()})(config, "${appDir}"));`
99-
: `(${makeHandler().toString()})(config, "${appDir}");`
167+
? `builder((${makeHandler().toString()})(config, "${appDir}", pageRoot, staticManifest));`
168+
: `(${makeHandler().toString()})(config, "${appDir}", pageRoot, staticManifest);`
100169
}
101170
`
102171

0 commit comments

Comments
 (0)