Skip to content

Commit 2731d66

Browse files
ijjkztanner
ijjk
and
ztanner
authored
Add store only mode for experimental tracing (#68548)
This adds new handling to allow seeding a cache with the experimental tracing but not leverage the stitch handling so that more cache hits can occur without fully opting all environments into the experimental behavior fully. This is tested in our flying-shuttle suite by the initial build in the test only seeding the cache and successive builds using the experimental flag fully. --------- Co-authored-by: Zack Tanner <[email protected]>
1 parent 03a6b18 commit 2731d66

File tree

8 files changed

+93
-54
lines changed

8 files changed

+93
-54
lines changed

‎packages/next/src/build/entries.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ export async function createEntrypoints(
677677
basePath: config.basePath,
678678
assetPrefix: config.assetPrefix,
679679
nextConfigOutput: config.output,
680-
flyingShuttle: config.experimental.flyingShuttle,
680+
flyingShuttle: Boolean(config.experimental.flyingShuttle),
681681
nextConfigExperimentalUseEarlyImport: config.experimental
682682
.useEarlyImport
683683
? true
@@ -743,7 +743,7 @@ export async function createEntrypoints(
743743
basePath: config.basePath,
744744
assetPrefix: config.assetPrefix,
745745
nextConfigOutput: config.output,
746-
flyingShuttle: config.experimental.flyingShuttle,
746+
flyingShuttle: Boolean(config.experimental.flyingShuttle),
747747
// This isn't used with edge as it needs to be set on the entry module, which will be the `edgeServerEntry` instead.
748748
// Still passing it here for consistency.
749749
preferredRegion: staticInfo.preferredRegion,

‎packages/next/src/build/index.ts

Lines changed: 49 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -726,9 +726,13 @@ export default async function build(
726726
)
727727
NextBuildContext.buildId = buildId
728728

729+
const { flyingShuttle } = config.experimental
730+
const isFullFlyingShuttle = flyingShuttle?.mode === 'full'
731+
const isStoreOnlyFlyingShuttle = flyingShuttle?.mode === 'store-only'
732+
729733
const shuttleDir = path.join(distDir, 'cache', 'shuttle')
730734

731-
if (config.experimental.flyingShuttle) {
735+
if (flyingShuttle) {
732736
await fs.mkdir(shuttleDir, {
733737
recursive: true,
734738
})
@@ -865,7 +869,7 @@ export default async function build(
865869
unchanged: DetectedEntriesResult
866870
}
867871

868-
if (pagesPaths && config.experimental.flyingShuttle) {
872+
if (pagesPaths && isFullFlyingShuttle) {
869873
changedPagePathsResult = await detectChangedEntries({
870874
pagesPaths,
871875
pageExtensions: config.pageExtensions,
@@ -965,7 +969,7 @@ export default async function build(
965969
})
966970
)
967971

968-
if (appPaths && config.experimental.flyingShuttle) {
972+
if (appPaths && isFullFlyingShuttle) {
969973
changedAppPathsResult = await detectChangedEntries({
970974
appPaths,
971975
pageExtensions: config.pageExtensions,
@@ -1198,7 +1202,7 @@ export default async function build(
11981202
)
11991203
const filterPaths: string[] = []
12001204

1201-
if (config.experimental.flyingShuttle) {
1205+
if (flyingShuttle) {
12021206
filterPaths.push(
12031207
...[
12041208
// client filter always has all app paths
@@ -1707,7 +1711,7 @@ export default async function build(
17071711
hasSsrAmpPages: false,
17081712
buildTraceContext,
17091713
outputFileTracingRoot,
1710-
isFlyingShuttle: !!config.experimental.flyingShuttle,
1714+
isFlyingShuttle: Boolean(flyingShuttle),
17111715
})
17121716
.catch((err) => {
17131717
console.error(err)
@@ -2410,7 +2414,7 @@ export default async function build(
24102414
hasSsrAmpPages,
24112415
buildTraceContext,
24122416
outputFileTracingRoot,
2413-
isFlyingShuttle: !!config.experimental.flyingShuttle,
2417+
isFlyingShuttle: Boolean(flyingShuttle),
24142418
}).catch((err) => {
24152419
console.error(err)
24162420
process.exit(1)
@@ -2517,45 +2521,49 @@ export default async function build(
25172521
)
25182522

25192523
if (!isGenerateMode) {
2520-
if (config.experimental.flyingShuttle) {
2524+
if (flyingShuttle) {
25212525
await buildTracesPromise
25222526

2523-
console.log('stitching builds...')
2524-
const stitchResult = await stitchBuilds(
2525-
{
2526-
config,
2527-
buildId,
2528-
distDir,
2529-
shuttleDir,
2530-
rewrites,
2531-
redirects,
2532-
edgePreviewProps: {
2533-
__NEXT_PREVIEW_MODE_ID:
2534-
NextBuildContext.previewProps!.previewModeId,
2535-
__NEXT_PREVIEW_MODE_ENCRYPTION_KEY:
2536-
NextBuildContext.previewProps!.previewModeEncryptionKey,
2537-
__NEXT_PREVIEW_MODE_SIGNING_KEY:
2538-
NextBuildContext.previewProps!.previewModeSigningKey,
2539-
},
2540-
encryptionKey,
2541-
allowedErrorRate:
2542-
config.experimental.clientRouterFilterAllowedRate,
2543-
},
2544-
{
2545-
changed: {
2546-
pages: changedPagePathsResult?.changed.pages || [],
2547-
app: changedAppPathsResult?.changed.app || [],
2548-
},
2549-
unchanged: {
2550-
pages: changedPagePathsResult?.unchanged.pages || [],
2551-
app: changedAppPathsResult?.unchanged.app || [],
2527+
if (isStoreOnlyFlyingShuttle) {
2528+
console.log('skipping stitching builds due to store-only mode')
2529+
} else {
2530+
console.log('stitching builds...')
2531+
const stitchResult = await stitchBuilds(
2532+
{
2533+
config,
2534+
buildId,
2535+
distDir,
2536+
shuttleDir,
2537+
rewrites,
2538+
redirects,
2539+
edgePreviewProps: {
2540+
__NEXT_PREVIEW_MODE_ID:
2541+
NextBuildContext.previewProps!.previewModeId,
2542+
__NEXT_PREVIEW_MODE_ENCRYPTION_KEY:
2543+
NextBuildContext.previewProps!.previewModeEncryptionKey,
2544+
__NEXT_PREVIEW_MODE_SIGNING_KEY:
2545+
NextBuildContext.previewProps!.previewModeSigningKey,
2546+
},
2547+
encryptionKey,
2548+
allowedErrorRate:
2549+
config.experimental.clientRouterFilterAllowedRate,
25522550
},
2553-
pageExtensions: config.pageExtensions,
2551+
{
2552+
changed: {
2553+
pages: changedPagePathsResult?.changed.pages || [],
2554+
app: changedAppPathsResult?.changed.app || [],
2555+
},
2556+
unchanged: {
2557+
pages: changedPagePathsResult?.unchanged.pages || [],
2558+
app: changedAppPathsResult?.unchanged.app || [],
2559+
},
2560+
pageExtensions: config.pageExtensions,
2561+
}
2562+
)
2563+
// reload pagesManifest since it's been updated on disk
2564+
if (stitchResult.pagesManifest) {
2565+
pagesManifest = stitchResult.pagesManifest
25542566
}
2555-
)
2556-
// reload pagesManifest since it's been updated on disk
2557-
if (stitchResult.pagesManifest) {
2558-
pagesManifest = stitchResult.pagesManifest
25592567
}
25602568

25612569
console.log('storing shuttle')

‎packages/next/src/build/webpack-config.ts

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,10 @@ export default async function getBaseWebpackConfig(
728728
isNodeServer ? new OptionalPeerDependencyResolverPlugin() : undefined,
729729
].filter(Boolean) as webpack.ResolvePluginInstance[],
730730
}
731+
// we don't want to modify the outputs naming if we're
732+
// in store-only mode
733+
const { flyingShuttle } = config.experimental
734+
const isFullFlyingShuttle = flyingShuttle?.mode === 'full'
731735

732736
// Packages which will be split into the 'framework' chunk.
733737
// Only top-level packages are included, e.g. nested copies like
@@ -968,7 +972,7 @@ export default async function getBaseWebpackConfig(
968972

969973
if (isNodeServer || isEdgeServer) {
970974
return {
971-
filename: `${isEdgeServer ? `edge-chunks${config.experimental.flyingShuttle ? `-${buildId}` : ''}/` : ''}[name].js`,
975+
filename: `${isEdgeServer ? `edge-chunks${isFullFlyingShuttle ? `-${buildId}` : ''}/` : ''}[name].js`,
972976
chunks: 'all',
973977
minChunks: 2,
974978
}
@@ -1135,7 +1139,7 @@ export default async function getBaseWebpackConfig(
11351139
hashFunction: 'xxhash64',
11361140
hashDigestLength: 16,
11371141

1138-
...(config.experimental.flyingShuttle
1142+
...(isFullFlyingShuttle
11391143
? {
11401144
// ensure we only use contenthash as it's more deterministic
11411145
filename: (p) => {
@@ -1770,7 +1774,7 @@ export default async function getBaseWebpackConfig(
17701774
dev,
17711775
}),
17721776
(isClient || isEdgeServer) && new DropClientPage(),
1773-
(isNodeServer || (config.experimental.flyingShuttle && isEdgeServer)) &&
1777+
(isNodeServer || (flyingShuttle && isEdgeServer)) &&
17741778
!dev &&
17751779
new (require('./webpack/plugins/next-trace-entrypoints-plugin')
17761780
.TraceEntryPointsPlugin as typeof import('./webpack/plugins/next-trace-entrypoints-plugin').TraceEntryPointsPlugin)(
@@ -1784,7 +1788,7 @@ export default async function getBaseWebpackConfig(
17841788
turbotrace: config.experimental.turbotrace,
17851789
optOutBundlingPackages,
17861790
traceIgnores: [],
1787-
flyingShuttle: !!config.experimental.flyingShuttle,
1791+
flyingShuttle: Boolean(flyingShuttle),
17881792
compilerType,
17891793
swcLoaderConfig: swcDefaultLoader,
17901794
}

‎packages/next/src/build/webpack/plugins/define-env-plugin.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ export function getDefineEnv({
141141
}: DefineEnvPluginOptions): SerializedDefineEnv {
142142
const nextPublicEnv = getNextPublicEnvironmentVariables()
143143

144-
if (config.experimental.flyingShuttle) {
144+
if (Boolean(config.experimental.flyingShuttle)) {
145145
// we delay inlining these values until after the build
146146
// with flying shuttle enabled so we can update them
147147
// without invalidating entries
@@ -213,8 +213,9 @@ export function getDefineEnv({
213213
? 5 * 60 // 5 minutes
214214
: config.experimental.staleTimes?.static
215215
),
216-
'process.env.__NEXT_FLYING_SHUTTLE':
217-
config.experimental.flyingShuttle ?? false,
216+
'process.env.__NEXT_FLYING_SHUTTLE': Boolean(
217+
config.experimental.flyingShuttle
218+
),
218219
'process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED':
219220
config.experimental.clientRouterFilter ?? true,
220221
'process.env.__NEXT_CLIENT_ROUTER_S_FILTER':

‎packages/next/src/server/config-schema.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,13 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
293293
externalMiddlewareRewritesResolve: z.boolean().optional(),
294294
fallbackNodePolyfills: z.literal(false).optional(),
295295
fetchCacheKeyPrefix: z.string().optional(),
296-
flyingShuttle: z.boolean().optional(),
296+
flyingShuttle: z
297+
.strictObject({
298+
mode: z
299+
.union([z.literal('full'), z.literal('store-only')])
300+
.optional(),
301+
})
302+
.optional(),
297303
forceSwcTransforms: z.boolean().optional(),
298304
fullySpecified: z.boolean().optional(),
299305
gzipSize: z.boolean().optional(),

‎packages/next/src/server/config-shared.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ export interface LoggingConfig {
212212

213213
export interface ExperimentalConfig {
214214
appNavFailHandling?: boolean
215-
flyingShuttle?: boolean
215+
flyingShuttle?: { mode?: 'full' | 'store-only' }
216216
prerenderEarlyExit?: boolean
217217
linkNoTouchStart?: boolean
218218
caseSensitiveRoutes?: boolean
@@ -970,7 +970,11 @@ export const defaultConfig: NextConfig = {
970970
outputFileTracingRoot: process.env.NEXT_PRIVATE_OUTPUT_TRACE_ROOT || '',
971971
experimental: {
972972
appNavFailHandling: Boolean(process.env.NEXT_PRIVATE_FLYING_SHUTTLE),
973-
flyingShuttle: Boolean(process.env.NEXT_PRIVATE_FLYING_SHUTTLE),
973+
flyingShuttle: Boolean(process.env.NEXT_PRIVATE_FLYING_SHUTTLE)
974+
? {
975+
mode: 'full',
976+
}
977+
: undefined,
974978
prerenderEarlyExit: true,
975979
serverMinification: true,
976980
serverSourceMaps: false,

‎test/e2e/app-dir/app/flying-shuttle.test.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import { nextTestSetup, isNextStart } from 'e2e-utils'
1919
nanoid: '4.0.1',
2020
},
2121
env: {
22-
NEXT_PRIVATE_FLYING_SHUTTLE: '1',
22+
NEXT_PRIVATE_FLYING_SHUTTLE_STORE_ONLY: '1',
2323
},
2424
})
2525
let initialConfig: Record<string, any> = {}
@@ -248,6 +248,11 @@ import { nextTestSetup, isNextStart } from 'e2e-utils'
248248
}
249249

250250
it('should only rebuild just a changed app route correctly', async () => {
251+
// our initial build was built in store-only mode so
252+
// enable full version in successive builds
253+
delete next.env['NEXT_PRIVATE_FLYING_SHUTTLE_STORE_ONLY']
254+
next.env['NEXT_PRIVATE_FLYING_SHUTTLE'] = '1'
255+
251256
await next.stop()
252257

253258
const dataPath = 'app/dashboard/deployments/[id]/data.json'

‎test/e2e/app-dir/app/next.config.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,17 @@ module.exports = {
77
parallelServerCompiles: true,
88
parallelServerBuildTraces: true,
99
webpackBuildWorker: true,
10+
appNavFailHandling: Boolean(
11+
process.env.NEXT_PRIVATE_FLYING_SHUTTLE_STORE_ONLY ||
12+
process.env.NEXT_PRIVATE_FLYING_SHUTTLE
13+
),
14+
flyingShuttle: Boolean(process.env.NEXT_PRIVATE_FLYING_SHUTTLE_STORE_ONLY)
15+
? { mode: 'store-only' }
16+
: Boolean(process.env.NEXT_PRIVATE_FLYING_SHUTTLE)
17+
? {
18+
mode: 'full',
19+
}
20+
: undefined,
1021
},
1122
// output: 'standalone',
1223
rewrites: async () => {

0 commit comments

Comments
 (0)