Skip to content

Commit f85ecb4

Browse files
michaelfigmichaelfig
authored
Merge pull request #1544 from endojs/mfig-bundle-cache
feat(bundle-source): separate the `cache.js` library from the tool
2 parents 59bfe7c + b049ff4 commit f85ecb4

File tree

5 files changed

+421
-253
lines changed

5 files changed

+421
-253
lines changed

‎packages/bundle-source/cache.js

+362
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,362 @@
1+
// @ts-check
2+
import { makePromiseKit } from '@endo/promise-kit';
3+
import { makeReadPowers } from '@endo/compartment-mapper/node-powers.js';
4+
5+
import { asyncGenerate } from 'jessie.js';
6+
7+
import bundleSource from './src/index.js';
8+
9+
const { Fail, quote: q } = assert;
10+
11+
/**
12+
* @typedef {(...args: unknown[]) => void} Logger A message logger.
13+
*/
14+
15+
/**
16+
* @typedef {object} BundleMeta
17+
* @property {string} bundleFileName
18+
* @property {string} bundleTime ISO format
19+
* @property {{ relative: string, absolute: string }} moduleSource
20+
* @property {Array<{ relativePath: string, mtime: string }>} contents
21+
*/
22+
23+
/**
24+
* @param {string} fileName
25+
* @param {{
26+
* fs: {
27+
* promises: Pick<import('fs/promises'),'readFile' | 'stat'>
28+
* },
29+
* path: Pick<import('path'), 'resolve' | 'relative' | 'normalize'>,
30+
* }} powers
31+
*/
32+
export const makeFileReader = (fileName, { fs, path }) => {
33+
const make = there => makeFileReader(there, { fs, path });
34+
35+
// fs.promises.exists isn't implemented in Node.js apparently because it's pure
36+
// sugar.
37+
const exists = fn =>
38+
fs.promises.stat(fn).then(
39+
() => true,
40+
e => {
41+
if (e.code === 'ENOENT') {
42+
return false;
43+
}
44+
throw e;
45+
},
46+
);
47+
48+
return harden({
49+
toString: () => fileName,
50+
readText: () => fs.promises.readFile(fileName, 'utf-8'),
51+
neighbor: ref => make(path.resolve(fileName, ref)),
52+
stat: () => fs.promises.stat(fileName),
53+
absolute: () => path.normalize(fileName),
54+
relative: there => path.relative(fileName, there),
55+
exists: () => exists(fileName),
56+
});
57+
};
58+
59+
/**
60+
* @param {string} fileName
61+
* @param {{
62+
* fs: Pick<import('fs'), 'existsSync'> &
63+
* { promises: Pick<
64+
* import('fs/promises'),
65+
* 'readFile' | 'stat' | 'writeFile' | 'mkdir' | 'rm'
66+
* >,
67+
* },
68+
* path: Pick<import('path'), 'resolve' | 'relative' | 'normalize'>,
69+
* }} io
70+
*/
71+
export const makeFileWriter = (fileName, { fs, path }) => {
72+
const make = there => makeFileWriter(there, { fs, path });
73+
return harden({
74+
toString: () => fileName,
75+
writeText: (txt, opts) => fs.promises.writeFile(fileName, txt, opts),
76+
readOnly: () => makeFileReader(fileName, { fs, path }),
77+
neighbor: ref => make(path.resolve(fileName, ref)),
78+
mkdir: opts => fs.promises.mkdir(fileName, opts),
79+
rm: opts => fs.promises.rm(fileName, opts),
80+
});
81+
};
82+
83+
export const jsOpts = {
84+
encodeBundle: bundle => `export default ${JSON.stringify(bundle)};\n`,
85+
toBundleName: n => `bundle-${n}.js`,
86+
toBundleMeta: n => `bundle-${n}-js-meta.json`,
87+
toBundleLock: n => `bundle-${n}-js.lock`,
88+
};
89+
90+
export const jsonOpts = {
91+
encodeBundle: bundle => `${JSON.stringify(bundle)}\n`,
92+
toBundleName: n => `bundle-${n}.json`,
93+
toBundleMeta: n => `bundle-${n}-json-meta.json`,
94+
toBundleLock: n => `bundle-${n}-json.lock`,
95+
};
96+
97+
export const makeBundleCache = (wr, cwd, readPowers, opts) => {
98+
const {
99+
cacheOpts: {
100+
encodeBundle,
101+
toBundleName,
102+
toBundleMeta,
103+
toBundleLock,
104+
} = jsOpts,
105+
log: defaultLog = console.warn,
106+
...bundleOptions
107+
} = opts || {};
108+
109+
const add = async (rootPath, targetName, log = defaultLog) => {
110+
const srcRd = cwd.neighbor(rootPath);
111+
112+
const modTimeByPath = new Map();
113+
114+
const loggedRead = async loc => {
115+
if (!loc.match(/\bpackage.json$/)) {
116+
try {
117+
const itemRd = cwd.neighbor(new URL(loc).pathname);
118+
const ref = srcRd.relative(itemRd.absolute());
119+
/** @type {import('fs').Stats} */
120+
const { mtime } = await itemRd.stat();
121+
modTimeByPath.set(ref, mtime);
122+
// console.log({ loc, mtime, ref });
123+
} catch (oops) {
124+
log(oops);
125+
}
126+
}
127+
return readPowers.read(loc);
128+
};
129+
130+
await wr.mkdir({ recursive: true });
131+
132+
const lockWr = wr.neighbor(toBundleLock(targetName));
133+
134+
// Check the bundle/meta file write lock.
135+
try {
136+
await lockWr.writeText('', { flag: 'wx' });
137+
} catch (oops) {
138+
if (oops.code !== 'EEXIST') {
139+
throw oops;
140+
}
141+
142+
// The lock exists, so something is already writing the bundle on our
143+
// behalf.
144+
//
145+
// All we need to do is try validating the bundle, which will first wait
146+
// for the lock to disappear before reading the freshly-written bundle.
147+
148+
// eslint-disable-next-line no-use-before-define
149+
return validate(targetName, rootPath);
150+
}
151+
152+
try {
153+
const bundleFileName = toBundleName(targetName);
154+
const bundleWr = wr.neighbor(bundleFileName);
155+
const metaWr = wr.neighbor(toBundleMeta(targetName));
156+
157+
// Prevent other processes from doing too much work just to see that we're
158+
// already on it.
159+
await metaWr.rm({ force: true });
160+
await bundleWr.rm({ force: true });
161+
162+
const bundle = await bundleSource(rootPath, bundleOptions, {
163+
...readPowers,
164+
read: loggedRead,
165+
});
166+
167+
const { moduleFormat } = bundle;
168+
assert.equal(moduleFormat, 'endoZipBase64');
169+
170+
const code = encodeBundle(bundle);
171+
await wr.mkdir({ recursive: true });
172+
await bundleWr.writeText(code);
173+
174+
/** @type {import('fs').Stats} */
175+
const { mtime: bundleTime } = await bundleWr.readOnly().stat();
176+
177+
/** @type {BundleMeta} */
178+
const meta = {
179+
bundleFileName,
180+
bundleTime: bundleTime.toISOString(),
181+
moduleSource: {
182+
relative: bundleWr.readOnly().relative(srcRd.absolute()),
183+
absolute: srcRd.absolute(),
184+
},
185+
contents: [...modTimeByPath.entries()].map(([relativePath, mtime]) => ({
186+
relativePath,
187+
mtime: mtime.toISOString(),
188+
})),
189+
};
190+
191+
await metaWr.writeText(JSON.stringify(meta, null, 2));
192+
return meta;
193+
} finally {
194+
await lockWr.rm({ force: true });
195+
}
196+
};
197+
198+
const validate = async (targetName, rootOpt, log = defaultLog) => {
199+
const metaRd = wr.readOnly().neighbor(toBundleMeta(targetName));
200+
const lockRd = wr.readOnly().neighbor(toBundleLock(targetName));
201+
202+
// Wait for the bundle to be written.
203+
const lockDone = async () => {
204+
const notDone = await lockRd.exists();
205+
return {
206+
done: !notDone,
207+
value: undefined,
208+
};
209+
};
210+
for await (const _ of asyncGenerate(lockDone)) {
211+
log(`${wr}`, 'waiting for bundle read lock:', `${lockRd}`, 'in', rootOpt);
212+
await readPowers.delay(1000);
213+
}
214+
215+
let txt;
216+
try {
217+
txt = await metaRd.readText();
218+
} catch (ioErr) {
219+
Fail`${q(targetName)}: cannot read bundle metadata: ${q(ioErr)}`;
220+
}
221+
/** @type {BundleMeta} */
222+
const meta = JSON.parse(txt);
223+
const {
224+
bundleFileName,
225+
bundleTime,
226+
contents,
227+
moduleSource: { absolute: moduleSource },
228+
} = meta;
229+
assert.equal(bundleFileName, toBundleName(targetName));
230+
if (rootOpt) {
231+
moduleSource === cwd.neighbor(rootOpt).absolute() ||
232+
Fail`bundle ${targetName} was for ${moduleSource}, not ${rootOpt}`;
233+
}
234+
/** @type {import('fs').Stats} */
235+
const { mtime: actualBundleTime } = await wr
236+
.readOnly()
237+
.neighbor(bundleFileName)
238+
.stat();
239+
assert.equal(actualBundleTime.toISOString(), bundleTime);
240+
const moduleRd = wr.readOnly().neighbor(moduleSource);
241+
const actualTimes = await Promise.all(
242+
contents.map(async ({ relativePath }) => {
243+
const itemRd = moduleRd.neighbor(relativePath);
244+
/** @type {import('fs').Stats} */
245+
const { mtime } = await itemRd.stat();
246+
return { relativePath, mtime: mtime.toISOString() };
247+
}),
248+
);
249+
const outOfDate = actualTimes.filter(({ mtime }) => mtime > bundleTime);
250+
outOfDate.length === 0 ||
251+
Fail`out of date: ${q(outOfDate)}. ${q(targetName)} bundled at ${q(
252+
bundleTime,
253+
)}`;
254+
return meta;
255+
};
256+
257+
/**
258+
* @param {string} rootPath
259+
* @param {string} targetName
260+
* @param {Logger} [log]
261+
* @returns {Promise<BundleMeta>}
262+
*/
263+
const validateOrAdd = async (rootPath, targetName, log = defaultLog) => {
264+
let meta;
265+
const metaExists = await wr
266+
.readOnly()
267+
.neighbor(toBundleMeta(targetName))
268+
.exists();
269+
if (metaExists) {
270+
try {
271+
meta = await validate(targetName, rootPath, log);
272+
const { bundleTime, contents } = meta;
273+
log(
274+
`${wr}`,
275+
toBundleName(targetName),
276+
'valid:',
277+
contents.length,
278+
'files bundled at',
279+
bundleTime,
280+
);
281+
} catch (invalid) {
282+
log(invalid);
283+
}
284+
}
285+
if (!meta) {
286+
log(`${wr}`, 'add:', targetName, 'from', rootPath);
287+
meta = await add(rootPath, targetName, log);
288+
const { bundleFileName, bundleTime, contents } = meta;
289+
log(
290+
`${wr}`,
291+
'bundled',
292+
contents.length,
293+
'files in',
294+
bundleFileName,
295+
'at',
296+
bundleTime,
297+
);
298+
}
299+
return meta;
300+
};
301+
302+
const loaded = new Map();
303+
/**
304+
* @param {string} rootPath
305+
* @param {string} [targetName]
306+
* @param {Logger} [log]
307+
*/
308+
const load = async (
309+
rootPath,
310+
targetName = readPowers.basename(rootPath, '.js'),
311+
log = defaultLog,
312+
) => {
313+
const found = loaded.get(targetName);
314+
// console.log('load', { targetName, found: !!found, rootPath });
315+
if (found && found.rootPath === rootPath) {
316+
return found.bundle;
317+
}
318+
const todo = makePromiseKit();
319+
loaded.set(targetName, { rootPath, bundle: todo.promise });
320+
const bundle = await validateOrAdd(rootPath, targetName, log)
321+
.then(({ bundleFileName }) =>
322+
import(`${wr.readOnly().neighbor(bundleFileName)}`),
323+
)
324+
.then(m => harden(m.default));
325+
assert.equal(bundle.moduleFormat, 'endoZipBase64');
326+
todo.resolve(bundle);
327+
return bundle;
328+
};
329+
330+
return harden({
331+
add,
332+
validate,
333+
validateOrAdd,
334+
load,
335+
});
336+
};
337+
338+
/**
339+
* @param {string} dest
340+
* @param {{ format?: string, dev?: boolean }} options
341+
* @param {(id: string) => Promise<any>} loadModule
342+
*/
343+
export const makeNodeBundleCache = async (dest, options, loadModule) => {
344+
const [fs, path, url, crypto, timers] = await Promise.all([
345+
await loadModule('fs'),
346+
await loadModule('path'),
347+
await loadModule('url'),
348+
await loadModule('crypto'),
349+
await loadModule('timers'),
350+
]);
351+
352+
const readPowers = {
353+
...makeReadPowers({ fs, url, crypto }),
354+
delay: ms => new Promise(resolve => timers.setTimeout(resolve, ms)),
355+
basename: path.basename,
356+
};
357+
358+
const cwd = makeFileReader('', { fs, path });
359+
await fs.promises.mkdir(dest, { recursive: true });
360+
const destWr = makeFileWriter(dest, { fs, path });
361+
return makeBundleCache(destWr, cwd, readPowers, options);
362+
};

‎packages/bundle-source/package.json

+3
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"exports": {
1111
".": "./src/index.js",
1212
"./exported.js": "./exported.js",
13+
"./cache.js": "./cache.js",
1314
"./package.json": "./package.json"
1415
},
1516
"scripts": {
@@ -32,9 +33,11 @@
3233
"@endo/base64": "^0.2.30",
3334
"@endo/compartment-mapper": "^0.8.2",
3435
"@endo/init": "^0.5.54",
36+
"@endo/promise-kit": "^0.2.54",
3537
"@rollup/plugin-commonjs": "^19.0.0",
3638
"@rollup/plugin-node-resolve": "^13.0.0",
3739
"acorn": "^8.2.4",
40+
"jessie.js": "^0.3.2",
3841
"rollup": "endojs/endo#rollup-2.7.1-patch-1",
3942
"source-map": "^0.7.3"
4043
},

0 commit comments

Comments
 (0)