Skip to content

Commit b543af6

Browse files
committed
Patched MicroPython mountNativeFS
1 parent 009229d commit b543af6

File tree

7 files changed

+346
-4
lines changed

7 files changed

+346
-4
lines changed

docs/index.js

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

docs/index.js.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

esm/interpreter/_nativefs.js

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
import { mkdirTree } from './_utils.js';
2+
3+
// (C) Pyodide https://github.com/pyodide/pyodide - Mozilla Public License Version 2.0
4+
// JS port of https://github.com/pyodide/pyodide/blob/34fcd02172895d75db369994011409324f9e3cce/src/js/nativefs.ts
5+
export function initializeNativeFS(module) {
6+
const FS = module.FS;
7+
const MEMFS = module.FS.filesystems.MEMFS;
8+
const PATH = module.PATH;
9+
10+
const nativeFSAsync = {
11+
// DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */,
12+
// FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */,
13+
DIR_MODE: 16384 | 511,
14+
FILE_MODE: 32768 | 511,
15+
mount: function (mount) {
16+
if (!mount.opts.fileSystemHandle) {
17+
throw new Error('opts.fileSystemHandle is required');
18+
}
19+
20+
// reuse all of the core MEMFS functionality
21+
return MEMFS.mount.apply(null, arguments);
22+
},
23+
syncfs: async (mount, populate, callback) => {
24+
try {
25+
const local = nativeFSAsync.getLocalSet(mount);
26+
const remote = await nativeFSAsync.getRemoteSet(mount);
27+
const src = populate ? remote : local;
28+
const dst = populate ? local : remote;
29+
await nativeFSAsync.reconcile(mount, src, dst);
30+
callback(null);
31+
} catch (e) {
32+
callback(e);
33+
}
34+
},
35+
// Returns file set of emscripten's filesystem at the mountpoint.
36+
getLocalSet: (mount) => {
37+
let entries = Object.create(null);
38+
39+
function isRealDir(p) {
40+
return p !== '.' && p !== '..';
41+
}
42+
43+
function toAbsolute(root) {
44+
return (p) => {
45+
return PATH.join2(root, p);
46+
};
47+
}
48+
49+
let check = FS.readdir(mount.mountpoint)
50+
.filter(isRealDir)
51+
.map(toAbsolute(mount.mountpoint));
52+
53+
while (check.length) {
54+
let path = check.pop();
55+
let stat = FS.stat(path);
56+
57+
if (FS.isDir(stat.mode)) {
58+
check.push.apply(
59+
check,
60+
FS.readdir(path).filter(isRealDir).map(toAbsolute(path)),
61+
);
62+
}
63+
64+
entries[path] = { timestamp: stat.mtime, mode: stat.mode };
65+
}
66+
67+
return { type: 'local', entries: entries };
68+
},
69+
// Returns file set of the real, on-disk filesystem at the mountpoint.
70+
getRemoteSet: async (mount) => {
71+
// TODO: this should be a map.
72+
const entries = Object.create(null);
73+
74+
const handles = await getFsHandles(mount.opts.fileSystemHandle);
75+
for (const [path, handle] of handles) {
76+
if (path === '.') continue;
77+
78+
entries[PATH.join2(mount.mountpoint, path)] = {
79+
timestamp:
80+
handle.kind === 'file'
81+
? (await handle.getFile()).lastModifiedDate
82+
: new Date(),
83+
mode:
84+
handle.kind === 'file'
85+
? nativeFSAsync.FILE_MODE
86+
: nativeFSAsync.DIR_MODE,
87+
};
88+
}
89+
90+
return { type: 'remote', entries, handles };
91+
},
92+
loadLocalEntry: (path) => {
93+
const lookup = FS.lookupPath(path);
94+
const node = lookup.node;
95+
const stat = FS.stat(path);
96+
97+
if (FS.isDir(stat.mode)) {
98+
return { timestamp: stat.mtime, mode: stat.mode };
99+
} else if (FS.isFile(stat.mode)) {
100+
node.contents = MEMFS.getFileDataAsTypedArray(node);
101+
return {
102+
timestamp: stat.mtime,
103+
mode: stat.mode,
104+
contents: node.contents,
105+
};
106+
} else {
107+
throw new Error('node type not supported');
108+
}
109+
},
110+
storeLocalEntry: (path, entry) => {
111+
if (FS.isDir(entry['mode'])) {
112+
FS.mkdirTree(path, entry['mode']);
113+
} else if (FS.isFile(entry['mode'])) {
114+
FS.writeFile(path, entry['contents'], { canOwn: true });
115+
} else {
116+
throw new Error('node type not supported');
117+
}
118+
119+
FS.chmod(path, entry['mode']);
120+
FS.utime(path, entry['timestamp'], entry['timestamp']);
121+
},
122+
removeLocalEntry: (path) => {
123+
var stat = FS.stat(path);
124+
125+
if (FS.isDir(stat.mode)) {
126+
FS.rmdir(path);
127+
} else if (FS.isFile(stat.mode)) {
128+
FS.unlink(path);
129+
}
130+
},
131+
loadRemoteEntry: async (handle) => {
132+
if (handle.kind === 'file') {
133+
const file = await handle.getFile();
134+
return {
135+
contents: new Uint8Array(await file.arrayBuffer()),
136+
mode: nativeFSAsync.FILE_MODE,
137+
timestamp: file.lastModifiedDate,
138+
};
139+
} else if (handle.kind === 'directory') {
140+
return {
141+
mode: nativeFSAsync.DIR_MODE,
142+
timestamp: new Date(),
143+
};
144+
} else {
145+
throw new Error('unknown kind: ' + handle.kind);
146+
}
147+
},
148+
storeRemoteEntry: async (handles, path, entry) => {
149+
const parentDirHandle = handles.get(PATH.dirname(path));
150+
const handle = FS.isFile(entry.mode)
151+
? await parentDirHandle.getFileHandle(PATH.basename(path), {
152+
create: true,
153+
})
154+
: await parentDirHandle.getDirectoryHandle(PATH.basename(path), {
155+
create: true,
156+
});
157+
if (handle.kind === 'file') {
158+
const writable = await handle.createWritable();
159+
await writable.write(entry.contents);
160+
await writable.close();
161+
}
162+
handles.set(path, handle);
163+
},
164+
removeRemoteEntry: async (handles, path) => {
165+
const parentDirHandle = handles.get(PATH.dirname(path));
166+
await parentDirHandle.removeEntry(PATH.basename(path));
167+
handles.delete(path);
168+
},
169+
reconcile: async (mount, src, dst) => {
170+
let total = 0;
171+
172+
const create = [];
173+
Object.keys(src.entries).forEach(function (key) {
174+
const e = src.entries[key];
175+
const e2 = dst.entries[key];
176+
if (
177+
!e2 ||
178+
(FS.isFile(e.mode) &&
179+
e['timestamp'].getTime() > e2['timestamp'].getTime())
180+
) {
181+
create.push(key);
182+
total++;
183+
}
184+
});
185+
// sort paths in ascending order so directory entries are created
186+
// before the files inside them
187+
create.sort();
188+
189+
const remove = [];
190+
Object.keys(dst.entries).forEach(function (key) {
191+
if (!src.entries[key]) {
192+
remove.push(key);
193+
total++;
194+
}
195+
});
196+
// sort paths in descending order so files are deleted before their
197+
// parent directories
198+
remove.sort().reverse();
199+
200+
if (!total) {
201+
return;
202+
}
203+
204+
const handles = src.type === 'remote' ? src.handles : dst.handles;
205+
206+
for (const path of create) {
207+
const relPath = PATH.normalize(
208+
path.replace(mount.mountpoint, '/'),
209+
).substring(1);
210+
if (dst.type === 'local') {
211+
const handle = handles.get(relPath);
212+
const entry = await nativeFSAsync.loadRemoteEntry(handle);
213+
nativeFSAsync.storeLocalEntry(path, entry);
214+
} else {
215+
const entry = nativeFSAsync.loadLocalEntry(path);
216+
await nativeFSAsync.storeRemoteEntry(handles, relPath, entry);
217+
}
218+
}
219+
220+
for (const path of remove) {
221+
if (dst.type === 'local') {
222+
nativeFSAsync.removeLocalEntry(path);
223+
} else {
224+
const relPath = PATH.normalize(
225+
path.replace(mount.mountpoint, '/'),
226+
).substring(1);
227+
await nativeFSAsync.removeRemoteEntry(handles, relPath);
228+
}
229+
}
230+
},
231+
};
232+
233+
module.FS.filesystems.NATIVEFS_ASYNC = nativeFSAsync;
234+
235+
function ensureMountPathExists(path) {
236+
if (FS.mkdirTree) FS.mkdirTree(path);
237+
else mkdirTree(FS, path);
238+
239+
const { node } = FS.lookupPath(path, {
240+
follow_mount: false,
241+
});
242+
243+
if (FS.isMountpoint(node)) {
244+
throw new Error(`path '${path}' is already a file system mount point`);
245+
}
246+
if (!FS.isDir(node.mode)) {
247+
throw new Error(`path '${path}' points to a file not a directory`);
248+
}
249+
// eslint-disable-next-line
250+
for (const _ in node.contents) {
251+
throw new Error(`directory '${path}' is not empty`);
252+
}
253+
}
254+
255+
return async function mountNativeFS(path, fileSystemHandle) {
256+
if (fileSystemHandle.constructor.name !== 'FileSystemDirectoryHandle') {
257+
throw new TypeError(
258+
'Expected argument \'fileSystemHandle\' to be a FileSystemDirectoryHandle',
259+
);
260+
}
261+
ensureMountPathExists(path);
262+
263+
FS.mount(
264+
FS.filesystems.NATIVEFS_ASYNC,
265+
{ fileSystemHandle },
266+
path,
267+
);
268+
269+
// sync native ==> browser
270+
await new Promise($ => FS.syncfs(true, $));
271+
272+
return {
273+
// sync browser ==> native
274+
syncfs: () => new Promise($ => FS.syncfs(false, $)),
275+
};
276+
};
277+
}
278+
279+
const getFsHandles = async (dirHandle) => {
280+
const handles = [];
281+
282+
async function collect(curDirHandle) {
283+
for await (const entry of curDirHandle.values()) {
284+
handles.push(entry);
285+
if (entry.kind === 'directory') {
286+
await collect(entry);
287+
}
288+
}
289+
}
290+
291+
await collect(dirHandle);
292+
293+
const result = new Map();
294+
result.set('.', dirHandle);
295+
for (const handle of handles) {
296+
const relativePath = (await dirHandle.resolve(handle)).join('/');
297+
result.set(relativePath, handle);
298+
}
299+
return result;
300+
};

esm/interpreter/_utils.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ const dirname = (path) => {
3232
return tree.join('/');
3333
};
3434

35-
const mkdirTree = (FS, path) => {
35+
export const mkdirTree = (FS, path) => {
3636
const current = [];
3737
for (const branch of path.split('/')) {
3838
if (branch === '.' || branch === '..') continue;

esm/interpreter/micropython.js

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import { absoluteURL } from '../utils.js';
77
import mip from '../python/mip.js';
88
import { zip } from '../3rd-party.js';
99

10+
import { initializeNativeFS } from './_nativefs.js';
11+
1012
const type = 'micropython';
1113

1214
// REQUIRES INTEGRATION TEST
@@ -46,6 +48,8 @@ export default {
4648
progress('Loaded packages');
4749
}
4850
progress('Loaded MicroPython');
51+
if (!interpreter.mountNativeFS)
52+
interpreter.mountNativeFS = initializeNativeFS(interpreter._module);
4953
return interpreter;
5054
},
5155
registerJSModule,

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,6 @@
9494
"to-json-callback": "^0.1.1"
9595
},
9696
"worker": {
97-
"blob": "sha256-1qaY0F5H5wUad62pU3B7we3i6Jd4rGL0+RxIwT/kfaI="
97+
"blob": "sha256-0wd916BW20JpkmHfpwIyLYYvtV2iGIgaef2TXDirEao="
9898
}
9999
}

test/fs/index.html

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<meta charset="utf-8">
5+
<meta name="viewport" content="width=device-width,initial-scale=1">
6+
</head>
7+
<body>
8+
<script type="module">
9+
import { define } from '../../dist/index.js';
10+
define('mpy', {
11+
interpreter: 'micropython',
12+
hooks: {
13+
main: {
14+
onReady(wrap, element) {
15+
const { interpreter: micropython } = wrap;
16+
mount.onclick = async () => {
17+
const options = { mode: "readwrite" };
18+
const dirHandle = await showDirectoryPicker(options);
19+
const permissionStatus = await dirHandle.requestPermission(options);
20+
if (permissionStatus !== "granted") {
21+
throw new Error("readwrite access to directory not granted");
22+
}
23+
24+
const nativefs = await micropython.mountNativeFS("/mount_dir", dirHandle);
25+
micropython.runPython(`
26+
import os
27+
print(os.listdir('/mount_dir'))
28+
`);
29+
};
30+
}
31+
}
32+
}
33+
});
34+
</script>
35+
<script type="mpy"></script>
36+
<button id="mount">mount fs</button>
37+
</body>
38+
</html>

0 commit comments

Comments
 (0)