File system support (#36)

* readFile/writeFile WIP

* Added loadModule

* added watch module

* Tidied up new methods, docs for file system and watch modules

* Updated custom module docs

* Fixed broken docs link

Co-authored-by: James Ansley <[email protected]>

Author: elilambnz

.prettierrc

@@ -1,5 +1,6 @@
 {
   "tabWidth": 2,
   "singleQuote": true,
-  "semi": false
+  "semi": false,
+  "trailingComma": "none"
 }

src/hooks/usePython.ts

@@ -2,7 +2,7 @@ import { useContext, useEffect, useMemo, useRef, useState } from 'react'
 import {
   Packages,
   PythonContext,
-  suppressedMessages,
+  suppressedMessages
 } from '../providers/PythonProvider'
 import { proxy, Remote, wrap } from 'comlink'
 
@@ -14,6 +14,10 @@ interface Runner {
   ) => Promise<void>
   run: (code: string) => Promise<void>
   interruptExecution: () => void
+  readFile: (name: string) => void
+  writeFile: (name: string, data: string) => void
+  mkdir: (name: string) => void
+  rmdir: (name: string) => void
 }
 
 interface UsePythonProps {
@@ -31,12 +35,13 @@ export default function usePython(props?: UsePythonProps) {
   const [stderr, setStderr] = useState('')
   const [pendingCode, setPendingCode] = useState<string | undefined>()
   const [hasRun, setHasRun] = useState(false)
+  const [watchedModules, setWatchedModules] = useState<Set<string>>(new Set())
 
   const {
     packages: globalPackages,
     timeout,
     lazy,
-    terminateOnCompletion,
+    terminateOnCompletion
   } = useContext(PythonContext)
 
   const workerRef = useRef<Worker>()
@@ -65,14 +70,14 @@ export default function usePython(props?: UsePythonProps) {
     const official = [
       ...new Set([
         ...(globalPackages.official ?? []),
-        ...(packages.official ?? []),
-      ]),
+        ...(packages.official ?? [])
+      ])
     ]
     const micropip = [
       ...new Set([
         ...(globalPackages.micropip ?? []),
-        ...(packages.micropip ?? []),
-      ]),
+        ...(packages.micropip ?? [])
+      ])
     ]
     return [official, micropip]
   }, [globalPackages, packages])
@@ -165,6 +170,18 @@ def run(code, preamble=''):
         print()
 `
 
+  // prettier-ignore
+  const moduleReloadCode = (modules: Set<string>) => `
+import importlib
+import sys
+${Array.from(modules).map((name) => `
+if """${name}""" in sys.modules:
+    importlib.reload(sys.modules["""${name}"""])
+`).join('')}
+del importlib
+del sys
+`
+
   const runPython = async (code: string, preamble = '') => {
     // Clear stdout and stderr
     setStdout('')
@@ -201,6 +218,9 @@ def run(code, preamble=''):
           interruptExecution()
         }, timeout)
       }
+      if (watchedModules.size > 0) {
+        await runnerRef.current.run(moduleReloadCode(watchedModules))
+      }
       await runnerRef.current.run(code)
       // eslint-disable-next-line
     } catch (error: any) {
@@ -211,6 +231,32 @@ def run(code, preamble=''):
     }
   }
 
+  const readFile = (name: string) => {
+    return runnerRef.current?.readFile(name)
+  }
+
+  const writeFile = (name: string, data: string) => {
+    return runnerRef.current?.writeFile(name, data)
+  }
+
+  const mkdir = (name: string) => {
+    return runnerRef.current?.mkdir(name)
+  }
+
+  const rmdir = (name: string) => {
+    return runnerRef.current?.rmdir(name)
+  }
+
+  const watchModules = (moduleNames: string[]) => {
+    setWatchedModules((prev) => new Set([...prev, ...moduleNames]))
+  }
+
+  const unwatchModules = (moduleNames: string[]) => {
+    setWatchedModules(
+      (prev) => new Set([...prev].filter((e) => !moduleNames.includes(e)))
+    )
+  }
+
   const interruptExecution = () => {
     cleanup()
     setIsRunning(false)
@@ -236,5 +282,11 @@ def run(code, preamble=''):
     isLoading,
     isRunning,
     interruptExecution,
+    readFile,
+    writeFile,
+    watchModules,
+    unwatchModules,
+    mkdir,
+    rmdir
   }
 }

src/providers/PythonProvider.tsx

@@ -4,7 +4,7 @@ const PythonContext = createContext({
   packages: {} as Packages,
   timeout: 0,
   lazy: false,
-  terminateOnCompletion: false,
+  terminateOnCompletion: false
 })
 
 export const suppressedMessages = ['Python initialization complete']
@@ -28,7 +28,7 @@ function PythonProvider(props: PythonProviderProps) {
     packages = {},
     timeout = 0,
     lazy = false,
-    terminateOnCompletion = false,
+    terminateOnCompletion = false
   } = props
 
   return (
@@ -37,7 +37,7 @@ function PythonProvider(props: PythonProviderProps) {
         packages,
         timeout,
         lazy,
-        terminateOnCompletion,
+        terminateOnCompletion
       }}
       {...props}
     />

src/workers/python-worker.ts

@@ -5,6 +5,12 @@ interface Pyodide {
   runPythonAsync: (code: string) => Promise<void>
   loadPackage: (packages: string[]) => Promise<void>
   pyimport: (pkg: string) => micropip
+  FS: {
+    readFile: (name: string, options: unknown) => void
+    writeFile: (name: string, data: string, options: unknown) => void
+    mkdir: (name: string) => void
+    rmdir: (name: string) => void
+  }
 }
 
 interface micropip {
@@ -14,7 +20,7 @@ interface micropip {
 declare global {
   interface Window {
     loadPyodide: ({
-      stdout,
+      stdout
     }: {
       stdout: (msg: string) => void
     }) => Promise<Pyodide>
@@ -35,7 +41,7 @@ const python = {
     packages: string[][]
   ) {
     self.pyodide = await self.loadPyodide({
-      stdout: (msg: string) => stdout(msg),
+      stdout: (msg: string) => stdout(msg)
     })
     if (packages[0].length > 0) {
       await self.pyodide.loadPackage(packages[0])
@@ -50,6 +56,18 @@ const python = {
   async run(code: string) {
     await self.pyodide.runPythonAsync(code)
   },
+  readFile(name: string) {
+    return self.pyodide.FS.readFile(name, { encoding: 'utf8' })
+  },
+  writeFile(name: string, data: string) {
+    return self.pyodide.FS.writeFile(name, data, { encoding: 'utf8' })
+  },
+  mkdir(name: string) {
+    self.pyodide.FS.mkdir(name)
+  },
+  rmdir(name: string) {
+    self.pyodide.FS.rmdir(name)
+  }
 }
 
 expose(python)

website/docs/examples/custom-modules.mdx

@@ -0,0 +1,21 @@
+---
+sidebar_position: 5
+---
+
+import CustomModuleExample from '../../src/components/CustomModuleExample'
+
+# Custom Modules
+
+By default, Python modules are cached. If you intend to make changes to an imported module, it needs to be watched and reloaded.
+
+Create a module using the file system APIs or write the file using Python, [examples here](file-system).
+
+The following example shows this in action. To try it out:
+
+- Click `Write file` and then `Run`
+- Make a change to `utils.py` and click `Write file`, note that when you `Run` again, the imported module is unchanged
+- Click `Watch` and then observe the module has been reloaded when you `Run` again
+
+<CustomModuleExample />
+
+You find the source code for this example [here](https://github.com/elilambnz/react-py/blob/main/website/src/components/CustomModuleExample.tsx).

website/docs/examples/file-system.md

@@ -0,0 +1,53 @@
+---
+sidebar_position: 4
+---
+
+# File System
+
+Some internal Pyodide file system methods are exposed.
+
+For more info, see the [API reference docs](../introduction/api-reference).
+
+```tsx
+import { useEffect } from 'react'
+import { usePython } from 'react-py'
+
+function Codeblock() {
+  const { readFile, writeFile, mkdir, rmdir ... } = usePython()
+
+  function read() {
+    const file = readFile('/hello.txt')
+    console.log(file)
+  }
+
+  function write() {
+    const data = 'hello world!'
+    writeFile('/hello.txt', data)
+  }
+
+  function createDir() {
+    mkdir('lib')
+  }
+
+  function deleteDir() {
+    rmdir('cruft')
+  }
+
+  ...
+}
+```
+
+You can also use Python to read and write files:
+
+```python
+with open("hello.txt", "w") as f:
+  f.write('hello world!')
+
+with open('hello.txt') as f:
+  contents = f.read()
+  print(contents)
+```
+
+:::note
+Files are not shared between instances, each usage of the `usePython` hook has an independent file system. Files are also not persisted on page reload.
+:::

website/docs/examples/playground.md

@@ -8,5 +8,13 @@ draft: true
 Used for testing bugs etc.
 
 ```python
-print("hello", end="")
+with open("/hello.txt", "w") as fh:
+  fh.write("hello world!")
+  print("done")
+```
+
+```python
+with open("/hello.txt", "r") as fh:
+  data = fh.read()
+print(data)
 ```

website/docs/introduction/api-reference.md

@@ -63,6 +63,40 @@ True if code is being executed. False if idle.
 
 Can be called to immediately interrupt ongoing execution. Will terminate the running worker and spawn a new one.
 
+### readFile
+
+`(name: string) => void`
+
+Exposes `pyodide.FS.readFile`, encoding is `utf8`.
+
+### writeFile
+
+`(name: string, data: string) => void`
+
+Exposes `pyodide.FS.writeFile`, encoding is `utf8`.
+
+### mkdir
+
+`(name: string) => void`
+
+Exposes `pyodide.FS.mkdir`.
+
+### rmdir
+
+Exposes `pyodide.FS.rmdir`.
+
+### watchModules
+
+`(modules: string[]) => void`
+
+Adds modules to be reloaded before code is run.
+
+### unwatchModules
+
+`(modules: string[]) => void`
+
+Removes modules to be reloaded before code is run.
+
 ## Types
 
 ### Packages

website/src/components/CodeEditor.tsx

@@ -8,7 +8,7 @@ const editorOptions = {
   enableBasicAutocompletion: true,
   enableLiveAutocompletion: true,
   highlightActiveLine: false,
-  showPrintMargin: false,
+  showPrintMargin: false
 }
 
 const editorOnLoad = (editor) => {
@@ -38,7 +38,7 @@ export default function CodeEditor(props: CodeEditorProps) {
     stderr,
     isLoading,
     isRunning,
-    interruptExecution,
+    interruptExecution
   } = usePython()
 
   function run() {