Support for Automatic <py-env> with dependency + few tweaks to contributing and readme docs

.gitignore

@@ -1,3 +1,6 @@
+# OSx rubbish
+.DS_Store
+
 # Byte-compiled
 __pycache__/
 *.py[cod]
@@ -12,9 +15,11 @@ docs/_build/
 .venv
 env/
 venv/
+.idea
 
 # Poetry lock file
 poetry.lock
+.python-version
 
 # Unit test / coverage reports
 htmlcov/

CONTRIBUTING.md

@@ -1,5 +1,45 @@
 # Contribution guide for developers
 
+## Development
+
+### Setting up the environment
+
+To get started, you will need to setup the development environment. This goes
+through two main steps, namely [(1) Installing `pyscript-cli` dependencies](#install-pyscript-cli-dependencies)
+and [(2) Installing `pyscript-cli` Dev dependencies](#install-the-development-dependencies).
+
+### Install `pyscript-cli` dependencies
+
+`pyscript-cli` requires [`poetry`](https://python-poetry.org/) to manage dependencies
+and setup the environment.
+
+Therefore, the first thing to do is to make sure that you have `poetry` installed
+in your current Python environment. Please refer to the official `poetry`
+[documentation](https://python-poetry.org/docs/master/#installation) for installation
+instructions.
+
+To install `pyscript-cli` dependencies, you will need to run the following command from the project root:
+
+```shell
+poetry install
+```
+
+### Install the development dependencies
+
+There are a few extra dependencies that are solely required for development.
+To install these packages, you will need to run the following command from the project root:
+
+```shell
+poetry install --with dev-dependencies
+```
+
+Once all the dependencies are installed, you will only need to setup the git hooks
+via [`pre-commit`](https://pre-commit.com/):
+
+```shell
+pre-commit install
+```
+
 ## Documentation
 
 ### Install the documentation dependencies

README.md

@@ -21,12 +21,18 @@ $ pip install pyscript
 
 ## Usage
 
-### Embed a Python script into a PyScript HTML file
+### Embed Python code into a PyScript HTML file
 
 ```shell
 $ pyscript wrap <filename.py>
 ```
 
+Alternatively you could also use a **Jupyter notebook** as input file:
+
+```shell
+$ python wrap <filename.ipynb>
+```
+
 This will generate a file called `<filename.html>` by default.
 This can be overwritten with the `-o` or `--output` option:
 

pyproject.toml

@@ -1,7 +1,10 @@
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=1.1.0b3"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.isort]
+profile = "black"
+
 [tool.poetry]
 name = "pyscript"
 version = "0.2.4"
@@ -32,12 +35,17 @@ sphinx-autobuild = {version = "^2021.3.14", optional = true}
 sphinx-autodoc-typehints = {version = "^1.19.2", optional = true}
 myst-parser = {version = "^0.18.0", optional = true}
 pydata-sphinx-theme = {version = "^0.9.0", optional = true}
+requests = "^2.28.1"
+six = "^1.16.0"
+nbconvert = "^7.0.0"
+ipykernel = "^6.15.2"
 
 [tool.poetry.dev-dependencies]
 coverage = "^6.3.2"
 mypy = "^0.950"
 pytest = "^7.1.2"
 types-toml = "^0.10.8"
+pre-commit = "^2.20.0"
 
 [tool.poetry.extras]
 docs = [

src/pyscript/_generator.py

@@ -5,21 +5,65 @@
 import jinja2
 import toml
 
-_env = jinja2.Environment(loader=jinja2.PackageLoader("pyscript"))
+from ._node_parser import FinderResult, _convert_notebook, find_imports
 
 
-def string_to_html(input_str: str, title: str, output_path: Path) -> None:
+class UnsupportedFileType(Exception):
+    pass
+
+
+_env = jinja2.Environment(
+    loader=jinja2.PackageLoader("pyscript"), trim_blocks=True, lstrip_blocks=True
+)
+
+
+def string_to_html(
+    input_str: str, title: str, output_path: Path, pyenv: FinderResult = None
+) -> None:
     """Write a Python script string to an HTML file template."""
     template = _env.get_template("basic.html")
+    if pyenv is not None:
+        modules, paths = pyenv.packages, pyenv.paths
+    else:
+        modules = paths = ()
     with output_path.open("w") as fp:
-        fp.write(template.render(code=input_str, title=title))
+        fp.write(
+            template.render(code=input_str, title=title, modules=modules, paths=paths)
+        )
 
 
-def file_to_html(input_path: Path, title: str, output_path: Optional[Path]) -> None:
-    """Write a Python script string to an HTML file template."""
+def file_to_html(
+    input_path: Path, title: str, output_path: Optional[Path]
+) -> FinderResult:
+    """Write a Python script string to an HTML file template.
+
+    Warnings will be returned when scanning for environment, if any.
+    """
     output_path = output_path or input_path.with_suffix(".html")
-    with input_path.open("r") as fp:
-        string_to_html(fp.read(), title, output_path)
+
+    fname, extension = input_path.name, input_path.suffix
+    if extension == ".py":
+        with open(input_path, "rt") as f:
+            source = f.read()
+
+    elif extension == ".ipynb":
+        try:
+            import nbconvert  # noqa
+        except ImportError as e:  # pragma no cover
+            raise ImportError(
+                "Please install nbconvert to serve Jupyter Notebooks."
+            ) from e
+        else:
+            source = _convert_notebook(input_path)
+
+    else:
+        raise UnsupportedFileType(
+            "{} is neither a script (.py) nor a notebook (.ipynb)".format(fname)
+        )
+
+    import_results = find_imports(source, input_path)
+    string_to_html(source, title, output_path, pyenv=import_results)
+    return import_results
 
 
 def create_project(

src/pyscript/_node_parser.py

@@ -0,0 +1,194 @@
+"""
+ast-based parser to gather modules/package dependencies of a Python module.
+Code adapted from the find-imports project, currently in graveyard archive.
+"""
+from __future__ import annotations
+
+import ast
+import os
+import pkgutil
+from collections import defaultdict
+from itertools import chain, filterfalse
+from pathlib import Path
+
+from ._supported_packages import PACKAGE_RENAMES, PYODIDE_PACKAGES, STANDARD_LIBRARY
+
+
+class NamespaceInfo:
+    def __init__(self, source_fpath: Path) -> None:
+        # expanding base_folder to absolute as pkgutils.
+        # FileFinder will do so - easier for later purging
+        self.base_folder = str(source_fpath.parent.absolute())
+        self.source_mod_name = source_fpath.stem
+        self._collect()
+        # storing this as it will be useful for multiple lookups
+        self._all_namespace = set(chain(self.modules, self._packages))
+
+    def _collect(self):
+        iter_modules_paths = [self.base_folder]
+        for root, dirs, files in os.walk(self.base_folder):
+            for dirname in dirs:
+                iter_modules_paths.append(os.path.join(root, dirname))
+
+        # need to consume generator as I will iterate
+        # two times for _packages, and modules
+        pkg_mods = tuple(pkgutil.iter_modules(iter_modules_paths))
+        modules = map(
+            lambda mi: os.path.join(mi.module_finder.path, mi.name),
+            filterfalse(
+                lambda mi: mi.ispkg or mi.name == self.source_mod_name, pkg_mods
+            ),
+        )
+        _packages = map(
+            lambda mi: os.path.join(mi.module_finder.path, mi.name),
+            filter(lambda mi: mi.ispkg, pkg_mods),
+        )
+        self.modules = set(map(self._dotted_path, modules))
+        self._packages = set(map(self._dotted_path, _packages))
+
+    def _dotted_path(self, p: str):
+        p = p.replace(self.base_folder, "").replace(os.path.sep, ".")
+        if p.startswith("."):
+            p = p[1:]
+        return p
+
+    def __contains__(self, item: str) -> bool:
+        return item in self._all_namespace
+
+    def __str__(self) -> str:
+        return (
+            f"NameSpace info for {self.base_folder} \n\t "
+            f"Modules: {self.modules} \n\t Packages: {self._packages}"
+        )
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
+class FinderResult:
+    def __init__(self) -> None:
+        self._packages: set[str] = set()
+        self._locals: set[str] = set()
+        self._unsupported: defaultdict[str, set] = defaultdict(set)
+
+    def add_package(self, pkg_name: str) -> None:
+        self._packages.add(pkg_name)
+
+    def add_locals(self, pkg_name: str) -> None:
+        self._locals.add(pkg_name)
+
+    def add_unsupported_external_package(self, pkg_name: str) -> None:
+        self._unsupported["external"].add(pkg_name)
+
+    def add_unsupported_local_package(self, pkg_name: str) -> None:
+        self._unsupported["local"].add(pkg_name)
+
+    @property
+    def has_warnings(self):
+        return len(self._unsupported) > 0
+
+    @property
+    def unsupported_packages(self):
+        return self._unsupported["external"]
+
+    @property
+    def unsupported_paths(self):
+        return self._unsupported["local"]
+
+    @property
+    def packages(self):
+        return self._packages
+
+    @property
+    def paths(self):
+        pyenv_paths = map(
+            lambda l: "{}.py".format(l.replace(".", os.path.sep)), self._locals
+        )
+        return set(pyenv_paths)
+
+
+# https://stackoverflow.com/a/58847554
+class ModuleFinder(ast.NodeVisitor):
+    def __init__(self, context: NamespaceInfo, *args, **kwargs):
+        # list of all potential local imports
+        self.context = context
+        self.results = FinderResult()
+        super().__init__(*args, **kwargs)
+
+    def visit_Import(self, node):
+        for name in node.names:
+            # need to check for absolute module import here as they won't work in PyScript
+            # absolute package imports will be found later in _import_name
+            if len(name.name.split(".")) > 1 and name.name in self.context:
+                self.results.add_unsupported_local_package(name.name)
+            else:
+                self._import_name(name.name)
+
+    def visit_ImportFrom(self, node):
+        # if node.module is missing it's a "from . import ..." statement
+        # if level > 0 it's a "from .submodule import ..." statement
+        if node.module is not None:
+            self._import_name(node.module)
+
+    def _import_name(self, imported):
+        if imported in self.context:
+            if imported not in self.context._packages:
+                self.results.add_locals(imported)
+            else:
+                self.results.add_unsupported_local_package(imported)
+        else:
+            imported = imported.split(".")[0]
+            pkg_name = PACKAGE_RENAMES.get(imported, imported)
+            if pkg_name not in STANDARD_LIBRARY:
+                if pkg_name in PYODIDE_PACKAGES:
+                    self.results.add_package(pkg_name)
+                else:
+                    self.results.add_unsupported_external_package(pkg_name)
+
+
+def _find_modules(source: str, source_fpath: Path):
+    fname = source_fpath.name
+    # importing all local modules from source_fpath
+    namespace_info = NamespaceInfo(source_fpath=source_fpath)
+    # passing mode='exec' just in case defaults will change in the future
+    nodes = ast.parse(source, fname, mode="exec")
+
+    finder = ModuleFinder(context=namespace_info)
+    finder.visit(nodes)
+    return finder.results
+
+
+def _convert_notebook(source_fpath: Path) -> str:
+    from nbconvert import ScriptExporter
+
+    exporter = ScriptExporter()
+    source, _ = exporter.from_filename(source_fpath)
+
+    return source
+
+
+def find_imports(
+    source: str,
+    source_fpath: Path,
+) -> FinderResult:
+    """
+    Parse the input source, and returns its dependencies, as organised in
+    the sets of external _packages, and local modules, respectively.
+    Any modules or package with the same name found in the local
+
+    Parameters
+    ----------
+    source  : str
+        Python source code to parse
+    source_fpath : Path
+        Path to the input Python module to parse
+
+    Returns
+    -------
+    FinderResult
+        Return the results of parsing as a `FinderResult` instance.
+        This instance provides reference to packages and paths to
+        include in the py-env, as well as any unsupported import.
+    """
+
+    return _find_modules(source, source_fpath)