Load ANTLR files based on dynamic runtime version (#8)

aspcompiler · web-flow · commit 99b6b4d5e0fe · 2022-10-18T13:44:31.000-07:00
* Load ANTLR files based on dynamic runtime version

* Add missing files

* Incorporated a couple of jakelishman PR feedbacks
diff --git a/.github/workflows/build-ast.yml b/.github/workflows/build-ast.yml
@@ -0,0 +1,88 @@
+name: Build Python Package
+
+on:
+  workflow_call:
+
+defaults:
+  run:
+    working-directory: source/openpulse
+
+jobs:
+  build:
+    name: Build wheels
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v3
+        # This is pure Python, so it shouldn't matter what version we use to
+        # build the sdist and wheel.
+
+      - name: Update pip
+        run: pip install --upgrade pip
+
+      - uses: actions/setup-java@v2
+        with:
+          java-version: '15'
+          distribution: 'adopt'
+
+      - name: Generate all ANTLR files
+        run: |
+          set -e
+
+          antlr_jar_dir="$PWD/.antlr_jars"
+          antlr_out_dir="$PWD/openpulse/_antlr"
+          mkdir -p "${antlr_jar_dir}"
+          mkdir -p "${antlr_out_dir}"
+
+          # Parse the full ANTLR versions we need.  The 'sed' strips out
+          # comments from the file.  We have to use `<<<` redirection rather
+          # than pipelining to avoid running in a subshell, which would prevent
+          # use from modifying the `antlr_versions` variable in the loop.
+          declare -a antlr_versions
+          while read -r line; do
+            if [ -n "$line" ]; then
+              antlr_versions+=("$line")
+            fi
+          done <<< $(sed 's/#.*//g' ANTLR_VERSIONS.txt);
+
+          # Download ANTLR.
+          pushd "${antlr_jar_dir}"
+          for version_string in "${antlr_versions[@]}"; do
+            curl -LO "https://www.antlr.org/download/antlr-${version_string}-complete.jar"
+          done
+          popd
+
+          # Build the ANTLR files.
+          pushd ${{ github.workspace }}/source/grammar
+          for version_string in "${antlr_versions[@]}"; do
+            echo "Handling version ${version_string}"
+            IFS=. read -ra version <<< "${version_string}"
+            out_dir="${antlr_out_dir}/_${version[0]}_${version[1]}"
+            mkdir -p "$out_dir"
+            java -Xmx500M -jar "${antlr_jar_dir}/antlr-${version_string}-complete.jar" -o "$out_dir" -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
+          done
+          popd
+
+          # Replace version requirements in setup.cfg.
+          python tools/update_antlr_version_requirements.py setup.cfg ANTLR_VERSIONS.txt
+
+      - name: Install Python build dependencies
+        run: pip install --upgrade build
+
+      - name: Build package
+        run: python -m build --wheel --sdist .
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: openpulse-python-wheel
+          path: ./source/openpulse/dist/*.whl
+          if-no-files-found: error
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: openpulse-python-sdist
+          path: ./source/openpulse/dist/*.tar.gz
+          if-no-files-found: error
diff --git a/.github/workflows/tests-ast.yml b/.github/workflows/tests-ast.yml
@@ -4,49 +4,43 @@ on:
   [push, pull_request]
 
 jobs:
+  build:
+      uses: ./.github/workflows/build-ast.yml
   tests:
     name: OpenPulse AST tests
+    needs: build
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
-        antlr-version: ['4.9.2']
+        # Just using minimum and maximum to avoid exploding the matrix.
+        python-version: ['3.7', '3.10']
+        antlr-version: ['4.7', '4.11']
     defaults:
       run:
         working-directory: source/openpulse
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          submodules: recursive
+          fetch-depth: 0
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
 
-      - uses: actions/setup-java@v2
+      - uses: actions/download-artifact@v3
         with:
-          java-version: '15'
-          distribution: 'adopt'
+          name: openpulse-python-wheel
+          path: ./source/openpulse/
 
-      - name: Update pip
-        run: python -mpip install --upgrade pip
-
-      - name: Install ANTLR4
-        working-directory: .
-        run: curl -O https://www.antlr.org/download/antlr-${{ matrix.antlr-version }}-complete.jar
-
-      - name: Install ANTLR4 Python runtime
-        run: python -mpip install antlr4-python3-runtime==${{ matrix.antlr-version }}
-
-      - name: Install Python dependencies
-        working-directory: source/openpulse
-        run: python -mpip install -r requirements.txt -r requirements-dev.txt
-
-      - name: Generate openpulse grammar
-        working-directory: source/grammar
-        run: java -Xmx500M -jar ../../antlr-${{ matrix.antlr-version }}-complete.jar -o ../openpulse/openpulse/antlr -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
+      - name: Install package
+        run: |
+          set -e
+          pip install --upgrade pip wheel
+          pip install -r requirements-dev.txt
+          pip install 'antlr4_python3_runtime==${{ matrix.antlr-version }}'
+          pip install "$(echo openpulse-*.whl)[all]"
 
       - name: Check openpulse format
         run: black --check --diff openpulse tests
@@ -60,4 +54,10 @@ jobs:
         run: pylint .
 
       - name: Run openpulse tests
-        run: pytest -vv --color=yes tests
+        run: |
+          # Swap into the testing directory so the imported `openpulse` is the
+          # wheel version not the current-directory version.  The
+          # `--import-mode=importlib` stops pytest from modifying the path to
+          # accidentally put the checked-out version (with no ANTLR) back.
+          cd tests
+          pytest -vv --color=yes --import-mode=importlib .
diff --git a/README.md b/README.md
@@ -18,7 +18,7 @@ https://github.com/openqasm/openqasm/tree/main/source/grammar into the `source/g
 Now build the `openpulse` grammar. Change to the `source/grammar` directory and run:
 
 ```
-antlr4 -o ../openpulse/openpulse/antlr -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
+antlr4 -o ../openpulse/openpulse/_antlr/_<major>_<minor> -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
 ```
 
 Finally, you can change to the `source/openpulse` directory and run:
diff --git a/source/openpulse/.pylintrc b/source/openpulse/.pylintrc
@@ -31,10 +31,10 @@ enable=
     unnecessary-semicolon,
     unused-variable,
     wrong-import-order,
-    wrong-import-position,
 
 # Ignore long lines containing urls or pylint directives.
 ignore-long-lines=^(.*#\w*pylint: disable.*|\s*(# )?<?https?://\S+>?)$
 
-# Ignore the ANTLR-generated files
-ignore-paths=^openpulse/antlr/.*$
+# Ignore errors caused by pylint being unable to statically resolve imports in
+# the `_antlr` module, which dynamically dispatches to a versioned subpackage.
+ignored-modules=_antlr
diff --git a/source/openpulse/ANTLR_VERSIONS.txt b/source/openpulse/ANTLR_VERSIONS.txt
@@ -0,0 +1,5 @@
+4.7.2
+4.8
+4.9.2
+4.10.1
+4.11.1
diff --git a/source/openpulse/MANIFEST.in b/source/openpulse/MANIFEST.in
@@ -6,5 +6,5 @@ include requirements.txt
 include openpulse/py.typed
 
 # ANTLR-generated files.
-include openpulse/antlr/*.interp
-include openpulse/antlr/*.tokens
+graft openpulse/_antlr/_4_*/
+prune **/__pycache__/
diff --git a/source/openpulse/openpulse/__init__.py b/source/openpulse/openpulse/__init__.py
@@ -14,13 +14,8 @@
 the :obj:`~parser.parse` function.
 """
 
-__version__ = "0.2.5"
+__version__ = "0.4.0"
 
 from . import ast
-
-try:
-    from . import parser
-    from .parser import parse
-except ImportError:
-    # Installed without the parsing extra.
-    pass
+from . import parser
+from .parser import parse
diff --git a/source/openpulse/openpulse/_antlr/__init__.py b/source/openpulse/openpulse/_antlr/__init__.py
@@ -0,0 +1,68 @@
+"""ANTLR-generated files for parsing OpenPulse files.
+
+This package sets up its import contents to be taken from the generated files
+whose ANTLR version matches the installed version of the ANTLR runtime.  The
+generated files should be placed in directories called ``_<major>_<minor>``,
+where `major` is 4, and `minor` is the minor version of ANTLR (e.g. if ANTLR
+4.10 was used, those files should be in ``_4_10``).
+
+The ANTLR files from more than one version of ANTLR can be present at once.  This package will
+dynamically load the correct version based on the installed version of the runtime.
+"""
+
+from importlib.abc import MetaPathFinder as _MetaPathFinder, Loader as _Loader
+import pathlib
+import sys
+
+if sys.version_info < (3, 10):
+    from importlib_metadata import version as _version
+else:
+    from importlib.metadata import version as _version
+
+# The `antlr4` package is supplied by `antlr4_python3_runtime`.
+_parts = [int(x) for x in _version("antlr4_python3_runtime").split(".")]
+_resolved_dir = f"_{_parts[0]}_{_parts[1]}"
+_antlr_dir = pathlib.Path(__file__).parent
+if not (_antlr_dir / _resolved_dir).is_dir():
+    _available = [path.parent.name[1:] for path in _antlr_dir.rglob("openpulseParser.py")]
+    if not _available:
+        raise ImportError("No ANTLR-generated parsers found.")
+    raise ImportError(
+        f"Missing ANTLR-generated parser for version '{_parts[0]}.{_parts[1]}'."
+        f" Available versions: {_available!r}"
+    )
+
+
+class ANTLRMetaPathFinder(_MetaPathFinder):
+    """Redirect module/package lookups in `openpulse.antlr` to the concrete implementations
+    pre-generated by the ANTLR version that matches the installed version of the runtime."""
+
+    def __init__(self, version_package: str):
+        top_level = __package__.rsplit(".")[0]
+        # Note the extra `.` in the domain because we don't want to handle ourselves.
+        self._domain = f"{top_level}._antlr."
+        self._versioned = f"{top_level}._antlr.{version_package}"
+
+    def find_spec(self, fullname, path, target=None):
+        from importlib.machinery import SourceFileLoader
+        from importlib.util import spec_from_loader, find_spec
+
+        if not fullname.startswith(self._domain) or fullname.startswith(self._versioned):
+            return None
+        newname = f"{self._versioned}.{fullname[len(self._domain):]}"
+        # Get the spec and loader for the direct path to the versioned file, and rewrap them to have
+        # the unversioned module name.  The modules aren't loaded (or executed) by this, but the
+        # loader is configured so that when they are, their scopes all carry the unversioned name.
+        return spec_from_loader(fullname, SourceFileLoader(fullname, find_spec(newname).origin))
+
+
+sys.meta_path = [ANTLRMetaPathFinder(_resolved_dir)] + sys.meta_path
+
+# ... and now the additional content of this module.
+
+RUNTIME_VERSION = tuple(int(x) for x in _parts)
+"""The runtime-detected version of the ANTLR runtime, as a tuple like ``sys.version_info``."""
+
+# These imports are re-directed into concrete versioned ones.  Doing them
+# manually here helps stop pylint complaining.
+from . import openpulseLexer, openpulseParser, openpulseParserVisitor, openpulseParserListener
diff --git a/source/openpulse/openpulse/antlr/__init__.py b/source/openpulse/openpulse/antlr/__init__.py
diff --git a/source/openpulse/openpulse/parser.py b/source/openpulse/openpulse/parser.py
@@ -35,7 +35,7 @@
     ) from exc
 
 import openpulse.ast as openpulse_ast
-from openqasm3.antlr.qasm3Parser import qasm3Parser
+from openqasm3._antlr.qasm3Parser import qasm3Parser
 from openqasm3 import ast
 from openqasm3.parser import (
     span,
@@ -45,9 +45,9 @@
 )
 from openqasm3.visitor import QASMVisitor
 
-from .antlr.openpulseLexer import openpulseLexer
-from .antlr.openpulseParser import openpulseParser
-from .antlr.openpulseParserVisitor import openpulseParserVisitor
+from ._antlr.openpulseLexer import openpulseLexer
+from ._antlr.openpulseParser import openpulseParser
+from ._antlr.openpulseParserVisitor import openpulseParserVisitor
 
 
 def parse(input_: str) -> ast.Program:
diff --git a/source/openpulse/requirements.txt b/source/openpulse/requirements.txt
@@ -1,2 +1,2 @@
-antlr4-python3-runtime==4.9.2
-openqasm3==0.3.0
+antlr4-python3-runtime
+openqasm3==0.4.0
diff --git a/source/openpulse/setup.cfg b/source/openpulse/setup.cfg
@@ -25,8 +25,9 @@ python_requires = '>=3.7'
 packages = find:
 include_package_data = True
 install_requires =
-    antlr4-python3-runtime==4.9.2
-    openqasm3==0.3.0
+    antlr4-python3-runtime # __ANTLR_VERSIONS__
+    importlib_metadata; python_version<'3.10'
+    openqasm3[parser]==0.4.0
 
 [options.packages.find]
 exclude = tests*
diff --git a/source/openpulse/tools/update_antlr_version_requirements.py b/source/openpulse/tools/update_antlr_version_requirements.py
@@ -0,0 +1,49 @@
+import sys
+import re
+
+
+def parse_versions():
+    with open(sys.argv[2], "r") as version_file:
+        for line in version_file:
+            comment_start = line.find("#")
+            if comment_start >= 0:
+                line = line[: line.find("#")]
+            line = line.strip()
+            if not line:
+                continue
+            version_parts = tuple(int(x) for x in line.split("."))
+            yield version_parts[0], version_parts[1]
+
+
+def main():
+    if len(sys.argv) != 3:
+        print(f"Usage: {__file__} path/to/setup.cfg path/to/antlr_versions.txt", file=sys.stderr)
+        sys.exit(1)
+
+    versions = sorted(parse_versions())
+    if not versions:
+        print("didn't receive any versions to support", file=sys.stderr)
+        sys.exit(2)
+    if versions[0][0] != 4 or versions[-1][0] != 4:
+        print("can only handle ANTLR 4", file=sys.stderr)
+        sys.exit(4)
+    min_minor = versions[0][1]
+    max_minor = versions[-1][1]
+    if {minor for _, minor in versions} != set(range(min_minor, max_minor + 1)):
+        print("supplied minor versions must be continuous", file=sys.stderr)
+        sys.exit(8)
+    constraint = f"antlr4_python3_runtime>=4.{min_minor},<4.{max_minor + 1}"
+    with open(sys.argv[1], "r") as setup_file:
+        contents = setup_file.read()
+    new_contents, count = re.subn(
+        r"antlr4[-_]python3[-_]runtime\s*#\s*__ANTLR_VERSIONS__", constraint, contents
+    )
+    if not count:
+        print("given setup.cfg file did not seem to contain an antlr4 dependency", file=sys.stderr)
+        sys.exit(16)
+    with open(sys.argv[1], "w") as setup_file:
+        setup_file.write(new_contents)
+
+
+if __name__ == "__main__":
+    main()

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +4.7.2
 +4.8
 +4.9.2
 +4.10.1
 +4.11.1