Skip to content

Commit 99b6b4d

Browse files
authored
Load ANTLR files based on dynamic runtime version (#8)
* Load ANTLR files based on dynamic runtime version * Add missing files * Incorporated a couple of jakelishman PR feedbacks
1 parent 5ad0db0 commit 99b6b4d

File tree

13 files changed

+254
-48
lines changed

13 files changed

+254
-48
lines changed

.github/workflows/build-ast.yml

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Build Python Package
2+
3+
on:
4+
workflow_call:
5+
6+
defaults:
7+
run:
8+
working-directory: source/openpulse
9+
10+
jobs:
11+
build:
12+
name: Build wheels
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v3
16+
with:
17+
fetch-depth: 0
18+
19+
- uses: actions/setup-python@v3
20+
# This is pure Python, so it shouldn't matter what version we use to
21+
# build the sdist and wheel.
22+
23+
- name: Update pip
24+
run: pip install --upgrade pip
25+
26+
- uses: actions/setup-java@v2
27+
with:
28+
java-version: '15'
29+
distribution: 'adopt'
30+
31+
- name: Generate all ANTLR files
32+
run: |
33+
set -e
34+
35+
antlr_jar_dir="$PWD/.antlr_jars"
36+
antlr_out_dir="$PWD/openpulse/_antlr"
37+
mkdir -p "${antlr_jar_dir}"
38+
mkdir -p "${antlr_out_dir}"
39+
40+
# Parse the full ANTLR versions we need. The 'sed' strips out
41+
# comments from the file. We have to use `<<<` redirection rather
42+
# than pipelining to avoid running in a subshell, which would prevent
43+
# use from modifying the `antlr_versions` variable in the loop.
44+
declare -a antlr_versions
45+
while read -r line; do
46+
if [ -n "$line" ]; then
47+
antlr_versions+=("$line")
48+
fi
49+
done <<< $(sed 's/#.*//g' ANTLR_VERSIONS.txt);
50+
51+
# Download ANTLR.
52+
pushd "${antlr_jar_dir}"
53+
for version_string in "${antlr_versions[@]}"; do
54+
curl -LO "https://www.antlr.org/download/antlr-${version_string}-complete.jar"
55+
done
56+
popd
57+
58+
# Build the ANTLR files.
59+
pushd ${{ github.workspace }}/source/grammar
60+
for version_string in "${antlr_versions[@]}"; do
61+
echo "Handling version ${version_string}"
62+
IFS=. read -ra version <<< "${version_string}"
63+
out_dir="${antlr_out_dir}/_${version[0]}_${version[1]}"
64+
mkdir -p "$out_dir"
65+
java -Xmx500M -jar "${antlr_jar_dir}/antlr-${version_string}-complete.jar" -o "$out_dir" -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
66+
done
67+
popd
68+
69+
# Replace version requirements in setup.cfg.
70+
python tools/update_antlr_version_requirements.py setup.cfg ANTLR_VERSIONS.txt
71+
72+
- name: Install Python build dependencies
73+
run: pip install --upgrade build
74+
75+
- name: Build package
76+
run: python -m build --wheel --sdist .
77+
78+
- uses: actions/upload-artifact@v2
79+
with:
80+
name: openpulse-python-wheel
81+
path: ./source/openpulse/dist/*.whl
82+
if-no-files-found: error
83+
84+
- uses: actions/upload-artifact@v2
85+
with:
86+
name: openpulse-python-sdist
87+
path: ./source/openpulse/dist/*.tar.gz
88+
if-no-files-found: error

.github/workflows/tests-ast.yml

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,49 +4,43 @@ on:
44
[push, pull_request]
55

66
jobs:
7+
build:
8+
uses: ./.github/workflows/build-ast.yml
79
tests:
810
name: OpenPulse AST tests
11+
needs: build
912
runs-on: ubuntu-latest
1013
strategy:
1114
fail-fast: false
1215
matrix:
13-
python-version: ['3.7', '3.8', '3.9', '3.10']
14-
antlr-version: ['4.9.2']
16+
# Just using minimum and maximum to avoid exploding the matrix.
17+
python-version: ['3.7', '3.10']
18+
antlr-version: ['4.7', '4.11']
1519
defaults:
1620
run:
1721
working-directory: source/openpulse
1822

1923
steps:
20-
- uses: actions/checkout@v2
24+
- uses: actions/checkout@v3
2125
with:
22-
submodules: recursive
26+
fetch-depth: 0
2327

24-
- uses: actions/setup-python@v2
28+
- uses: actions/setup-python@v3
2529
with:
2630
python-version: ${{ matrix.python-version }}
2731

28-
- uses: actions/setup-java@v2
32+
- uses: actions/download-artifact@v3
2933
with:
30-
java-version: '15'
31-
distribution: 'adopt'
34+
name: openpulse-python-wheel
35+
path: ./source/openpulse/
3236

33-
- name: Update pip
34-
run: python -mpip install --upgrade pip
35-
36-
- name: Install ANTLR4
37-
working-directory: .
38-
run: curl -O https://www.antlr.org/download/antlr-${{ matrix.antlr-version }}-complete.jar
39-
40-
- name: Install ANTLR4 Python runtime
41-
run: python -mpip install antlr4-python3-runtime==${{ matrix.antlr-version }}
42-
43-
- name: Install Python dependencies
44-
working-directory: source/openpulse
45-
run: python -mpip install -r requirements.txt -r requirements-dev.txt
46-
47-
- name: Generate openpulse grammar
48-
working-directory: source/grammar
49-
run: java -Xmx500M -jar ../../antlr-${{ matrix.antlr-version }}-complete.jar -o ../openpulse/openpulse/antlr -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
37+
- name: Install package
38+
run: |
39+
set -e
40+
pip install --upgrade pip wheel
41+
pip install -r requirements-dev.txt
42+
pip install 'antlr4_python3_runtime==${{ matrix.antlr-version }}'
43+
pip install "$(echo openpulse-*.whl)[all]"
5044
5145
- name: Check openpulse format
5246
run: black --check --diff openpulse tests
@@ -60,4 +54,10 @@ jobs:
6054
run: pylint .
6155

6256
- name: Run openpulse tests
63-
run: pytest -vv --color=yes tests
57+
run: |
58+
# Swap into the testing directory so the imported `openpulse` is the
59+
# wheel version not the current-directory version. The
60+
# `--import-mode=importlib` stops pytest from modifying the path to
61+
# accidentally put the checked-out version (with no ANTLR) back.
62+
cd tests
63+
pytest -vv --color=yes --import-mode=importlib .

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ https://github.com/openqasm/openqasm/tree/main/source/grammar into the `source/g
1818
Now build the `openpulse` grammar. Change to the `source/grammar` directory and run:
1919

2020
```
21-
antlr4 -o ../openpulse/openpulse/antlr -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
21+
antlr4 -o ../openpulse/openpulse/_antlr/_<major>_<minor> -Dlanguage=Python3 -visitor openpulseLexer.g4 openpulseParser.g4
2222
```
2323

2424
Finally, you can change to the `source/openpulse` directory and run:

source/openpulse/.pylintrc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ enable=
3131
unnecessary-semicolon,
3232
unused-variable,
3333
wrong-import-order,
34-
wrong-import-position,
3534

3635
# Ignore long lines containing urls or pylint directives.
3736
ignore-long-lines=^(.*#\w*pylint: disable.*|\s*(# )?<?https?://\S+>?)$
3837

39-
# Ignore the ANTLR-generated files
40-
ignore-paths=^openpulse/antlr/.*$
38+
# Ignore errors caused by pylint being unable to statically resolve imports in
39+
# the `_antlr` module, which dynamically dispatches to a versioned subpackage.
40+
ignored-modules=_antlr

source/openpulse/ANTLR_VERSIONS.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
4.7.2
2+
4.8
3+
4.9.2
4+
4.10.1
5+
4.11.1

source/openpulse/MANIFEST.in

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ include requirements.txt
66
include openpulse/py.typed
77

88
# ANTLR-generated files.
9-
include openpulse/antlr/*.interp
10-
include openpulse/antlr/*.tokens
9+
graft openpulse/_antlr/_4_*/
10+
prune **/__pycache__/

source/openpulse/openpulse/__init__.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,8 @@
1414
the :obj:`~parser.parse` function.
1515
"""
1616

17-
__version__ = "0.2.5"
17+
__version__ = "0.4.0"
1818

1919
from . import ast
20-
21-
try:
22-
from . import parser
23-
from .parser import parse
24-
except ImportError:
25-
# Installed without the parsing extra.
26-
pass
20+
from . import parser
21+
from .parser import parse
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""ANTLR-generated files for parsing OpenPulse files.
2+
3+
This package sets up its import contents to be taken from the generated files
4+
whose ANTLR version matches the installed version of the ANTLR runtime. The
5+
generated files should be placed in directories called ``_<major>_<minor>``,
6+
where `major` is 4, and `minor` is the minor version of ANTLR (e.g. if ANTLR
7+
4.10 was used, those files should be in ``_4_10``).
8+
9+
The ANTLR files from more than one version of ANTLR can be present at once. This package will
10+
dynamically load the correct version based on the installed version of the runtime.
11+
"""
12+
13+
from importlib.abc import MetaPathFinder as _MetaPathFinder, Loader as _Loader
14+
import pathlib
15+
import sys
16+
17+
if sys.version_info < (3, 10):
18+
from importlib_metadata import version as _version
19+
else:
20+
from importlib.metadata import version as _version
21+
22+
# The `antlr4` package is supplied by `antlr4_python3_runtime`.
23+
_parts = [int(x) for x in _version("antlr4_python3_runtime").split(".")]
24+
_resolved_dir = f"_{_parts[0]}_{_parts[1]}"
25+
_antlr_dir = pathlib.Path(__file__).parent
26+
if not (_antlr_dir / _resolved_dir).is_dir():
27+
_available = [path.parent.name[1:] for path in _antlr_dir.rglob("openpulseParser.py")]
28+
if not _available:
29+
raise ImportError("No ANTLR-generated parsers found.")
30+
raise ImportError(
31+
f"Missing ANTLR-generated parser for version '{_parts[0]}.{_parts[1]}'."
32+
f" Available versions: {_available!r}"
33+
)
34+
35+
36+
class ANTLRMetaPathFinder(_MetaPathFinder):
37+
"""Redirect module/package lookups in `openpulse.antlr` to the concrete implementations
38+
pre-generated by the ANTLR version that matches the installed version of the runtime."""
39+
40+
def __init__(self, version_package: str):
41+
top_level = __package__.rsplit(".")[0]
42+
# Note the extra `.` in the domain because we don't want to handle ourselves.
43+
self._domain = f"{top_level}._antlr."
44+
self._versioned = f"{top_level}._antlr.{version_package}"
45+
46+
def find_spec(self, fullname, path, target=None):
47+
from importlib.machinery import SourceFileLoader
48+
from importlib.util import spec_from_loader, find_spec
49+
50+
if not fullname.startswith(self._domain) or fullname.startswith(self._versioned):
51+
return None
52+
newname = f"{self._versioned}.{fullname[len(self._domain):]}"
53+
# Get the spec and loader for the direct path to the versioned file, and rewrap them to have
54+
# the unversioned module name. The modules aren't loaded (or executed) by this, but the
55+
# loader is configured so that when they are, their scopes all carry the unversioned name.
56+
return spec_from_loader(fullname, SourceFileLoader(fullname, find_spec(newname).origin))
57+
58+
59+
sys.meta_path = [ANTLRMetaPathFinder(_resolved_dir)] + sys.meta_path
60+
61+
# ... and now the additional content of this module.
62+
63+
RUNTIME_VERSION = tuple(int(x) for x in _parts)
64+
"""The runtime-detected version of the ANTLR runtime, as a tuple like ``sys.version_info``."""
65+
66+
# These imports are re-directed into concrete versioned ones. Doing them
67+
# manually here helps stop pylint complaining.
68+
from . import openpulseLexer, openpulseParser, openpulseParserVisitor, openpulseParserListener

source/openpulse/openpulse/antlr/__init__.py

Whitespace-only changes.

source/openpulse/openpulse/parser.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
) from exc
3636

3737
import openpulse.ast as openpulse_ast
38-
from openqasm3.antlr.qasm3Parser import qasm3Parser
38+
from openqasm3._antlr.qasm3Parser import qasm3Parser
3939
from openqasm3 import ast
4040
from openqasm3.parser import (
4141
span,
@@ -45,9 +45,9 @@
4545
)
4646
from openqasm3.visitor import QASMVisitor
4747

48-
from .antlr.openpulseLexer import openpulseLexer
49-
from .antlr.openpulseParser import openpulseParser
50-
from .antlr.openpulseParserVisitor import openpulseParserVisitor
48+
from ._antlr.openpulseLexer import openpulseLexer
49+
from ._antlr.openpulseParser import openpulseParser
50+
from ._antlr.openpulseParserVisitor import openpulseParserVisitor
5151

5252

5353
def parse(input_: str) -> ast.Program:

0 commit comments

Comments
 (0)