Use eval_type_backport on Python 3.9 if it's installed to resolve int | None etc. (#773)

* Use eval_type_backport on Python 3.9 if it's installed to resolve `int | None` etc.

This uses the same module that pydantic does, and it allows people to use the
new pipe syntax if they have to support Python3.9 too -- very useful for
libraries.

(Also it works better with many type checkers which seem to mistakenly think
that with `from __future__ import annotations` means `int| None` will work,
but it doesn't out of the box.)

---------

Co-authored-by: Jim Crist-Harif <[email protected]>

Author: ashb

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Build msgspec and install dependencies
         run: |
-          pip install coverage -e ".[test]"
+          pip install -e ".[dev]"
 
       - name: Run pre-commit hooks
         uses: pre-commit/[email protected]
@@ -78,7 +78,7 @@ jobs:
         os: [ubuntu-latest, macos-13, windows-latest]
 
     env:
-      CIBW_TEST_REQUIRES: "pytest msgpack pyyaml tomli tomli_w"
+      CIBW_TEST_EXTRAS: "test"
       CIBW_TEST_COMMAND: "pytest {project}/tests"
       CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*"
       CIBW_SKIP: "*-win32 *_i686 *_s390x *_ppc64le"
@@ -102,7 +102,7 @@ jobs:
             echo "CIBW_SKIP=${CIBW_SKIP} *-musllinux_* cp39-*_aarch64 cp311-*_aarch64 cp312-*_aarch64 cp313-*_aarch64" >> $GITHUB_ENV
 
       - name: Build & Test Wheels
-        uses: pypa/cibuildwheel@v2.21.3
+        uses: pypa/cibuildwheel@v2.22.0
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4

msgspec/_utils.py

@@ -45,6 +45,28 @@ def _forward_ref(value):
 
     def _eval_type(t, globalns, localns):
         return typing._eval_type(t, globalns, localns, ())
+elif sys.version_info < (3, 10):
+
+    def _eval_type(t, globalns, localns):
+        try:
+            return typing._eval_type(t, globalns, localns)
+        except TypeError as e:
+            try:
+                from eval_type_backport import eval_type_backport
+            except ImportError:
+                raise TypeError(
+                    f"Unable to evaluate type annotation {t.__forward_arg__!r}. If you are making use "
+                    "of the new typing syntax (unions using `|` since Python 3.10 or builtins subscripting "
+                    "since Python 3.9), you should either replace the use of new syntax with the existing "
+                    "`typing` constructs or install the `eval_type_backport` package."
+                ) from e
+
+            return eval_type_backport(
+                t,
+                globalns,
+                localns,
+                try_default=False,
+            )
 else:
     _eval_type = typing._eval_type
 

setup.py

@@ -51,8 +51,15 @@
 yaml_deps = ["pyyaml"]
 toml_deps = ['tomli ; python_version < "3.11"', "tomli_w"]
 doc_deps = ["sphinx", "furo", "sphinx-copybutton", "sphinx-design", "ipython"]
-test_deps = ["pytest", "mypy", "pyright", "msgpack", "attrs", *yaml_deps, *toml_deps]
-dev_deps = ["pre-commit", "coverage", "gcovr", *doc_deps, *test_deps]
+test_deps = [
+    "pytest",
+    "msgpack",
+    "attrs",
+    'eval-type-backport ; python_version < "3.10"',
+    *yaml_deps,
+    *toml_deps,
+]
+dev_deps = ["pre-commit", "coverage", "mypy", "pyright", *doc_deps, *test_deps]
 
 extras_require = {
     "yaml": yaml_deps,

tests/test_utils.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
-from typing import Generic, List, Set, TypeVar
+import sys
+from typing import Generic, List, Optional, Set, TypeVar
 
 import pytest
-from utils import temp_module
+from utils import temp_module, package_not_installed
 
 from msgspec._utils import get_class_annotations
 
+PY310 = sys.version_info[:2] >= (3, 10)
+
 T = TypeVar("T")
 S = TypeVar("S")
 U = TypeVar("U")
@@ -201,3 +204,23 @@ class Sub(Base[Invalid]):
             pass
 
         assert get_class_annotations(Sub) == {"x": Invalid}
+
+    @pytest.mark.skipif(PY310, reason="<3.10 only")
+    def test_union_backport_not_installed(self):
+        class Ex:
+            x: int | None = None
+
+        with package_not_installed("eval_type_backport"):
+            with pytest.raises(
+                TypeError, match=r"or install the `eval_type_backport` package."
+            ):
+                get_class_annotations(Ex)
+
+    @pytest.mark.skipif(PY310, reason="<3.10 only")
+    def test_union_backport_installed(self):
+        class Ex:
+            x: int | None = None
+
+        pytest.importorskip("eval_type_backport")
+
+        assert get_class_annotations(Ex) == {"x": Optional[int]}

tests/utils.py

@@ -45,3 +45,16 @@ def max_call_depth(n):
         yield
     finally:
         sys.setrecursionlimit(orig)
+
+
+@contextmanager
+def package_not_installed(name):
+    try:
+        orig = sys.modules.get(name)
+        sys.modules[name] = None
+        yield
+    finally:
+        if orig is not None:
+            sys.modules[name] = orig
+        else:
+            del sys.modules[name]