Move common utility functions to a common library (#12773)

Author: srittau

.gitignore

@@ -8,12 +8,11 @@ __pycache__/
 # Distribution / packaging
 .Python
 /env/
-/build/
+/lib/build/
 /develop-eggs/
 /dist/
 /downloads/
 /eggs/
-/lib/
 /lib64/
 /parts/
 /sdist/

lib/pyproject.toml

@@ -0,0 +1 @@
+# Utilities for typeshed infrastructure scripts.

lib/ts_utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities for typeshed infrastructure scripts."""

tests/_metadata.py renamed to lib/ts_utils/metadata.py

@@ -19,7 +19,7 @@
 from packaging.specifiers import Specifier
 from packaging.version import Version
 
-from _utils import cache
+from .utils import cache
 
 __all__ = [
     "NoSuchStubError",

lib/ts_utils/py.typed

@@ -162,7 +162,7 @@ extra-standard-library = [
     "pyexpat",
     "zoneinfo",
 ]
-known-first-party = ["_metadata", "_utils"]
+known-first-party = ["ts_utils", "_utils"]
 
 [tool.typeshed]
 oldest_supported_python = "3.8"

tests/_utils.py renamed to lib/ts_utils/utils.py

@@ -2,9 +2,13 @@
     "$schema": "https://raw.githubusercontent.com/microsoft/pyright/main/packages/vscode-pyright/schemas/pyrightconfig.schema.json",
     "typeshedPath": ".",
     "include": [
+        "lib",
         "scripts",
         "tests",
     ],
+    "extraPaths": [
+        "lib",
+    ],
     "typeCheckingMode": "strict",
     // More of a lint. Unwanted for typeshed's own code.
     "reportImplicitStringConcatenation": "none",

pyproject.toml

@@ -22,3 +22,6 @@ tomli==2.0.2
 tomlkit==0.13.2
 typing_extensions>=4.12.0rc1
 uv==0.4.18
+
+# Utilities for typeshed infrastructure scripts.
+ts_utils @ file:lib

pyrightconfig.scripts_and_tests.json

@@ -312,7 +312,7 @@ async def get_github_repo_info(session: aiohttp.ClientSession, stub_info: StubIn
     Else, return None.
     """
     if stub_info.upstream_repository:
-        # We have various sanity checks for the upstream_repository field in tests/_metadata.py,
+        # We have various sanity checks for the upstream_repository field in ts_utils.metadata,
         # so no need to repeat all of them here
         split_url = urllib.parse.urlsplit(stub_info.upstream_repository)
         if split_url.netloc == "github.com":

requirements-tests.txt

@@ -12,8 +12,8 @@
 import sys
 from pathlib import Path
 
-from _metadata import read_metadata
-from _utils import (
+from ts_utils.metadata import read_metadata
+from ts_utils.utils import (
     REQS_FILE,
     STDLIB_PATH,
     TEST_CASES_DIR,

scripts/stubsabot.py

@@ -11,7 +11,7 @@
 
 from packaging.requirements import Requirement
 
-from _metadata import read_dependencies
+from ts_utils.metadata import read_dependencies
 
 distributions = sys.argv[1:]
 if not distributions:

tests/check_typeshed_structure.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
+
 import os
 import sys
 
-from _metadata import read_stubtest_settings
+from ts_utils.metadata import read_stubtest_settings
 
 platform = sys.platform
 distributions = sys.argv[1:]

tests/get_external_stub_requirements.py

@@ -22,8 +22,8 @@
 import tomli
 from packaging.requirements import Requirement
 
-from _metadata import PackageDependencies, get_recursive_requirements, read_metadata
-from _utils import (
+from ts_utils.metadata import PackageDependencies, get_recursive_requirements, read_metadata
+from ts_utils.utils import (
     PYTHON_VERSION,
     STDLIB_PATH,
     TESTS_DIR,
@@ -173,7 +173,7 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) ->
     with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
         data = tomli.load(f)
 
-    # TODO: This could be added to _metadata.py, but is currently unused
+    # TODO: This could be added to ts_utils.metadata, but is currently unused
     mypy_tests_conf: dict[str, dict[str, Any]] = data.get("mypy-tests", {})
     if not mypy_tests_conf:
         return

tests/get_stubtest_system_requirements.py

@@ -6,7 +6,7 @@
 import sys
 from pathlib import Path
 
-from _utils import parse_requirements, print_command
+from ts_utils.utils import parse_requirements, print_command
 
 _WELL_KNOWN_FILE = Path("tests", "pyright_test.py")
 

tests/mypy_test.py

@@ -35,8 +35,8 @@
 from pytype import config as pytype_config, load_pytd  # type: ignore[import]
 from pytype.imports import typeshed  # type: ignore[import]
 
-from _metadata import read_dependencies
-from _utils import SupportedVersionsDict, parse_stdlib_versions_file, supported_versions_for_module
+from ts_utils.metadata import read_dependencies
+from ts_utils.utils import SupportedVersionsDict, parse_stdlib_versions_file, supported_versions_for_module
 
 TYPESHED_SUBDIRS = ["stdlib", "stubs"]
 TYPESHED_HOME = "TYPESHED_HOME"

tests/pyright_test.py

@@ -21,8 +21,8 @@
 from pathlib import Path
 from typing_extensions import TypeAlias
 
-from _metadata import get_recursive_requirements, read_metadata
-from _utils import (
+from ts_utils.metadata import get_recursive_requirements, read_metadata
+from ts_utils.utils import (
     PYTHON_VERSION,
     TEST_CASES_DIR,
     DistributionTests,

tests/pytype_test.py

@@ -9,17 +9,8 @@
 import sys
 from importlib.util import find_spec
 from pathlib import Path
-from typing import Any
-
-from _utils import TEST_CASES_DIR, test_cases_path
-
-try:
-    from termcolor import colored  # pyright: ignore[reportAssignmentType]
-except ImportError:
-
-    def colored(text: str, color: str | None = None, **kwargs: Any) -> str:  # type: ignore[misc]
-        return text
 
+from ts_utils.utils import TEST_CASES_DIR, colored, test_cases_path
 
 _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
 _TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json"

tests/regr_test.py

@@ -13,7 +13,7 @@
 import sys
 from pathlib import Path
 
-from _utils import allowlist_stubtest_arguments, allowlists_path
+from ts_utils.utils import allowlist_stubtest_arguments, allowlists_path
 
 
 def run_stubtest(typeshed_dir: Path) -> int:

tests/runtests.py

@@ -13,8 +13,8 @@
 from textwrap import dedent
 from typing import NoReturn
 
-from _metadata import NoSuchStubError, get_recursive_requirements, read_metadata
-from _utils import (
+from ts_utils.metadata import NoSuchStubError, get_recursive_requirements, read_metadata
+from ts_utils.utils import (
     PYTHON_VERSION,
     allowlist_stubtest_arguments,
     allowlists_path,

tests/stubtest_stdlib.py

@@ -9,7 +9,7 @@
 from itertools import product
 from typing_extensions import TypeAlias
 
-from _utils import colored, print_error
+from ts_utils.utils import colored, print_error
 
 ReturnCode: TypeAlias = int