Skip to content

Install build dependencies in-process #13450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: main
Choose a base branch
from
388 changes: 303 additions & 85 deletions src/pip/_internal/build_env.py

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions src/pip/_internal/cli/cmdoptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -1068,6 +1068,7 @@ def check_list_path_option(options: Values) -> None:
default=[],
choices=[
"fast-deps",
"inprocess-build-deps",
]
+ ALWAYS_ENABLED_FEATURES,
help="Enable new functionality, that may be backward incompatible.",
Expand Down
18 changes: 18 additions & 0 deletions src/pip/_internal/cli/req_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@
from optparse import Values
from typing import Any

from pip._internal.build_env import (
BuildEnvironmentInstaller,
InprocessBuildEnvironmentInstaller,
SubprocessBuildEnvironmentInstaller,
)
from pip._internal.cache import WheelCache
from pip._internal.cli import cmdoptions
from pip._internal.cli.index_command import IndexGroupCommand
Expand Down Expand Up @@ -131,11 +136,24 @@ def make_requirement_preparer(
"fast-deps has no effect when used with the legacy resolver."
)

build_isolation_installer: BuildEnvironmentInstaller
if "inprocess-build-deps" in options.features_enabled:
if resolver_variant == "legacy":
raise CommandError(
"inprocess-build-deps cannot be used with the legacy resolver."
)
build_isolation_installer = InprocessBuildEnvironmentInstaller(
finder, session, build_tracker, temp_build_dir_path, verbosity, options
)
else:
build_isolation_installer = SubprocessBuildEnvironmentInstaller(finder)

return RequirementPreparer(
build_dir=temp_build_dir_path,
src_dir=options.src_dir,
download_dir=download_dir,
build_isolation=options.build_isolation,
build_isolation_installer=build_isolation_installer,
check_build_deps=options.check_build_deps,
build_tracker=build_tracker,
session=session,
Expand Down
77 changes: 73 additions & 4 deletions src/pip/_internal/cli/spinners.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,26 @@
import sys
import time
from collections.abc import Generator
from typing import IO
from typing import IO, Final

from pip._vendor.rich.console import (
Console,
ConsoleOptions,
RenderableType,
RenderResult,
)
from pip._vendor.rich.live import Live
from pip._vendor.rich.measure import Measurement
from pip._vendor.rich.text import Text

from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.logging import get_indentation
from pip._internal.utils.logging import get_console, get_indentation

logger = logging.getLogger(__name__)

SPINNER_CHARS: Final = r"-\|/"
SPINS_PER_SECOND: Final = 8


class SpinnerInterface:
def spin(self) -> None:
Expand All @@ -27,9 +40,9 @@ def __init__(
self,
message: str,
file: IO[str] | None = None,
spin_chars: str = "-\\|/",
spin_chars: str = SPINNER_CHARS,
# Empirically, 8 updates/second looks nice
min_update_interval_seconds: float = 0.125,
min_update_interval_seconds: float = 1 / SPINS_PER_SECOND,
):
self._message = message
if file is None:
Expand Down Expand Up @@ -139,6 +152,62 @@ def open_spinner(message: str) -> Generator[SpinnerInterface, None, None]:
spinner.finish("done")


class PipRichSpinner:
"""
Custom rich spinner that matches the style and API* of the legacy spinners.

(*) Updates will be handled in a background thread by a rich live panel
which will call render() automatically at the appropriate time.
"""

def __init__(self, label: str) -> None:
self.label = label
self._spin_cycle = itertools.cycle(SPINNER_CHARS)
self._spinner_text = ""
self._finished = False
self._indent = get_indentation() * " "

def __rich_console__(
self, console: Console, options: ConsoleOptions
) -> RenderResult:
yield self.render()

def __rich_measure__(
self, console: Console, options: ConsoleOptions
) -> Measurement:
text = self.render()
return Measurement.get(console, options, text)

def render(self) -> RenderableType:
# The wrapping rich Live instance will call this method at the
# appropriate interval.
if not self._finished:
self._spinner_text = next(self._spin_cycle)

return Text.assemble(self._indent, self.label, " ... ", self._spinner_text)

def finish(self, status: str) -> None:
"""Stop spinning and set a final status message."""
self._spinner_text = status
self._finished = True


@contextlib.contextmanager
def open_rich_spinner(label: str) -> Generator[None, None, None]:
spinner = PipRichSpinner(label)
with Live(spinner, refresh_per_second=SPINS_PER_SECOND, console=get_console()):
try:
yield
except KeyboardInterrupt:
spinner.finish("canceled")
raise
except Exception:
spinner.finish("error")
raise
else:
spinner.finish("done")


HIDE_CURSOR = "\x1b[?25l"
SHOW_CURSOR = "\x1b[?25h"

Expand Down
66 changes: 34 additions & 32 deletions src/pip/_internal/commands/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,17 @@
with_cleanup,
)
from pip._internal.cli.status_codes import ERROR, SUCCESS
from pip._internal.exceptions import CommandError, InstallationError
from pip._internal.exceptions import (
CommandError,
InstallationError,
InstallWheelBuildError,
)
from pip._internal.locations import get_scheme
from pip._internal.metadata import get_environment
from pip._internal.metadata import BaseEnvironment, get_environment
from pip._internal.models.installation_report import InstallationReport
from pip._internal.operations.build.build_tracker import get_build_tracker
from pip._internal.operations.check import ConflictDetails, check_install_conflicts
from pip._internal.req import install_given_reqs
from pip._internal.req import InstallationResult, install_given_reqs
from pip._internal.req.req_install import (
InstallRequirement,
check_legacy_setup_py_options,
Expand Down Expand Up @@ -434,12 +438,7 @@ def run(self, options: Values, args: list[str]) -> int:
)

if build_failures:
raise InstallationError(
"Failed to build installable wheels for some "
"pyproject.toml based projects ({})".format(
", ".join(r.name for r in build_failures) # type: ignore
)
)
raise InstallWheelBuildError(build_failures)

to_install = resolver.get_installation_order(requirement_set)

Expand Down Expand Up @@ -478,34 +477,13 @@ def run(self, options: Values, args: list[str]) -> int:
)
env = get_environment(lib_locations)

# Display a summary of installed packages, with extra care to
# display a package name as it was requested by the user.
installed.sort(key=operator.attrgetter("name"))
summary = []
installed_versions = {}
for distribution in env.iter_all_distributions():
installed_versions[distribution.canonical_name] = distribution.version
for package in installed:
display_name = package.name
version = installed_versions.get(canonicalize_name(display_name), None)
if version:
text = f"{display_name}-{version}"
else:
text = display_name
summary.append(text)

if conflicts is not None:
self._warn_about_conflicts(
conflicts,
resolver_variant=self.determine_resolver_variant(options),
)

installed_desc = " ".join(summary)
if installed_desc:
write_output(
"Successfully installed %s",
installed_desc,
)
if summary := installed_packages_summary(installed, env):
write_output(summary)
except OSError as error:
show_traceback = self.verbosity >= 1

Expand Down Expand Up @@ -644,6 +622,30 @@ def _warn_about_conflicts(
logger.critical("\n".join(parts))


def installed_packages_summary(
installed: list[InstallationResult], env: BaseEnvironment
) -> str:
# Format a summary of installed packages, with extra care to
# display a package name as it was requested by the user.
installed.sort(key=operator.attrgetter("name"))
summary = []
installed_versions = {}
for distribution in env.iter_all_distributions():
installed_versions[distribution.canonical_name] = distribution.version
for package in installed:
display_name = package.name
version = installed_versions.get(canonicalize_name(display_name), None)
if version:
text = f"{display_name}-{version}"
else:
text = display_name
summary.append(text)

if not summary:
return ""
return f"Successfully installed {' '.join(summary)}"


def get_lib_location_guesses(
user: bool = False,
home: str | None = None,
Expand Down
4 changes: 2 additions & 2 deletions src/pip/_internal/distributions/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from pip._internal.req import InstallRequirement

if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
from pip._internal.build_env import BuildEnvironmentInstaller


class AbstractDistribution(metaclass=abc.ABCMeta):
Expand Down Expand Up @@ -48,7 +48,7 @@ def get_metadata_distribution(self) -> BaseDistribution:
@abc.abstractmethod
def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand Down
8 changes: 6 additions & 2 deletions src/pip/_internal/distributions/installed.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
from __future__ import annotations

from typing import TYPE_CHECKING

from pip._internal.distributions.base import AbstractDistribution
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import BaseDistribution

if TYPE_CHECKING:
from pip._internal.build_env import BuildEnvironmentInstaller


class InstalledDistribution(AbstractDistribution):
"""Represents an installed package.
Expand All @@ -22,7 +26,7 @@ def get_metadata_distribution(self) -> BaseDistribution:

def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand Down
22 changes: 13 additions & 9 deletions src/pip/_internal/distributions/sdist.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from pip._internal.utils.subprocess import runner_with_spinner_message

if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
from pip._internal.build_env import BuildEnvironmentInstaller

logger = logging.getLogger(__name__)

Expand All @@ -34,7 +34,7 @@ def get_metadata_distribution(self) -> BaseDistribution:

def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand All @@ -46,7 +46,7 @@ def prepare_distribution_metadata(
if should_isolate:
# Setup an isolated environment and install the build backend static
# requirements in it.
self._prepare_build_backend(finder)
self._prepare_build_backend(build_env_installer)
# Check that if the requirement is editable, it either supports PEP 660 or
# has a setup.py or a setup.cfg. This cannot be done earlier because we need
# to setup the build backend to verify it supports build_editable, nor can
Expand All @@ -56,7 +56,7 @@ def prepare_distribution_metadata(
# without setup.py nor setup.cfg.
self.req.isolated_editable_sanity_check()
# Install the dynamic build requirements.
self._install_build_reqs(finder)
self._install_build_reqs(build_env_installer)
# Check if the current environment provides build dependencies
should_check_deps = self.req.use_pep517 and check_build_deps
if should_check_deps:
Expand All @@ -71,15 +71,17 @@ def prepare_distribution_metadata(
self._raise_missing_reqs(missing)
self.req.prepare_metadata()

def _prepare_build_backend(self, finder: PackageFinder) -> None:
def _prepare_build_backend(
self, build_env_installer: BuildEnvironmentInstaller
) -> None:
# Isolate in a BuildEnvironment and install the build-time
# requirements.
pyproject_requires = self.req.pyproject_requires
assert pyproject_requires is not None

self.req.build_env = BuildEnvironment()
self.req.build_env = BuildEnvironment(build_env_installer)
self.req.build_env.install_requirements(
finder, pyproject_requires, "overlay", kind="build dependencies"
pyproject_requires, "overlay", kind="build dependencies", for_req=self.req
)
conflicting, missing = self.req.build_env.check_requirements(
self.req.requirements_to_check
Expand Down Expand Up @@ -115,7 +117,9 @@ def _get_build_requires_editable(self) -> Iterable[str]:
with backend.subprocess_runner(runner):
return backend.get_requires_for_build_editable()

def _install_build_reqs(self, finder: PackageFinder) -> None:
def _install_build_reqs(
self, build_env_installer: BuildEnvironmentInstaller
) -> None:
# Install any extra build dependencies that the backend requests.
# This must be done in a second pass, as the pyproject.toml
# dependencies must be installed before we can call the backend.
Expand All @@ -131,7 +135,7 @@ def _install_build_reqs(self, finder: PackageFinder) -> None:
if conflicting:
self._raise_conflicts("the backend dependencies", conflicting)
self.req.build_env.install_requirements(
finder, missing, "normal", kind="backend dependencies"
missing, "normal", kind="backend dependencies", for_req=self.req
)

def _raise_conflicts(
Expand Down
4 changes: 2 additions & 2 deletions src/pip/_internal/distributions/wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
)

if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
from pip._internal.build_env import BuildEnvironmentInstaller


class WheelDistribution(AbstractDistribution):
Expand All @@ -37,7 +37,7 @@ def get_metadata_distribution(self) -> BaseDistribution:

def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand Down
Loading