Skip to content

pytest plugin: Improve performance via caching #472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Oct 12, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGES
Original file line number Diff line number Diff line change
@@ -17,6 +17,11 @@ $ pip install --user --upgrade --pre libvcs

### Breaking changes

#### pytest plugin: Improve performacne via session-based scoping (#472)

Improved test execution speed by over 54% by eliminated repetitive repository reinitialization between test runs.
Git, Subversion, and Mercurial repositories are now cached from an initial starter repository

#### pytest fixtures: `git_local_clone` renamed to `example_git_repo` (#468)

Renamed `git_local_clone` to `example_git_repo` for better understandability in
340 changes: 271 additions & 69 deletions src/libvcs/pytest_plugin.py
Original file line number Diff line number Diff line change
@@ -180,7 +180,7 @@ def clean() -> None:
return path


@pytest.fixture
@pytest.fixture(scope="session")
def remote_repos_path(
user_path: pathlib.Path,
request: pytest.FixtureRequest,
@@ -239,26 +239,110 @@ def __call__(


def _create_git_remote_repo(
remote_repos_path: pathlib.Path,
remote_repo_name: str,
remote_repo_path: pathlib.Path,
remote_repo_post_init: Optional[CreateRepoPostInitFn] = None,
init_cmd_args: InitCmdArgs = DEFAULT_GIT_REMOTE_REPO_CMD_ARGS,
) -> pathlib.Path:
if init_cmd_args is None:
init_cmd_args = []
remote_repo_path = remote_repos_path / remote_repo_name
run(["git", "init", remote_repo_name, *init_cmd_args], cwd=remote_repos_path)
run(
["git", "init", remote_repo_path.stem, *init_cmd_args],
cwd=remote_repo_path.parent,
)

if remote_repo_post_init is not None and callable(remote_repo_post_init):
remote_repo_post_init(remote_repo_path=remote_repo_path)

return remote_repo_path


@pytest.fixture
@pytest.fixture(scope="session")
def libvcs_test_cache_path(tmp_path_factory: pytest.TempPathFactory) -> pathlib.Path:
"""Return temporary directory to use as cache path for libvcs tests."""
return tmp_path_factory.mktemp("libvcs-test-cache")


@pytest.fixture(scope="session")
def empty_git_repo_path(libvcs_test_cache_path: pathlib.Path) -> pathlib.Path:
"""Return temporary directory to use for master-copy of a git repo."""
return libvcs_test_cache_path / "empty_git_repo"


@pytest.fixture(scope="session")
def empty_git_bare_repo_path(libvcs_test_cache_path: pathlib.Path) -> pathlib.Path:
"""Return temporary directory to use for master-copy of a bare git repo."""
return libvcs_test_cache_path / "empty_git_bare_repo"


@pytest.fixture(scope="session")
@skip_if_git_missing
def empty_git_bare_repo(
empty_git_bare_repo_path: pathlib.Path,
) -> pathlib.Path:
"""Return factory to create git remote repo to for clone / push purposes."""
if (
empty_git_bare_repo_path.exists()
and (empty_git_bare_repo_path / ".git").exists()
):
return empty_git_bare_repo_path

return _create_git_remote_repo(
remote_repo_path=empty_git_bare_repo_path,
remote_repo_post_init=None,
init_cmd_args=DEFAULT_GIT_REMOTE_REPO_CMD_ARGS, # --bare
)


@pytest.fixture(scope="session")
@skip_if_git_missing
def empty_git_repo(
empty_git_repo_path: pathlib.Path,
) -> pathlib.Path:
"""Return factory to create git remote repo to for clone / push purposes."""
if empty_git_repo_path.exists() and (empty_git_repo_path / ".git").exists():
return empty_git_repo_path

return _create_git_remote_repo(
remote_repo_path=empty_git_repo_path,
remote_repo_post_init=None,
init_cmd_args=None,
)


@pytest.fixture(scope="session")
@skip_if_git_missing
def create_git_remote_bare_repo(
remote_repos_path: pathlib.Path,
empty_git_bare_repo: pathlib.Path,
) -> CreateRepoPytestFixtureFn:
"""Return factory to create git remote repo to for clone / push purposes."""

def fn(
remote_repos_path: pathlib.Path = remote_repos_path,
remote_repo_name: Optional[str] = None,
remote_repo_post_init: Optional[CreateRepoPostInitFn] = None,
init_cmd_args: InitCmdArgs = DEFAULT_GIT_REMOTE_REPO_CMD_ARGS,
) -> pathlib.Path:
if remote_repo_name is None:
remote_repo_name = unique_repo_name(remote_repos_path=remote_repos_path)
remote_repo_path = remote_repos_path / remote_repo_name

shutil.copytree(empty_git_bare_repo, remote_repo_path)

assert empty_git_bare_repo.exists()

assert remote_repo_path.exists()

return remote_repo_path

return fn


@pytest.fixture(scope="session")
@skip_if_git_missing
def create_git_remote_repo(
remote_repos_path: pathlib.Path,
empty_git_repo: pathlib.Path,
) -> CreateRepoPytestFixtureFn:
"""Return factory to create git remote repo to for clone / push purposes."""

@@ -268,14 +352,22 @@ def fn(
remote_repo_post_init: Optional[CreateRepoPostInitFn] = None,
init_cmd_args: InitCmdArgs = DEFAULT_GIT_REMOTE_REPO_CMD_ARGS,
) -> pathlib.Path:
return _create_git_remote_repo(
remote_repos_path=remote_repos_path,
remote_repo_name=remote_repo_name
if remote_repo_name is not None
else unique_repo_name(remote_repos_path=remote_repos_path),
remote_repo_post_init=remote_repo_post_init,
init_cmd_args=init_cmd_args,
)
if remote_repo_name is None:
remote_repo_name = unique_repo_name(remote_repos_path=remote_repos_path)
remote_repo_path = remote_repos_path / remote_repo_name

shutil.copytree(empty_git_repo, remote_repo_path)

if remote_repo_post_init is not None and callable(remote_repo_post_init):
remote_repo_post_init(remote_repo_path=remote_repo_path)

assert empty_git_repo.exists()
assert (empty_git_repo / ".git").exists()

assert remote_repo_path.exists()
assert (remote_repo_path / ".git").exists()

return remote_repo_path

return fn

@@ -288,29 +380,27 @@ def git_remote_repo_single_commit_post_init(remote_repo_path: pathlib.Path) -> N
run(["git", "commit", "-m", "test file for dummyrepo"], cwd=remote_repo_path)


@pytest.fixture
@pytest.fixture(scope="session")
@skip_if_git_missing
def git_remote_repo(remote_repos_path: pathlib.Path) -> pathlib.Path:
"""Pre-made git repo w/ 1 commit, used as a file:// remote to clone and push to."""
return _create_git_remote_repo(
remote_repos_path=remote_repos_path,
remote_repo_name="dummyrepo",
remote_repo_post_init=git_remote_repo_single_commit_post_init,
init_cmd_args=None, # Don't do --bare
)
def git_remote_repo(
create_git_remote_repo: CreateRepoPytestFixtureFn,
) -> pathlib.Path:
"""Copy the session-scoped Git repository to a temporary directory."""
# TODO: Cache the effect of of this in a session-based repo
repo_path = create_git_remote_repo()
git_remote_repo_single_commit_post_init(remote_repo_path=repo_path)
return repo_path


def _create_svn_remote_repo(
remote_repos_path: pathlib.Path,
remote_repo_name: str,
remote_repo_path: pathlib.Path,
remote_repo_post_init: Optional[CreateRepoPostInitFn] = None,
init_cmd_args: InitCmdArgs = None,
) -> pathlib.Path:
"""Create a test SVN repo to for checkout / commit purposes."""
if init_cmd_args is None:
init_cmd_args = []

remote_repo_path = remote_repos_path / remote_repo_name
run(["svnadmin", "create", str(remote_repo_path), *init_cmd_args])

assert remote_repo_path.exists()
@@ -340,10 +430,33 @@ def svn_remote_repo_single_commit_post_init(remote_repo_path: pathlib.Path) -> N
)


@pytest.fixture
@pytest.fixture(scope="session")
def empty_svn_repo_path(libvcs_test_cache_path: pathlib.Path) -> pathlib.Path:
"""Return temporary directory to use for master-copy of a svn repo."""
return libvcs_test_cache_path / "empty_svn_repo"


@pytest.fixture(scope="session")
@skip_if_svn_missing
def empty_svn_repo(
empty_svn_repo_path: pathlib.Path,
) -> pathlib.Path:
"""Return factory to create svn remote repo to for clone / push purposes."""
if empty_svn_repo_path.exists() and (empty_svn_repo_path / "conf").exists():
return empty_svn_repo_path

return _create_svn_remote_repo(
remote_repo_path=empty_svn_repo_path,
remote_repo_post_init=None,
init_cmd_args=None,
)


@pytest.fixture(scope="session")
@skip_if_svn_missing
def create_svn_remote_repo(
remote_repos_path: pathlib.Path,
empty_svn_repo: pathlib.Path,
) -> CreateRepoPytestFixtureFn:
"""Pre-made svn repo, bare, used as a file:// remote to checkout and commit to."""

@@ -353,41 +466,58 @@ def fn(
remote_repo_post_init: Optional[CreateRepoPostInitFn] = None,
init_cmd_args: InitCmdArgs = None,
) -> pathlib.Path:
return _create_svn_remote_repo(
remote_repos_path=remote_repos_path,
remote_repo_name=remote_repo_name
if remote_repo_name is not None
else unique_repo_name(remote_repos_path=remote_repos_path),
remote_repo_post_init=remote_repo_post_init,
init_cmd_args=init_cmd_args,
)
if remote_repo_name is None:
remote_repo_name = unique_repo_name(remote_repos_path=remote_repos_path)
remote_repo_path = remote_repos_path / remote_repo_name

shutil.copytree(empty_svn_repo, remote_repo_path)

if remote_repo_post_init is not None and callable(remote_repo_post_init):
remote_repo_post_init(remote_repo_path=remote_repo_path)

assert empty_svn_repo.exists()

assert remote_repo_path.exists()

return remote_repo_path

return fn


@pytest.fixture
@pytest.fixture(scope="session")
@skip_if_svn_missing
def svn_remote_repo(remote_repos_path: pathlib.Path) -> pathlib.Path:
def svn_remote_repo(
create_svn_remote_repo: CreateRepoPytestFixtureFn,
) -> pathlib.Path:
"""Pre-made. Local file:// based SVN server."""
return _create_svn_remote_repo(
remote_repos_path=remote_repos_path,
remote_repo_name="svn_server_dir",
remote_repo_post_init=None,
)
repo_path = create_svn_remote_repo()
return repo_path


@pytest.fixture(scope="session")
@skip_if_svn_missing
def svn_remote_repo_with_files(
create_svn_remote_repo: CreateRepoPytestFixtureFn,
) -> pathlib.Path:
"""Pre-made. Local file:// based SVN server."""
repo_path = create_svn_remote_repo()
svn_remote_repo_single_commit_post_init(remote_repo_path=repo_path)
return repo_path


def _create_hg_remote_repo(
remote_repos_path: pathlib.Path,
remote_repo_name: str,
remote_repo_path: pathlib.Path,
remote_repo_post_init: Optional[CreateRepoPostInitFn] = None,
init_cmd_args: InitCmdArgs = None,
) -> pathlib.Path:
"""Create a test hg repo to for checkout / commit purposes."""
if init_cmd_args is None:
init_cmd_args = []

remote_repo_path = remote_repos_path / remote_repo_name
run(["hg", "init", remote_repo_name, *init_cmd_args], cwd=remote_repos_path)
run(
["hg", "init", remote_repo_path.stem, *init_cmd_args],
cwd=remote_repo_path.parent,
)

if remote_repo_post_init is not None and callable(remote_repo_post_init):
remote_repo_post_init(remote_repo_path=remote_repo_path)
@@ -403,12 +533,33 @@ def hg_remote_repo_single_commit_post_init(remote_repo_path: pathlib.Path) -> No
run(["hg", "commit", "-m", "test file for hg repo"], cwd=remote_repo_path)


@pytest.fixture
@pytest.fixture(scope="session")
def empty_hg_repo_path(libvcs_test_cache_path: pathlib.Path) -> pathlib.Path:
"""Return temporary directory to use for master-copy of a hg repo."""
return libvcs_test_cache_path / "empty_hg_repo"


@pytest.fixture(scope="session")
@skip_if_hg_missing
def empty_hg_repo(
empty_hg_repo_path: pathlib.Path,
) -> pathlib.Path:
"""Return factory to create hg remote repo to for clone / push purposes."""
if empty_hg_repo_path.exists() and (empty_hg_repo_path / ".hg").exists():
return empty_hg_repo_path

return _create_hg_remote_repo(
remote_repo_path=empty_hg_repo_path,
remote_repo_post_init=None,
init_cmd_args=None,
)


@pytest.fixture(scope="session")
@skip_if_hg_missing
def create_hg_remote_repo(
remote_repos_path: pathlib.Path,
hgconfig: pathlib.Path,
set_home: pathlib.Path,
empty_hg_repo: pathlib.Path,
) -> CreateRepoPytestFixtureFn:
"""Pre-made hg repo, bare, used as a file:// remote to checkout and commit to."""

@@ -418,38 +569,57 @@ def fn(
remote_repo_post_init: Optional[CreateRepoPostInitFn] = None,
init_cmd_args: InitCmdArgs = None,
) -> pathlib.Path:
return _create_hg_remote_repo(
remote_repos_path=remote_repos_path,
remote_repo_name=remote_repo_name
if remote_repo_name is not None
else unique_repo_name(remote_repos_path=remote_repos_path),
remote_repo_post_init=remote_repo_post_init,
init_cmd_args=init_cmd_args,
)
if remote_repo_name is None:
remote_repo_name = unique_repo_name(remote_repos_path=remote_repos_path)
remote_repo_path = remote_repos_path / remote_repo_name

shutil.copytree(empty_hg_repo, remote_repo_path)

if remote_repo_post_init is not None and callable(remote_repo_post_init):
remote_repo_post_init(remote_repo_path=remote_repo_path)

assert empty_hg_repo.exists()

assert remote_repo_path.exists()

return remote_repo_path

return fn


@pytest.fixture
@pytest.fixture(scope="session")
@skip_if_hg_missing
def hg_remote_repo(
remote_repos_path: pathlib.Path,
hgconfig: pathlib.Path,
create_hg_remote_repo: CreateRepoPytestFixtureFn,
) -> pathlib.Path:
"""Pre-made, file-based repo for push and pull."""
return _create_hg_remote_repo(
remote_repos_path=remote_repos_path,
remote_repo_name="dummyrepo",
remote_repo_post_init=hg_remote_repo_single_commit_post_init,
)
repo_path = create_hg_remote_repo()
hg_remote_repo_single_commit_post_init(remote_repo_path=repo_path)
return repo_path


@pytest.fixture
def git_repo(projects_path: pathlib.Path, git_remote_repo: pathlib.Path) -> GitSync:
def git_repo(
remote_repos_path: pathlib.Path,
projects_path: pathlib.Path,
git_remote_repo: pathlib.Path,
) -> GitSync:
"""Pre-made git clone of remote repo checked out to user's projects dir."""
remote_repo_name = unique_repo_name(remote_repos_path=projects_path)
new_checkout_path = projects_path / remote_repo_name
master_copy = remote_repos_path / "git_repo"

if master_copy.exists():
shutil.copytree(master_copy, new_checkout_path)
return GitSync(
url=f"file://{git_remote_repo}",
path=str(new_checkout_path),
)

git_repo = GitSync(
url=f"file://{git_remote_repo}",
path=str(projects_path / "git_repo"),
path=master_copy,
remotes={
"origin": GitRemote(
name="origin",
@@ -463,19 +633,49 @@ def git_repo(projects_path: pathlib.Path, git_remote_repo: pathlib.Path) -> GitS


@pytest.fixture
def hg_repo(projects_path: pathlib.Path, hg_remote_repo: pathlib.Path) -> HgSync:
def hg_repo(
remote_repos_path: pathlib.Path,
projects_path: pathlib.Path,
hg_remote_repo: pathlib.Path,
) -> HgSync:
"""Pre-made hg clone of remote repo checked out to user's projects dir."""
remote_repo_name = unique_repo_name(remote_repos_path=projects_path)
new_checkout_path = projects_path / remote_repo_name
master_copy = remote_repos_path / "hg_repo"

if master_copy.exists():
shutil.copytree(master_copy, new_checkout_path)
return HgSync(
url=f"file://{hg_remote_repo}",
path=str(new_checkout_path),
)

hg_repo = HgSync(
url=f"file://{hg_remote_repo}",
path=str(projects_path / "hg_repo"),
path=master_copy,
)
hg_repo.obtain()
return hg_repo


@pytest.fixture
def svn_repo(projects_path: pathlib.Path, svn_remote_repo: pathlib.Path) -> SvnSync:
def svn_repo(
remote_repos_path: pathlib.Path,
projects_path: pathlib.Path,
svn_remote_repo: pathlib.Path,
) -> SvnSync:
"""Pre-made svn clone of remote repo checked out to user's projects dir."""
remote_repo_name = unique_repo_name(remote_repos_path=projects_path)
new_checkout_path = projects_path / remote_repo_name
master_copy = remote_repos_path / "svn_repo"

if master_copy.exists():
shutil.copytree(master_copy, new_checkout_path)
return SvnSync(
url=f"file://{svn_remote_repo}",
path=str(new_checkout_path),
)

svn_repo = SvnSync(
url=f"file://{svn_remote_repo}",
path=str(projects_path / "svn_repo"),
@@ -491,6 +691,7 @@ def add_doctest_fixtures(
tmp_path: pathlib.Path,
set_home: pathlib.Path,
gitconfig: pathlib.Path,
hgconfig: pathlib.Path,
create_git_remote_repo: CreateRepoPytestFixtureFn,
create_svn_remote_repo: CreateRepoPytestFixtureFn,
create_hg_remote_repo: CreateRepoPytestFixtureFn,
@@ -518,6 +719,7 @@ def add_doctest_fixtures(
remote_repo_post_init=svn_remote_repo_single_commit_post_init,
)
if shutil.which("hg"):
doctest_namespace["hgconfig"] = hgconfig
doctest_namespace["create_hg_remote_repo_bare"] = create_hg_remote_repo
doctest_namespace["create_hg_remote_repo"] = functools.partial(
create_hg_remote_repo,
6 changes: 3 additions & 3 deletions tests/sync/test_git.py
Original file line number Diff line number Diff line change
@@ -172,17 +172,17 @@ def test_repo_update_handle_cases(
)
def test_repo_update_stash_cases(
tmp_path: pathlib.Path,
create_git_remote_repo: CreateRepoPytestFixtureFn,
create_git_remote_bare_repo: CreateRepoPytestFixtureFn,
mocker: MockerFixture,
has_untracked_files: bool,
needs_stash: bool,
has_remote_changes: bool,
) -> None:
"""Test GitSync.update_repo() stash cases."""
git_remote_repo = create_git_remote_repo()
git_bare_repo = create_git_remote_bare_repo()

git_repo: GitSync = GitSync(
url=git_remote_repo.as_uri(),
url=git_bare_repo.as_uri(),
path=tmp_path / "myrepo",
vcs="git",
)
12 changes: 12 additions & 0 deletions tests/sync/test_hg.py
Original file line number Diff line number Diff line change
@@ -8,12 +8,24 @@
from libvcs import exc
from libvcs._internal.run import run
from libvcs._internal.shortcuts import create_project
from libvcs.pytest_plugin import CreateRepoPytestFixtureFn
from libvcs.sync.hg import HgSync

if not shutil.which("hg"):
pytestmark = pytest.mark.skip(reason="hg is not available")


@pytest.fixture
def hg_remote_repo(
set_home: pathlib.Path,
hgconfig: pathlib.Path,
create_hg_remote_repo: CreateRepoPytestFixtureFn,
) -> pathlib.Path:
"""Create a remote hg repository."""
return create_hg_remote_repo()


@pytest.mark.usefixtures("set_home", "hgconfig")
def test_hg_sync(
tmp_path: pathlib.Path,
projects_path: pathlib.Path,
20 changes: 20 additions & 0 deletions tests/sync/test_svn.py
Original file line number Diff line number Diff line change
@@ -30,6 +30,26 @@ def test_svn_sync(tmp_path: pathlib.Path, svn_remote_repo: pathlib.Path) -> None
assert (tmp_path / repo_name).exists()


def test_svn_sync_with_files(
tmp_path: pathlib.Path, svn_remote_repo_with_files: pathlib.Path
) -> None:
"""Tests for SvnSync."""
repo_name = "my_svn_project"

svn_repo = SvnSync(
url=f"file://{svn_remote_repo_with_files}",
path=str(tmp_path / repo_name),
)

svn_repo.obtain()
svn_repo.update_repo()

assert svn_repo.get_revision() == 0
assert svn_repo.get_revision_file("./") == 3

assert (tmp_path / repo_name).exists()


def test_repo_svn_remote_checkout(
create_svn_remote_repo: CreateRepoPytestFixtureFn,
tmp_path: pathlib.Path,
24 changes: 18 additions & 6 deletions tests/url/test_hg.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
"""Tests for mercurial URL module."""

import pathlib
import typing

import pytest

from libvcs.pytest_plugin import CreateRepoPytestFixtureFn
from libvcs.sync.hg import HgSync
from libvcs.url.base import RuleMap
from libvcs.url.hg import DEFAULT_RULES, PIP_DEFAULT_RULES, HgBaseURL, HgURL
@@ -17,6 +19,16 @@ class HgURLFixture(typing.NamedTuple):
hg_url: HgURL


@pytest.fixture
def hg_repo(
set_home: pathlib.Path,
hgconfig: pathlib.Path,
create_hg_remote_repo: CreateRepoPytestFixtureFn,
) -> pathlib.Path:
"""Create a remote hg repository."""
return create_hg_remote_repo()


TEST_FIXTURES: list[HgURLFixture] = [
HgURLFixture(
url="https://bitbucket.com/vcs-python/libvcs",
@@ -52,8 +64,8 @@ def test_hg_url(
hg_repo: HgSync,
) -> None:
"""Test HgURL."""
url = url.format(local_repo=hg_repo.path)
hg_url.url = hg_url.url.format(local_repo=hg_repo.path)
url = url.format(local_repo=hg_repo)
hg_url.url = hg_url.url.format(local_repo=hg_repo)

assert HgURL.is_valid(url) == is_valid, f"{url} compatibility should be {is_valid}"
assert HgURL(url) == hg_url
@@ -121,10 +133,10 @@ class HgURLWithPip(HgURL):
_rule_map={m.label: m for m in [*DEFAULT_RULES, *PIP_DEFAULT_RULES]},
)

hg_url_kwargs["url"] = hg_url_kwargs["url"].format(local_repo=hg_repo.path)
url = url.format(local_repo=hg_repo.path)
hg_url_kwargs["url"] = hg_url_kwargs["url"].format(local_repo=hg_repo)
url = url.format(local_repo=hg_repo)
hg_url = HgURLWithPip(**hg_url_kwargs)
hg_url.url = hg_url.url.format(local_repo=hg_repo.path)
hg_url.url = hg_url.url.format(local_repo=hg_repo)

assert (
HgBaseURL.is_valid(url) != is_valid
@@ -186,6 +198,6 @@ def test_hg_to_url(
hg_repo: HgSync,
) -> None:
"""Test HgURL.to_url()."""
hg_url.url = hg_url.url.format(local_repo=hg_repo.path)
hg_url.url = hg_url.url.format(local_repo=hg_repo)

assert hg_url.to_url() == expected