review prototype

.github/workflows/test.yml

@@ -0,0 +1,28 @@
+name: Test Workflow
+on: [pull_request, push]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        pip install -r test/requirements.txt
+    - name: Run tests
+      run: |
+        pytest --cov=colcon_cache --cov-branch
+    - uses: codecov/codecov-action@v1
+      if: matrix.python-version == '3.8'
+      with:
+        env_vars: OS,PYTHON

.vscode/settings.json

@@ -0,0 +1,12 @@
+{
+    "python.linting.enabled": true,
+    "python.linting.pylintEnabled": true,
+    "python.linting.flake8Enabled": true,
+    "python.linting.pycodestyleEnabled": true,
+    "cSpell.words": [
+        "CPUs",
+        "noqa",
+        "sched"
+    ],
+    "python.pythonPath": "/usr/bin/python3"
+}

README.md

@@ -1 +1,69 @@
-# colcon-snapshot
+# colcon-cache
+
+An extension for [colcon-core](https://github.com/colcon/colcon-core) to cache packages for processing.
+
+## Example usage
+
+Setup workspace:
+```
+mkdir -p ~/ws/src && cd ~/ws
+wget https://raw.githubusercontent.com/colcon/colcon.readthedocs.org/main/colcon.repos
+vcs import src < colcon.repos
+```
+
+Lock workspace by generating `cache` lockfiles:
+```
+colcon cache lock
+```
+
+Build and test workspace:
+```
+colcon build
+colcon test
+```
+
+Modify package source:
+```
+echo "#foo" >> src/colcon-cmake/setup.py
+```
+
+Update `cache` lockfiles:
+```
+colcon cache lock
+```
+
+List modified packges by comparing `cache` lockfile checksums
+```
+PKGS_MODIFIED=$(colcon list --packages-select-cache-modified | xarg)
+```
+
+Rebuild only modified packages and above:
+```
+colcon build --packages-above $PKGS_MODIFIED
+```
+
+Modify package source again:
+```
+echo "#bar" >> src/colcon-cmake/setup.py
+echo "#baz" >> src/colcon-package-information/setup.py
+```
+
+Update cache lockfiles again:
+```
+colcon cache lock
+```
+
+Rebuild by skipping packages with valid `build` lockfiles:
+```
+colcon build --packages-skip-cache-valid
+```
+
+Retest by skipping packages with valid `test` lockfiles:
+```
+colcon test --packages-skip-cache-valid
+```
+
+List generated lockfiles from each `verb`:
+```
+ls build/colcon-cmake/cache
+```

colcon_cache/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+__version__ = '0.0.0'

colcon_cache/cache/__init__.py

@@ -0,0 +1,113 @@
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+import json
+import pathlib
+
+from colcon_core.plugin_system import satisfies_version
+
+LOCKFILE_FILENAME = 'colcon_{verb_name}.json'
+LOCKFILE_VERSION = '0.0.1'
+
+
+class CacheChecksums:
+    """Cache checksums for package."""
+
+    def __init__(  # noqa: D107
+            self,
+            current=None,
+            reference=None):
+        self.current = current
+        self.reference = reference
+
+    def __eq__(self, other):  # noqa: D105
+        if not isinstance(other, CacheChecksums):
+            return False
+        return self.current == other.current
+
+    def is_modified(self):  # noqa: D10s
+        return self.current != self.reference
+
+
+class CacheLockfile:
+    """Cache lockfile for package."""
+
+    def __init__(  # noqa: D107
+            self,
+            lock_type=None,
+            checksums=None,
+            dependencies=None,
+            metadata=None):
+        self.version = LOCKFILE_VERSION
+        self.lock_type = lock_type
+
+        if checksums:
+            assert(isinstance(checksums, CacheChecksums))
+            self.checksums = checksums
+        else:
+            self.checksums = CacheChecksums()
+
+        if dependencies:
+            assert(isinstance(dependencies, dict))
+            self.dependencies = dependencies
+        else:
+            self.dependencies = {}
+
+        if metadata:
+            assert(isinstance(metadata, dict))
+            self.metadata = metadata
+        else:
+            self.metadata = {}
+
+    def __eq__(self, other):  # noqa: D105
+        if not isinstance(other, CacheLockfile):
+            return False
+        elif self.lock_type != other.lock_type:
+            return False
+        return self.checksums == other.checksums
+
+    def is_modified(self):  # noqa: D10s
+        return self.checksums.is_modified()
+
+    def update_dependencies(self, dep_lockfiles):  # noqa: D10s
+        self.dependencies.clear()
+        for dep_name, dep_lockfile in dep_lockfiles.items():
+            assert isinstance(dep_lockfile, CacheLockfile)
+            self.dependencies[dep_name] = \
+                dep_lockfile.checksums.current
+
+    def load(self, path):  # noqa: D102
+        with open(path, 'r') as f:
+            data = json.load(f)
+        satisfies_version(data['version'], '^0.0.1')
+
+        self.lock_type = data['lock_type']
+        self.checksums = CacheChecksums(**data['checksums'])
+        if data['dependencies']:
+            self.dependencies = data['dependencies']
+        if data['metadata']:
+            self.metadata = data['metadata']
+
+    def dump(self, path):  # noqa: D102
+        with open(path, 'w') as f:
+            json.dump(
+                obj=self,
+                fp=f,
+                indent=2,
+                default=lambda o: o.__dict__,
+                sort_keys=True)
+
+
+def get_lockfile_path(package_build_base, verb_name):
+    """
+    Get the lockfile path of a verb from the package build directory.
+
+    :param str package_build_base: The build directory of a package
+    :param str verb_name: The invoked verb name
+    :returns: The path for the lockfile
+    :rtype: Path
+    """
+    return pathlib.Path(
+        package_build_base,
+        'cache',
+        LOCKFILE_FILENAME.format_map(locals()))

colcon_cache/event_handler/__init__.py

@@ -0,0 +1,36 @@
+# Copyright 2019 Dirk Thomas
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+from colcon_cache.cache import CacheLockfile
+from colcon_cache.cache import get_lockfile_path
+
+
+def get_previous_lockfile(package_build_base, verb_name):
+    """
+    Get the lockfile of a verb from the package build directory.
+
+    :param str package_build_base: The build directory of a package
+    :param str verb_name: The invoked verb name
+    :returns: The previously persisted lockfile, otherwise None
+    :rtype: CacheLockfile
+    """
+    path = get_lockfile_path(package_build_base, verb_name)
+    if not path.exists():
+        return None
+    lockfile = CacheLockfile()
+    lockfile.load(path)
+    return lockfile
+
+
+def set_lockfile(package_build_base, verb_name, lockfile):
+    """
+    Persist the lockfile of a verb in the package build directory.
+
+    :param str package_build_base: The build directory of a package
+    :param str verb_name: The invoked verb name
+    :param CacheLockfile lockfile: The lockfile of the invocation
+    """
+    path = get_lockfile_path(package_build_base, verb_name)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lockfile.dump(path)

colcon_cache/event_handler/lockfile.py

@@ -0,0 +1,57 @@
+# Copyright 2019 Dirk Thomas
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+from colcon_cache.event_handler \
+    import set_lockfile
+from colcon_cache.verb_handler \
+    import get_verb_handler_extensions
+from colcon_core.event.job import JobEnded
+from colcon_core.event.test import TestFailure
+from colcon_core.event_handler import EventHandlerExtensionPoint
+from colcon_core.plugin_system import satisfies_version
+
+
+class LockfileEventHandler(EventHandlerExtensionPoint):
+    """
+    Persist the lockfile of a job in a file in its build directory.
+
+    The extension handles events of the following types:
+    - :py:class:`colcon_core.event.job.JobEnded`
+    - :py:class:`colcon_core.event.test.TestFailure`
+    """
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            EventHandlerExtensionPoint.EXTENSION_POINT_VERSION, '^1.0')
+        self._test_failures = set()
+
+    def __call__(self, event):  # noqa: D102
+        data = event[0]
+
+        if isinstance(data, TestFailure):
+            job = event[1]
+            self._test_failures.add(job)
+
+        elif isinstance(data, JobEnded):
+            job = event[1]
+
+            verb_name = self.context.args.verb_name
+            verb_handler_extensions = get_verb_handler_extensions()
+
+            if verb_name in verb_handler_extensions:
+                verb_handler_extension = verb_handler_extensions[verb_name]
+            else:
+                return
+
+            lockfile = verb_handler_extension.get_job_lockfile(job)
+
+            if job in self._test_failures:
+                return
+            if str(data.rc) != '0':
+                return
+
+            if lockfile:
+                set_lockfile(
+                    job.task_context.args.build_base, verb_name, lockfile)

colcon_cache/package_augmentation/dirhash.py

@@ -0,0 +1,28 @@
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+# from colcon_core.package_augmentation import logger
+from colcon_core.package_augmentation import PackageAugmentationExtensionPoint
+from colcon_core.package_augmentation import update_descriptor
+from colcon_core.plugin_system import satisfies_version
+
+VCS_TYPE = 'dirhash'
+
+
+class DirhashPackageAugmentation(PackageAugmentationExtensionPoint):
+    """Augment packages using no version control system."""
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION,
+            '^1.0')
+
+    def augment_package(  # noqa: D102
+        self, desc, *, additional_argument_names=None
+    ):
+        # deliberately ignore the package type
+        # since this extension can contribute meta information to any package
+        if 'vcs_type' not in desc.metadata:
+            data = {'vcs_type': VCS_TYPE}
+            update_descriptor(desc, data, additional_argument_names=['*'])

colcon_cache/package_augmentation/git.py

@@ -0,0 +1,37 @@
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+# from colcon_core.package_augmentation import logger
+from colcon_core.package_augmentation import PackageAugmentationExtensionPoint
+from colcon_core.package_augmentation import update_descriptor
+from colcon_core.plugin_system import satisfies_version
+from git import InvalidGitRepositoryError, Repo
+
+VCS_TYPE = 'git'
+
+
+class GitPackageAugmentation(PackageAugmentationExtensionPoint):
+    """Augment packages using git version control system."""
+
+    # the priority needs to be lower than the default extensions
+    # augmenting packages using no version control system
+    PRIORITY = 90
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION,
+            '^1.0')
+
+    def augment_package(  # noqa: D102
+        self, desc, *, additional_argument_names=None
+    ):
+        # deliberately ignore the package type
+        # since this extension can contribute meta information to any package
+        try:
+            repo = Repo(desc.path, search_parent_directories=True)
+        except InvalidGitRepositoryError:
+            repo = None
+        if repo:
+            data = {'vcs_type': VCS_TYPE}
+            update_descriptor(desc, data, additional_argument_names=['*'])