Merge pull request #17 from ruffsl/valid

Rephrase package selection extensions

Author: ruffsl

Diff for: README.md

@@ -4,44 +4,66 @@ An extension for [colcon-core](https://github.com/colcon/colcon-core) to cache p
 
 ## Example usage
 
+Setup workspace:
 ```
-# setup workspace
 mkdir -p ~/ws/src && cd ~/ws
 wget https://raw.githubusercontent.com/colcon/colcon.readthedocs.org/main/colcon.repos
 vcs import src < colcon.repos
+```
 
-# lock cache of workspace source
+Lock workspace by generating `cache` lockfiles:
+```
 colcon cache lock
+```
 
-# build and test workspace
+Build and test workspace:
+```
 colcon build
 colcon test
+```
 
-# change package source
+Modify package source:
+```
 echo "#foo" >> src/colcon-cmake/setup.py
+```
 
-# update cache lock
+Update `cache` lockfiles:
+```
 colcon cache lock
+```
 
-# list changed packges by comparing lockfile checksums
-PKGS_CHANGED=$(colcon list --packages-select-lock-changed | xarg)
+List modified packges by comparing `cache` lockfile checksums
+```
+PKGS_MODIFIED=$(colcon list --packages-select-cache-modified | xarg)
+```
 
-# rebuild only changed packages and above
-colcon build --packages-above $PKGS_CHANGED
+Rebuild only modified packages and above:
+```
+colcon build --packages-above $PKGS_MODIFIED
+```
 
-# alter package source again
+Modify package source again:
+```
 echo "#bar" >> src/colcon-cmake/setup.py
 echo "#baz" >> src/colcon-package-information/setup.py
+```
 
-# update cache lock again
+Update cache lockfiles again:
+```
 colcon cache lock
+```
 
-# rebuild changed packages by comparing verb lockfiles
+Rebuild by skipping packages with valid `build` lockfiles:
+```
 colcon build --packages-skip-cache-valid
+```
 
-# retest packages with any untested build changes
-colcon test  --packages-skip-cache-valid
+Retest by skipping packages with valid `test` lockfiles:
+```
+colcon test --packages-skip-cache-valid
+```
 
-# list generated lockfiles from each verb
+List generated lockfiles from each `verb`:
+```
 ls build/colcon-cmake/cache
 ```

Diff for: colcon_cache/cache/__init__.py

@@ -26,7 +26,7 @@ def __eq__(self, other):  # noqa: D105
             return False
         return self.current == other.current
 
-    def is_changed(self):  # noqa: D10s
+    def is_modified(self):  # noqa: D10s
         return self.current != self.reference
 
 
@@ -67,8 +67,8 @@ def __eq__(self, other):  # noqa: D105
             return False
         return self.checksums == other.checksums
 
-    def is_changed(self):  # noqa: D10s
-        return self.checksums.is_changed()
+    def is_modified(self):  # noqa: D10s
+        return self.checksums.is_modified()
 
     def update_dependencies(self, dep_lockfiles):  # noqa: D10s
         self.dependencies.clear()

Diff for: colcon_cache/event_handler/store_lockfile.py renamed to colcon_cache/event_handler/lockfile.py

@@ -12,7 +12,7 @@
 from colcon_core.plugin_system import satisfies_version
 
 
-class StoreLockfileEventHandler(EventHandlerExtensionPoint):
+class LockfileEventHandler(EventHandlerExtensionPoint):
     """
     Persist the lockfile of a job in a file in its build directory.
 

Diff for: colcon_cache/package_selection/key.py

@@ -0,0 +1,29 @@
+# Copyright 2019 Dirk Thomas
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+from colcon_cache.verb_handler import get_verb_handler_extensions
+from colcon_core.package_selection import PackageSelectionExtensionPoint
+from colcon_core.plugin_system import satisfies_version
+
+
+class KeyPackageSelection(PackageSelectionExtensionPoint):
+    """Skip a set of packages based on lockfiles from previous caches."""
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            PackageSelectionExtensionPoint.EXTENSION_POINT_VERSION, '^1.0')
+
+    def add_arguments(self, *, parser):  # noqa: D102
+        parser.add_argument(
+            '--packages-select-cache-key',
+            choices=get_verb_handler_extensions().keys(),
+            default=None,
+            help='Only process packages using considered cache key. '
+                 'Fallbacks using invoked verb handler if unspecified.')
+
+    def select_packages(self, args, decorators):  # noqa: D102
+        # Pass given added arguments here are considered elsewhere by other
+        # package selection extensions in colcon_cache
+        pass

Diff for: colcon_cache/package_selection/lock.py

@@ -0,0 +1,100 @@
+# Copyright 2019 Dirk Thomas
+# Copyright 2021 Ruffin White
+# Licensed under the Apache License, Version 2.0
+
+import os
+
+from colcon_cache.verb_handler import get_verb_handler_extensions
+from colcon_core.package_selection import logger
+from colcon_core.package_selection import PackageSelectionExtensionPoint
+from colcon_core.plugin_system import satisfies_version
+
+
+class ModifiedPackageSelection(PackageSelectionExtensionPoint):
+    """Skip a set of packages based on lockfiles from current lock."""
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            PackageSelectionExtensionPoint.EXTENSION_POINT_VERSION, '^1.0')
+
+    def add_arguments(self, *, parser):  # noqa: D102
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument(
+            '--packages-select-cache-modified', action='store_true',
+            help='Only process a subset of packages whose cache '
+                 'denote package modifications (packages without lockfiles '
+                 'are not considered as modified)')
+        group.add_argument(
+            '--packages-select-cache-unmodified', action='store_true',
+            help='Only process a subset of packages whose cache '
+                 'denote no package modifications (packages without lockfiles '
+                 'are not considered as unmodified)')
+
+    def select_packages(self, args, decorators):  # noqa: D102
+        if not any((
+            args.packages_select_cache_modified,
+            args.packages_select_cache_unmodified,
+        )):
+            return
+
+        if not hasattr(args, 'build_base'):
+            if args.packages_select_cache_modified:
+                argument = '--packages-select-cache-modified'
+            elif args.packages_select_cache_unmodified:
+                argument = '--packages-select-cache-unmodified'
+            else:
+                assert False
+            logger.warning(
+                "Ignoring '{argument}' since the invoked verb doesn't have a "
+                "'--build-base' argument and therefore can't access "
+                'information about the previous state of a package'
+                .format_map(locals()))
+            return
+
+        verb_name = args.packages_select_cache_key
+        if not verb_name:
+            verb_name = args.verb_name
+        verb_handler_extensions = get_verb_handler_extensions()
+
+        if verb_name in verb_handler_extensions:
+            verb_handler_extension = verb_handler_extensions[verb_name]
+        else:
+            logger.warning(
+                "Ignoring '{argument}' since the respective verb "
+                "'{verb_name}' doesn't have a colcon cache verb "
+                "handler extension and therefore can't access "
+                'information about the relative state of a package'
+                .format_map(locals()))
+            return
+
+        for decorator in decorators:
+            # skip packages which have already been ruled out
+            if not decorator.selected:
+                continue
+
+            pkg = decorator.descriptor
+
+            package_build_base = os.path.join(
+                args.build_base, pkg.name)
+
+            verb_lockfile = verb_handler_extension\
+                .get_current_lockfile(package_build_base)
+
+            package_kind = None
+            if verb_lockfile is None:
+                package_kind = ('without lockfile')
+            else:
+                if args.packages_select_cache_modified:
+                    if not verb_lockfile.is_modified():
+                        package_kind = ('unmodified')
+
+                if args.packages_select_cache_unmodified:
+                    if verb_lockfile.is_modified():
+                        package_kind = ('modified')
+
+            if package_kind:
+                logger.info(
+                    "Skipping {package_kind} package '{pkg.name}' in "
+                    "'{pkg.path}'".format_map(locals()))
+                decorator.selected = False