feat(connector-builder): add version constraint and wildcard support to manifest migrations (#569)

Author: lazebnyi

airbyte_cdk/manifest_migrations/README.md

@@ -20,7 +20,8 @@ This directory contains the logic and registry for manifest migrations in the Ai
 
 3. **Register the Migration:**
    - Open `migrations/registry.yaml`.
-   - Add an entry under the appropriate version, or create a new version section if needed.
+   - Add an entry under the appropriate version, or create a new version section if needed. 
+     - Version can be: "*", "==6.48.3", "~=1.2", ">=1.0.0,<2.0.0", "6.48.3" 
    - Each migration entry should include:
      - `name`: The filename (without `.py`)
      - `order`: The order in which this migration should be applied for the version

airbyte_cdk/manifest_migrations/migration_handler.py

@@ -5,9 +5,11 @@
 
 import copy
 import logging
+import re
 from datetime import datetime, timezone
-from typing import Type
+from typing import Tuple, Type
 
+from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 
 from airbyte_cdk.manifest_migrations.exceptions import (
@@ -25,7 +27,7 @@
 METADATA_TAG = "metadata"
 MANIFEST_VERSION_TAG = "version"
 APPLIED_MIGRATIONS_TAG = "applied_migrations"
-
+WILDCARD_VERSION_PATTERN = ".*"
 LOGGER = logging.getLogger("airbyte.cdk.manifest_migrations")
 
 
@@ -77,11 +79,14 @@ def _handle_migration(
         """
         try:
             migration_instance = migration_class()
-            if self._version_is_valid_for_migration(manifest_version, migration_version):
+            can_apply_migration, should_bump_version = self._version_is_valid_for_migration(
+                manifest_version, migration_version
+            )
+            if can_apply_migration:
                 migration_instance._process_manifest(self._migrated_manifest)
                 if migration_instance.is_migrated:
-                    # set the updated manifest version, after migration has been applied
-                    self._set_manifest_version(migration_version)
+                    if should_bump_version:
+                        self._set_manifest_version(migration_version)
                     self._set_migration_trace(migration_class, manifest_version, migration_version)
             else:
                 LOGGER.info(
@@ -112,18 +117,30 @@ def _version_is_valid_for_migration(
         self,
         manifest_version: str,
         migration_version: str,
-    ) -> bool:
+    ) -> Tuple[bool, bool]:
+        """
+        Decide whether *manifest_version* satisfies the *migration_version* rule.
+
+        Rules
+        -----
+        1. ``"*"``
+           – Wildcard: anything matches.
+        2. String starts with a PEP 440 operator (``==``, ``!=``, ``<=``, ``>=``,
+           ``<``, ``>``, ``~=``, etc.)
+           – Treat *migration_version* as a SpecifierSet and test the manifest
+             version against it.
+        3. Plain version
+           – Interpret both strings as concrete versions and return
+             ``manifest_version <= migration_version``.
         """
-        Checks if the given manifest version is less than or equal to the specified migration version.
+        if re.match(WILDCARD_VERSION_PATTERN, migration_version):
+            return True, False
 
-        Args:
-            manifest_version (str): The version of the manifest to check.
-            migration_version (str): The migration version to compare against.
+        if migration_version.startswith(("=", "!", ">", "<", "~")):
+            spec = SpecifierSet(migration_version)
+            return spec.contains(Version(manifest_version)), False
 
-        Returns:
-            bool: True if the manifest version is less than or equal to the migration version, False otherwise.
-        """
-        return Version(manifest_version) <= Version(migration_version)
+        return Version(manifest_version) <= Version(migration_version), True
 
     def _set_manifest_version(self, version: str) -> None:
         """

airbyte_cdk/manifest_migrations/migrations/__init__.py

@@ -2,3 +2,18 @@
 # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.manifest_migrations.migrations.http_requester_path_to_url import (
+    HttpRequesterPathToUrl,
+)
+from airbyte_cdk.manifest_migrations.migrations.http_requester_request_body_json_data_to_request_body import (
+    HttpRequesterRequestBodyJsonDataToRequestBody,
+)
+from airbyte_cdk.manifest_migrations.migrations.http_requester_url_base_to_url import (
+    HttpRequesterUrlBaseToUrl,
+)
+
+__all__ = [
+    "HttpRequesterPathToUrl",
+    "HttpRequesterRequestBodyJsonDataToRequestBody",
+    "HttpRequesterUrlBaseToUrl",
+]

airbyte_cdk/manifest_migrations/migrations/registry.yaml

@@ -3,7 +3,7 @@
 #
 
 manifest_migrations:
-  - version: 6.48.3
+  - version: "*"
     migrations: 
       - name: http_requester_url_base_to_url
         order: 1

unit_tests/manifest_migrations/conftest.py

@@ -6,11 +6,13 @@
 
 import pytest
 
+from airbyte_cdk.manifest_migrations.manifest_migration import ManifestMigration
+
 
 @pytest.fixture
 def manifest_with_url_base_to_migrate_to_url() -> Dict[str, Any]:
     return {
-        "version": "0.0.0",
+        "version": "6.48.3",
         "type": "DeclarativeSource",
         "check": {
             "type": "CheckStream",
@@ -493,14 +495,14 @@ def expected_manifest_with_url_base_migrated_to_url() -> Dict[str, Any]:
         "metadata": {
             "applied_migrations": [
                 {
-                    "from_version": "0.0.0",
-                    "to_version": "6.48.3",
+                    "from_version": "6.48.3",
+                    "to_version": ">=6.48.2,<6.50.0",
                     "migration": "HttpRequesterUrlBaseToUrl",
                     "migrated_at": "2025-04-01T00:00:00+00:00",  # time freezed in the test
                 },
                 {
-                    "from_version": "0.0.0",
-                    "to_version": "6.48.3",
+                    "from_version": "6.48.3",
+                    "to_version": ">=6.48.2,<6.50.0",
                     "migration": "HttpRequesterPathToUrl",
                     "migrated_at": "2025-04-01T00:00:00+00:00",  # time freezed in the test
                 },
@@ -832,7 +834,7 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic
 @pytest.fixture
 def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
     return {
-        "version": "6.48.3",
+        "version": "0.0.0",
         "type": "DeclarativeSource",
         "check": {"type": "CheckStream", "stream_names": ["A"]},
         "definitions": {
@@ -1195,22 +1197,36 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]:
             "applied_migrations": [
                 {
                     "from_version": "0.0.0",
-                    "to_version": "6.48.3",
+                    "to_version": "*",
                     "migration": "HttpRequesterUrlBaseToUrl",
                     "migrated_at": "2025-04-01T00:00:00+00:00",
                 },
                 {
                     "from_version": "0.0.0",
-                    "to_version": "6.48.3",
+                    "to_version": "*",
                     "migration": "HttpRequesterPathToUrl",
                     "migrated_at": "2025-04-01T00:00:00+00:00",
                 },
                 {
                     "from_version": "0.0.0",
-                    "to_version": "6.48.3",
+                    "to_version": "*",
                     "migration": "HttpRequesterRequestBodyJsonDataToRequestBody",
                     "migrated_at": "2025-04-01T00:00:00+00:00",
                 },
             ]
         },
     }
+
+
+class DummyMigration(ManifestMigration):
+    def _process_manifest(self, manifest):
+        self.is_migrated = False
+
+    def should_migrate(self, manifest):
+        return True
+
+    def validate(self, manifest):
+        return True
+
+    def migrate(self, manifest):
+        pass

unit_tests/manifest_migrations/test_manifest_migration.py

@@ -1,21 +1,39 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
+from unittest.mock import patch
 
 from freezegun import freeze_time
 
+from airbyte_cdk.manifest_migrations import migrations_registry
 from airbyte_cdk.manifest_migrations.migration_handler import (
     ManifestMigrationHandler,
 )
-from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from airbyte_cdk.manifest_migrations.migrations import (
+    HttpRequesterPathToUrl,
+    HttpRequesterRequestBodyJsonDataToRequestBody,
+    HttpRequesterUrlBaseToUrl,
+)
 from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import (
     ManifestReferenceResolver,
 )
+from unit_tests.manifest_migrations.conftest import DummyMigration
 
 resolver = ManifestReferenceResolver()
 
 
 @freeze_time("2025-04-01")
+@patch.dict(
+    migrations_registry.MANIFEST_MIGRATIONS,
+    {
+        ">=6.48.2,<6.50.0": [
+            HttpRequesterUrlBaseToUrl,
+            HttpRequesterPathToUrl,
+            HttpRequesterRequestBodyJsonDataToRequestBody,
+        ]
+    },
+    clear=True,
+)
 def test_manifest_resolve_migrate_url_base_and_path_to_url(
     manifest_with_url_base_to_migrate_to_url,
     expected_manifest_with_url_base_migrated_to_url,
@@ -25,7 +43,9 @@ def test_manifest_resolve_migrate_url_base_and_path_to_url(
     when the `url_base` is migrated to `url` and the `path` is joined to `url`.
     """
 
-    resolved_manifest = resolver.preprocess_manifest(manifest_with_url_base_to_migrate_to_url)
+    resolved_manifest = ManifestReferenceResolver().preprocess_manifest(
+        manifest_with_url_base_to_migrate_to_url
+    )
     migrated_manifest = ManifestMigrationHandler(dict(resolved_manifest)).apply_migrations()
 
     assert migrated_manifest == expected_manifest_with_url_base_migrated_to_url
@@ -50,6 +70,7 @@ def test_manifest_resolve_migrate_request_body_json_and_data_to_request_body(
 
 
 @freeze_time("2025-04-01")
+@patch.dict(migrations_registry.MANIFEST_MIGRATIONS, {"0.0.0": [DummyMigration]}, clear=True)
 def test_manifest_resolve_do_not_migrate(
     manifest_with_migrated_url_base_and_path_is_joined_to_url,
 ) -> None:
@@ -58,7 +79,7 @@ def test_manifest_resolve_do_not_migrate(
     after the `url_base` and `path` is joined to `url`.
     """
 
-    resolved_manifest = resolver.preprocess_manifest(
+    resolved_manifest = ManifestReferenceResolver().preprocess_manifest(
         manifest_with_migrated_url_base_and_path_is_joined_to_url
     )
     migrated_manifest = ManifestMigrationHandler(dict(resolved_manifest)).apply_migrations()