upgrade tests: Fix upgrade tests on old versions

Seen failing in https://buildkite.com/materialize/nightly/builds/12016

Author: def-

misc/python/materialize/checks/scenarios_upgrade.py

@@ -37,9 +37,7 @@ def get_minor_versions() -> list[MzVersion]:
         current_version = MzVersion.parse_cargo()
         _minor_versions = [
             v
-            for v in get_published_minor_mz_versions(
-                limit=4, exclude_current_minor_version=True
-            )
+            for v in get_published_minor_mz_versions(exclude_current_minor_version=True)
             if v < current_version
         ]
     return _minor_versions

misc/python/materialize/git.py

@@ -124,14 +124,21 @@ def get_version_tags(
 
 
 def get_latest_version(
-    version_type: type[VERSION_TYPE], excluded_versions: set[VERSION_TYPE] | None = None
+    version_type: type[VERSION_TYPE],
+    excluded_versions: set[VERSION_TYPE] | None = None,
+    current_version: VERSION_TYPE | None = None,
 ) -> VERSION_TYPE:
     all_version_tags: list[VERSION_TYPE] = get_version_tags(
         version_type=version_type, fetch=True
     )
 
     if excluded_versions is not None:
-        all_version_tags = [v for v in all_version_tags if v not in excluded_versions]
+        all_version_tags = [
+            v
+            for v in all_version_tags
+            if v not in excluded_versions
+            and (not current_version or v < current_version)
+        ]
 
     return max(all_version_tags)
 

misc/python/materialize/version_list.py

@@ -245,12 +245,15 @@ def resolve_image_tag(self) -> tuple[str, str]:
 
 
 def get_latest_published_version() -> MzVersion:
-    """Get the latest mz version for which an image is published."""
+    """Get the latest mz version, older than current state, for which an image is published."""
     excluded_versions = set()
+    current_version = MzVersion.parse_cargo()
 
     while True:
         latest_published_version = git.get_latest_version(
-            version_type=MzVersion, excluded_versions=excluded_versions
+            version_type=MzVersion,
+            excluded_versions=excluded_versions,
+            current_version=current_version,
         )
 
         if is_valid_release_image(latest_published_version):

test/legacy-upgrade/mzcompose.py

@@ -142,13 +142,20 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
 def get_all_and_latest_two_minor_mz_versions(
     use_versions_from_docs: bool,
 ) -> tuple[list[MzVersion], list[MzVersion]]:
+    current_version = MzVersion.parse_cargo()
     if use_versions_from_docs:
         version_list = VersionsFromDocs(respect_released_tag=False)
-        all_versions = version_list.all_versions()
+        all_versions = [v for v in version_list.all_versions() if v < current_version]
         tested_versions = version_list.minor_versions()[-2:]
     else:
-        tested_versions = get_published_minor_mz_versions(limit=2)
-        all_versions = get_all_published_mz_versions(newest_first=False)
+        tested_versions = [
+            v for v in get_published_minor_mz_versions() if v < current_version
+        ]
+        all_versions = [
+            v
+            for v in get_all_published_mz_versions(newest_first=False)
+            if v < current_version
+        ]
     return all_versions, tested_versions
 
 
@@ -249,8 +256,13 @@ def test_upgrade_from_version(
         c.rm(mz_service, "testdrive")
 
     if from_version != "current_source" and not lts_upgrade:
+        current_version = MzVersion.parse_cargo()
         # We can't skip in-between minor versions anymore, so go through all of them
-        for version in get_published_minor_mz_versions(newest_first=False):
+        for version in [
+            v
+            for v in get_published_minor_mz_versions(newest_first=False)
+            if v < current_version
+        ]:
             if version <= from_version:
                 continue
             if version >= MzVersion.parse_cargo():