Merge pull request #1189 from mokibit/add-merge-reset-override

Implement `override` and `reset` analog to docker-compose

Author: p12tic

newsfragments/implement-merge-reset-and-override.feature

@@ -0,0 +1 @@
+- Add support for  `reset` and `override` tags to be used when merging several compose files.

podman_compose.py

@@ -1407,6 +1407,57 @@ def flat_deps(services, with_extends=False):
         rec_deps(services, name)
 
 
+###################
+# Override and reset tags
+###################
+
+
+class OverrideTag(yaml.YAMLObject):
+    yaml_dumper = yaml.Dumper
+    yaml_loader = yaml.SafeLoader
+    yaml_tag = '!override'
+
+    def __init__(self, value):
+        if len(value) > 0 and isinstance(value[0], tuple):
+            self.value = {}
+            # item is a tuple representing service's lower level key and value
+            for item in value:
+                # value can actually be a list, then all the elements from the list have to be
+                # collected
+                if isinstance(item[1].value, list):
+                    self.value[item[0].value] = [item.value for item in item[1].value]
+                else:
+                    self.value[item[0].value] = item[1].value
+        else:
+            self.value = [item.value for item in value]
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        return OverrideTag(node.value)
+
+    @classmethod
+    def to_yaml(cls, dumper, data):
+        return dumper.represent_scalar(cls.yaml_tag, data.value)
+
+
+class ResetTag(yaml.YAMLObject):
+    yaml_dumper = yaml.Dumper
+    yaml_loader = yaml.SafeLoader
+    yaml_tag = '!reset'
+
+    @classmethod
+    def to_json(cls):
+        return cls.yaml_tag
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        return ResetTag()
+
+    @classmethod
+    def to_yaml(cls, dumper, data):
+        return dumper.represent_scalar(cls.yaml_tag, '')
+
+
 async def wait_with_timeout(coro, timeout):
     """
     Asynchronously waits for the given coroutine to complete with a timeout.
@@ -1605,6 +1656,12 @@ async def volume_ls(self):
 
 
 def normalize_service(service, sub_dir=""):
+    if isinstance(service, ResetTag):
+        return service
+
+    if isinstance(service, OverrideTag):
+        service = service.value
+
     if "build" in service:
         build = service["build"]
         if isinstance(build, str):
@@ -1708,6 +1765,8 @@ def rec_merge_one(target, source):
     update target from source recursively
     """
     done = set()
+    remove = set()
+
     for key, value in source.items():
         if key in target:
             continue
@@ -1717,15 +1776,37 @@ def rec_merge_one(target, source):
         if key in done:
             continue
         if key not in source:
+            if isinstance(value, ResetTag):
+                log("INFO: Unneeded !reset found for [{key}]")
+                remove.add(key)
+
+            if isinstance(value, OverrideTag):
+                log("INFO: Unneeded !override found for [{key}] with value '{value}'")
+                target[key] = clone(value.value)
+
             continue
+
         value2 = source[key]
+
+        if isinstance(value, ResetTag) or isinstance(value2, ResetTag):
+            remove.add(key)
+            continue
+
+        if isinstance(value, OverrideTag) or isinstance(value2, OverrideTag):
+            target[key] = (
+                clone(value.value) if isinstance(value, OverrideTag) else clone(value2.value)
+            )
+            continue
+
         if key in ("command", "entrypoint"):
             target[key] = clone(value2)
             continue
+
         if not isinstance(value2, type(value)):
             value_type = type(value)
             value2_type = type(value2)
             raise ValueError(f"can't merge value of [{key}] of type {value_type} and {value2_type}")
+
         if is_list(value2):
             if key == "volumes":
                 # clean duplicate mount targets
@@ -1742,6 +1823,10 @@ def rec_merge_one(target, source):
             rec_merge_one(value, value2)
         else:
             target[key] = value2
+
+    for key in remove:
+        del target[key]
+
     return target
 
 
@@ -2027,10 +2112,13 @@ def _parse_compose_file(self):
             content = rec_subs(content, self.environ)
             if isinstance(services := content.get('services'), dict):
                 for service in services.values():
-                    if 'extends' in service and (service_file := service['extends'].get('file')):
-                        service['extends']['file'] = os.path.join(
-                            os.path.dirname(filename), service_file
-                        )
+                    if not isinstance(service, OverrideTag) and not isinstance(service, ResetTag):
+                        if 'extends' in service and (
+                            service_file := service['extends'].get('file')
+                        ):
+                            service['extends']['file'] = os.path.join(
+                                os.path.dirname(filename), service_file
+                            )
 
             rec_merge(compose, content)
             # If `include` is used, append included files to files

tests/integration/volumes_merge/__init__.py renamed to tests/integration/merge/reset_and_override_tags/override_tag_attribute/__init__.py

@@ -0,0 +1,7 @@
+version: "3"
+services:
+    app:
+        image: busybox
+        command: ["/bin/busybox", "echo", "One"]
+        ports: !override
+            - "8111:81"

tests/integration/merge/reset_and_override_tags/override_tag_attribute/docker-compose.override_attribute.yaml

@@ -0,0 +1,7 @@
+version: "3"
+services:
+    app:
+        image: busybox
+        command: ["/bin/busybox", "echo", "Zero"]
+        ports:
+            - "8080:80"

tests/integration/merge/reset_and_override_tags/override_tag_attribute/docker-compose.yaml

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: GPL-2.0
+
+import json
+import os
+import unittest
+
+from tests.integration.test_utils import RunSubprocessMixin
+from tests.integration.test_utils import podman_compose_path
+from tests.integration.test_utils import test_path
+
+
+def compose_yaml_path():
+    return os.path.join(os.path.join(test_path(), "override_tag_attribute"), "docker-compose.yaml")
+
+
+class TestComposeOverrideTagAttribute(unittest.TestCase, RunSubprocessMixin):
+    # test if a service attribute from docker-compose.yaml file is overridden
+    def test_override_tag_attribute(self):
+        override_file = os.path.join(
+            os.path.join(test_path(), "override_tag_attribute"),
+            "docker-compose.override_attribute.yaml",
+        )
+        try:
+            self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "-f",
+                override_file,
+                "up",
+            ])
+            # merge rules are still applied
+            output, _ = self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "-f",
+                override_file,
+                "logs",
+            ])
+            self.assertEqual(output, b"One\n")
+
+            # only app service attribute "ports" was overridden
+            output, _ = self.run_subprocess_assert_returncode([
+                "podman",
+                "inspect",
+                "override_tag_attribute_app_1",
+            ])
+            container_info = json.loads(output.decode('utf-8'))[0]
+            self.assertEqual(
+                container_info['NetworkSettings']["Ports"],
+                {"81/tcp": [{"HostIp": "", "HostPort": "8111"}]},
+            )
+        finally:
+            self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "down",
+            ])

tests/integration/merge/reset_and_override_tags/override_tag_attribute/test_podman_compose_override_tag_attribute.py

@@ -0,0 +1 @@
+

tests/integration/merge/reset_and_override_tags/override_tag_service/__init__.py

@@ -0,0 +1,7 @@
+version: "3"
+services:
+    app: !override
+        image: busybox
+        command: ["/bin/busybox", "echo", "One"]
+        ports:
+            - "8111:81"

tests/integration/merge/reset_and_override_tags/override_tag_service/docker-compose.override_service.yaml

@@ -0,0 +1,7 @@
+version: "3"
+services:
+    app:
+        image: busybox
+        command: ["/bin/busybox", "echo", "Zero"]
+        ports:
+            - "8080:80"

tests/integration/merge/reset_and_override_tags/override_tag_service/docker-compose.yaml

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: GPL-2.0
+
+import json
+import os
+import unittest
+
+from tests.integration.test_utils import RunSubprocessMixin
+from tests.integration.test_utils import podman_compose_path
+from tests.integration.test_utils import test_path
+
+
+def compose_yaml_path():
+    return os.path.join(os.path.join(test_path(), "override_tag_service"), "docker-compose.yaml")
+
+
+class TestComposeOverrideTagService(unittest.TestCase, RunSubprocessMixin):
+    # test if whole service from docker-compose.yaml file is overridden in another file
+    def test_override_tag_service(self):
+        override_file = os.path.join(
+            os.path.join(test_path(), "override_tag_service"),
+            "docker-compose.override_service.yaml",
+        )
+        try:
+            self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "-f",
+                override_file,
+                "up",
+            ])
+
+            # Whole app service was overridden in the docker-compose.override_tag_service.yaml file.
+            # Command and port is overridden accordingly.
+            output, _ = self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "-f",
+                override_file,
+                "logs",
+            ])
+            self.assertEqual(output, b"One\n")
+
+            output, _ = self.run_subprocess_assert_returncode([
+                "podman",
+                "inspect",
+                "override_tag_service_app_1",
+            ])
+            container_info = json.loads(output.decode('utf-8'))[0]
+            self.assertEqual(
+                container_info['NetworkSettings']["Ports"],
+                {"81/tcp": [{"HostIp": "", "HostPort": "8111"}]},
+            )
+        finally:
+            self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "down",
+            ])

tests/integration/merge/reset_and_override_tags/override_tag_service/test_podman_compose_override_tag_service.py

@@ -0,0 +1 @@
+

tests/integration/merge/reset_and_override_tags/reset_tag_attribute/__init__.py

@@ -0,0 +1,5 @@
+version: "3"
+services:
+    app:
+        image: busybox
+        command: !reset {}

tests/integration/merge/reset_and_override_tags/reset_tag_attribute/docker-compose.reset_attribute.yaml

@@ -0,0 +1,5 @@
+version: "3"
+services:
+    app:
+        image: busybox
+        command: ["/bin/busybox", "echo", "Zero"]

tests/integration/merge/reset_and_override_tags/reset_tag_attribute/docker-compose.yaml

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: GPL-2.0
+
+import os
+import unittest
+
+from tests.integration.test_utils import RunSubprocessMixin
+from tests.integration.test_utils import podman_compose_path
+from tests.integration.test_utils import test_path
+
+
+def compose_yaml_path():
+    return os.path.join(os.path.join(test_path(), "reset_tag_attribute"), "docker-compose.yaml")
+
+
+class TestComposeResetTagAttribute(unittest.TestCase, RunSubprocessMixin):
+    # test if the attribute of the service is correctly reset
+    def test_reset_tag_attribute(self):
+        reset_file = os.path.join(
+            os.path.join(test_path(), "reset_tag_attribute"), "docker-compose.reset_attribute.yaml"
+        )
+        try:
+            self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "-f",
+                reset_file,
+                "up",
+            ])
+
+            # the service still exists, but its command attribute was reset in
+            # docker-compose.reset_tag_attribute.yaml file and is now empty
+            output, _ = self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "-f",
+                reset_file,
+                "ps",
+            ])
+            self.assertIn(b"reset_tag_attribute_app_1", output)
+
+            output, _ = self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "-f",
+                reset_file,
+                "logs",
+            ])
+            self.assertEqual(output, b"")
+        finally:
+            self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path(),
+                "down",
+            ])

tests/integration/merge/reset_and_override_tags/reset_tag_attribute/test_podman_compose_reset_tag_attribute.py

@@ -0,0 +1 @@
+

tests/integration/merge/reset_and_override_tags/reset_tag_service/__init__.py

@@ -0,0 +1,6 @@
+version: "3"
+services:
+    app: !reset
+    app2:
+        image: busybox
+        command: ["/bin/busybox", "echo", "One"]

tests/integration/merge/reset_and_override_tags/reset_tag_service/docker-compose.reset_service.yaml

@@ -0,0 +1,5 @@
+version: "3"
+services:
+    app:
+        image: busybox
+        command: ["/bin/busybox", "echo", "Zero"]