refactor: change of a sync_apps.py logic (#190)

Extend sync-apps with custom config file in tenant application directories / Refactor sync-apps

Author: makrelas

.gitignore

@@ -9,4 +9,5 @@ testrepo/
 *.iml
 .eggs/
 site/
+build/
 .mypy_cache

Dockerfile

@@ -10,7 +10,7 @@ FROM base AS dev
 WORKDIR /workdir
 RUN apk add --no-cache gcc linux-headers musl-dev make
 RUN python -m venv /opt/venv
-RUN pip install --upgrade pip
+RUN python -m pip install --upgrade pip
 
 # =========
 FROM dev AS deps

docs/commands/sync-apps.md

@@ -28,6 +28,22 @@ root-config-repo/
 └── bootstrap
     └── values.yaml
 ```
+### app specific values
+app specific values may be set using a .config.yaml file directly in the app directory. gitopscli will process these values and add them under customAppConfig parameter of application
+**tenantrepo.git/app1/app_value_file.yaml**
+```yaml
+customvalue: test
+```
+**rootrepo.git/apps/tenantrepo.yaml**
+```yaml
+config:
+  repository: https://tenantrepo.git
+  applications:
+    app1:
+      customAppConfig:
+        customvalue: test
+    app2: {}
+```
 
 **bootstrap/values.yaml**
 ```yaml

gitopscli/appconfig_api/__init__.py

@@ -0,0 +1,132 @@
+import logging
+from dataclasses import dataclass, field
+import os
+from typing import Any
+
+from gitopscli.git_api import GitRepo
+from gitopscli.io_api.yaml_util import yaml_load, yaml_file_load
+
+from gitopscli.gitops_exception import GitOpsException
+
+
+@dataclass
+class AppTenantConfig:
+    yaml: dict[str, dict[str, Any]]
+    tenant_config: dict[str, dict[str, Any]] = field(default_factory=dict)
+    repo_url: str = ""
+    file_path: str = ""
+    dirty: bool = False
+
+    def __post_init__(self) -> None:
+        if "config" in self.yaml:
+            self.tenant_config = self.yaml["config"]
+        else:
+            self.tenant_config = self.yaml
+        if "repository" not in self.tenant_config:
+            raise GitOpsException("Cannot find key 'repository' in " + self.file_path)
+        self.repo_url = str(self.tenant_config["repository"])
+
+    def list_apps(self) -> dict[str, dict[str, Any]]:
+        return dict(self.tenant_config["applications"])
+
+    def merge_applications(self, desired_tenant_config: "AppTenantConfig") -> None:
+        desired_apps = desired_tenant_config.list_apps()
+        self.__delete_removed_applications(desired_apps)
+        self.__add_new_applications(desired_apps)
+        self.__update_custom_app_config(desired_apps)
+
+    def __update_custom_app_config(self, desired_apps: dict[str, dict[str, Any]]) -> None:
+        for desired_app_name, desired_app_value in desired_apps.items():
+            if desired_app_name in self.list_apps():
+                existing_application_value = self.list_apps()[desired_app_name]
+                if "customAppConfig" not in desired_app_value:
+                    if existing_application_value and "customAppConfig" in existing_application_value:
+                        logging.info(
+                            "Removing customAppConfig in for %s in %s applications",
+                            existing_application_value,
+                            self.file_path,
+                        )
+                        del existing_application_value["customAppConfig"]
+                        self.__set_dirty()
+                else:
+                    if (
+                        "customAppConfig" not in existing_application_value
+                        or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"]
+                    ):
+                        logging.info(
+                            "Updating customAppConfig in for %s in %s applications",
+                            existing_application_value,
+                            self.file_path,
+                        )
+                        existing_application_value["customAppConfig"] = desired_app_value["customAppConfig"]
+                        self.__set_dirty()
+
+    def __add_new_applications(self, desired_apps: dict[str, Any]) -> None:
+        for desired_app_name, desired_app_value in desired_apps.items():
+            if desired_app_name not in self.list_apps().keys():
+                logging.info("Adding % in %s applications", desired_app_name, self.file_path)
+                self.tenant_config["applications"][desired_app_name] = desired_app_value
+                self.__set_dirty()
+
+    def __delete_removed_applications(self, desired_apps: dict[str, Any]) -> None:
+        for current_app in self.list_apps().keys():
+            if current_app not in desired_apps.keys():
+                logging.info("Removing %s from %s applications", current_app, self.file_path)
+                del self.tenant_config["applications"][current_app]
+                self.__set_dirty()
+
+    def __set_dirty(self) -> None:
+        self.dirty = True
+
+
+def __generate_config_from_tenant_repo(
+    tenant_repo: GitRepo,
+) -> Any:  # TODO: supposed to be ruamel object than Any  pylint: disable=fixme
+    tenant_app_dirs = __get_all_tenant_applications_dirs(tenant_repo)
+    tenant_config_template = """
+    config: 
+        repository: {}
+        applications: {{}}
+    """.format(
+        tenant_repo.get_clone_url()
+    )
+    yaml = yaml_load(tenant_config_template)
+    for app_dir in tenant_app_dirs:
+        tenant_application_template = """
+        {}: {{}}
+        """.format(
+            app_dir
+        )
+        tenant_applications_yaml = yaml_load(tenant_application_template)
+        # dict path hardcoded as object generated will always be in v2 or later
+        yaml["config"]["applications"].update(tenant_applications_yaml)
+        custom_app_config = __get_custom_config(app_dir, tenant_repo)
+        if custom_app_config:
+            yaml["config"]["applications"][app_dir]["customAppConfig"] = custom_app_config
+    return yaml
+
+
+def __get_all_tenant_applications_dirs(tenant_repo: GitRepo) -> set[str]:
+    repo_dir = tenant_repo.get_full_file_path(".")
+    applist = {
+        name
+        for name in os.listdir(repo_dir)
+        if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".")
+    }
+    return applist
+
+
+def __get_custom_config(appname: str, tenant_config_git_repo: GitRepo) -> Any:
+    custom_config_path = tenant_config_git_repo.get_full_file_path(f"{appname}/.config.yaml")
+    if os.path.exists(custom_config_path):
+        custom_config_content = yaml_file_load(custom_config_path)
+        return custom_config_content
+    return dict()
+
+
+def create_app_tenant_config_from_repo(
+    tenant_repo: GitRepo,
+) -> "AppTenantConfig":
+    tenant_repo.clone()
+    tenant_config_yaml = __generate_config_from_tenant_repo(tenant_repo)
+    return AppTenantConfig(yaml=tenant_config_yaml)

gitopscli/appconfig_api/app_tenant_config.py

@@ -0,0 +1,83 @@
+from dataclasses import dataclass
+from typing import List, Any, Optional
+
+from gitopscli.git_api import GitRepo
+from gitopscli.io_api.yaml_util import yaml_file_load
+
+from gitopscli.appconfig_api.app_tenant_config import AppTenantConfig
+from gitopscli.gitops_exception import GitOpsException
+
+
+@dataclass
+class RootRepo:
+    tenants: dict[str, AppTenantConfig]
+
+    def list_tenants(self) -> list[str]:
+        return list(self.tenants.keys())
+
+    def get_tenant_by_repo_url(self, repo_url: str) -> Optional[AppTenantConfig]:
+        for tenant in self.tenants.values():
+            if tenant.repo_url == repo_url:
+                return tenant
+        return None
+
+    def get_all_applications(self) -> list[str]:
+        apps: list[str] = list()
+        for tenant in self.tenants.values():
+            apps.extend(tenant.list_apps().keys())
+        return apps
+
+    def validate_tenant(self, tenant_config: AppTenantConfig) -> None:
+        apps_from_other_tenants: list[str] = list()
+        for tenant in self.tenants.values():
+            if tenant.repo_url != tenant_config.repo_url:
+                apps_from_other_tenants.extend(tenant.list_apps().keys())
+        for app_name in tenant_config.list_apps().keys():
+            if app_name in apps_from_other_tenants:
+                raise GitOpsException(f"Application '{app_name}' already exists in a different repository")
+
+
+def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[str, AppTenantConfig]:
+    boostrap_tenant_list = __get_bootstrap_tenant_list(root_repo)
+    tenants = dict()
+    for bootstrap_tenant in boostrap_tenant_list:
+        try:
+            tenant_name = bootstrap_tenant["name"]
+            absolute_tenant_file_path = root_repo.get_full_file_path("apps/" + tenant_name + ".yaml")
+            yaml = yaml_file_load(absolute_tenant_file_path)
+            tenants[tenant_name] = AppTenantConfig(
+                yaml=yaml,
+                file_path=absolute_tenant_file_path,
+            )
+        except FileNotFoundError as ex:
+            raise GitOpsException(f"File '{absolute_tenant_file_path}' not found in root repository.") from ex
+    return tenants
+
+
+def __get_bootstrap_tenant_list(root_repo: GitRepo) -> List[Any]:
+    root_repo.clone()
+    try:
+        boostrap_values_path = root_repo.get_full_file_path("bootstrap/values.yaml")
+        bootstrap_yaml = yaml_file_load(boostrap_values_path)
+    except FileNotFoundError as ex:
+        raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex
+    bootstrap_tenants = []
+    if "bootstrap" in bootstrap_yaml:
+        bootstrap_tenants = list(bootstrap_yaml["bootstrap"])
+    if "config" in bootstrap_yaml and "bootstrap" in bootstrap_yaml["config"]:
+        bootstrap_tenants = list(bootstrap_yaml["config"]["bootstrap"])
+    __validate_bootstrap_tenants(bootstrap_tenants)
+    return bootstrap_tenants
+
+
+def __validate_bootstrap_tenants(bootstrap_entries: Optional[List[Any]]) -> None:
+    if not bootstrap_entries:
+        raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'")
+    for bootstrap_entry in bootstrap_entries:
+        if "name" not in bootstrap_entry:
+            raise GitOpsException("Every bootstrap entry must have a 'name' property.")
+
+
+def create_root_repo(root_repo: GitRepo) -> "RootRepo":
+    root_repo_tenants = __load_tenants_from_bootstrap_values(root_repo)
+    return RootRepo(root_repo_tenants)