Merge pull request #7 from marchinho11/release/0.0.5

Release/0.0.5

.github/workflows/pull_request.yaml

@@ -0,0 +1,17 @@
+name: "Pull request"
+
+on:
+  pull_request:
+    branches:
+      - "main"
+      - "release/**"
+
+jobs:
+  lint:
+    uses: ./.github/workflows/lint.yaml
+
+  tests:
+    needs: [ lint ]
+    uses: ./.github/workflows/tests.yaml
+    secrets:
+      codecov_token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/release.yaml

@@ -2,7 +2,7 @@ name: "Release"
 
 on:
   push:
-    branches: [ main ]
+    tags: [ "*.*.*" ]
 
 jobs:
   lint:

README.md

@@ -91,8 +91,19 @@ Applied!
 ```
 
 The physical result of applied changes:
-
-<img src="assets/quick_start_result.png" height="150px">
+```
+┌───────────────────┐   ┌────────────────┐   ┌─────────────────┐
+│ group__user__name │   │ hub__user      │   │ attr__user__age │
+│                   │   │                │   │                 │
+│ + user_sk (FK)    ├──►│ + user_sk (PK) │◄──┤ + user_sk (FK)  │
+│ + first_name      │   │ + user_id_bk   │   │ + age           │
+│ + last_name       │   │ + valid_from   │   │ + valid_from    │
+│ + valid_from      │   │ + _source      │   │ + _source       │
+│ + valid_to        │   │ + _loaded_at   │   │ + _loaded_at    │
+│ + _source         │   └────────────────┘   └─────────────────┘
+│ + _loaded_at      │
+└───────────────────┘
+```
 
 ## DWH example
 Full DWH example including Entities, Links and Flows can be found in the [`dwh/`](dwh/) directory.

hnhm/cli.py

@@ -16,6 +16,8 @@
     RemoveEntity,
     CreateAttribute,
     RemoveAttribute,
+    AddGroupAttribute,
+    RemoveGroupAttribute,
 )
 
 
@@ -69,6 +71,10 @@ def print_plan(plan: Plan):
                     for attribute in group.attributes.values():
                         click.secho(f"    |attribute '{attribute.name}'", fg="green")
 
+                case AddGroupAttribute(entity=_, group=group, attribute=attribute):
+                    click.secho(f"  [u] group '{group.name}'", fg="yellow")
+                    click.secho(f"    +attribute '{attribute.name}'", fg="green")
+
                 case RemoveEntity(entity=entity):
                     if entity.layout.type == LayoutType.HNHM:
                         click.secho(f"  - hub '{entity.name}'", fg="red")
@@ -85,6 +91,10 @@ def print_plan(plan: Plan):
                     for attribute in group.attributes.values():
                         click.secho(f"    | attribute '{attribute.name}'", fg="red")
 
+                case RemoveGroupAttribute(entity=_, group=group, attribute=attribute):
+                    click.secho(f"  [u] group '{group.name}'", fg="yellow")
+                    click.secho(f"    -attribute '{attribute.name}'", fg="red")
+
     for link_name, plan_collection in links_mutations:
         if plan_collection.type == PlanType.CREATE:
             symbol, color = "+", "green"

hnhm/core/__init__.py

@@ -17,4 +17,6 @@
     RemoveEntity,
     CreateAttribute,
     RemoveAttribute,
+    AddGroupAttribute,
+    RemoveGroupAttribute,
 )

hnhm/core/mutations.py

@@ -65,6 +65,38 @@ def __str__(self):
         return f"<RemoveGroup '{self.group.name}' entity='{self.entity.name}'>"
 
 
+class AddGroupAttribute(Mutation):
+    """Add an Attribute to an existing Group."""
+
+    priority = Priority.SECOND
+    entity: Entity
+    group: Group
+    attribute: Attribute
+
+    def __str__(self):
+        return (
+            f"<AddGroupAttribute '{self.group.name}'"
+            f" entity='{self.entity.name}'"
+            f" attribute='{self.attribute.name}'>"
+        )
+
+
+class RemoveGroupAttribute(Mutation):
+    """Remove an Attribute from an existing Group."""
+
+    priority = Priority.SECOND
+    entity: Entity
+    group: Group
+    attribute: Attribute
+
+    def __str__(self):
+        return (
+            f"<RemoveGroupAttribute '{self.group.name}'"
+            f" entity='{self.entity.name}'"
+            f" attribute='{self.attribute.name}'>"
+        )
+
+
 class CreateLink(Mutation):
     priority = Priority.SECOND
     link: Link

hnhm/hnhm.py

@@ -17,6 +17,8 @@
     RemoveEntity,
     CreateAttribute,
     RemoveAttribute,
+    AddGroupAttribute,
+    RemoveGroupAttribute,
 )
 
 from .hnhm_link import HnhmLink
@@ -103,14 +105,11 @@ def plan(
                 mutations=mutations,
             )
 
-        # Entity: create/remove Attribute/Group
+        # Entity: create/remove/update Attribute/Group
         for entity in core_entities.values():
             if entity.name not in self.data.entities:
                 continue
 
-            if entity.layout.type != LayoutType.HNHM:
-                continue
-
             attributes_state = self.data.entities[entity.name].attributes
             groups_state = self.data.entities[entity.name].groups
 
@@ -125,9 +124,29 @@ def plan(
                 if attribute_name not in entity.attributes:
                     mutations.append(RemoveAttribute(entity=entity, attribute=attribute))
 
-            # Create Group
+            # Create/Update Group
             for group_name, group in entity.groups.items():
-                if group_name not in groups_state:
+                # Update
+                if group_name in groups_state:
+                    group_state = groups_state[group_name]
+                    # Add an Attribute to a Group
+                    for attribute_name, attribute in group.attributes.items():
+                        if attribute_name not in group_state.attributes:
+                            mutations.append(
+                                AddGroupAttribute(
+                                    entity=entity, group=group, attribute=attribute
+                                )
+                            )
+                    # Remove an Attribute from a Group
+                    for attribute_name, attribute in group_state.attributes.items():
+                        if attribute_name not in group.attributes:
+                            mutations.append(
+                                RemoveGroupAttribute(
+                                    entity=entity, group=group, attribute=attribute
+                                )
+                            )
+                # Create
+                else:
                     mutations.append(CreateGroup(entity=entity, group=group))
 
             # Remove Group
@@ -236,6 +255,36 @@ def apply(self, plan: Plan):
                     self.sql.execute(sql)
                     del self.data.entities[entity.name].groups[group.name]
 
+                case AddGroupAttribute(entity=entity, group=group, attribute=attribute):
+                    assert entity.name in self.data.entities
+                    assert group.name in self.data.entities[entity.name].groups
+                    assert (
+                        attribute.name
+                        not in self.data.entities[entity.name]
+                        .groups[group.name]
+                        .attributes
+                    )
+                    self.sql.execute(sql)
+                    self.data.entities[entity.name].groups[group.name].attributes[
+                        attribute.name
+                    ] = attribute
+
+                case RemoveGroupAttribute(
+                    entity=entity, group=group, attribute=attribute
+                ):
+                    assert entity.name in self.data.entities
+                    assert group.name in self.data.entities[entity.name].groups
+                    assert (
+                        attribute.name
+                        in self.data.entities[entity.name].groups[group.name].attributes
+                    )
+                    self.sql.execute(sql)
+                    del (
+                        self.data.entities[entity.name]
+                        .groups[group.name]
+                        .attributes[attribute.name]
+                    )
+
                 case CreateLink(link=link):
                     assert link.name not in self.data.links
                     self.sql.execute(sql)

hnhm/hnhm_entity.py

@@ -13,7 +13,7 @@ class HnhmEntity(abc.ABC):
     def to_core(self) -> Entity:
         inspected = dict(inspect.getmembers(self))
 
-        if "__layout__" not in inspected or not inspected["__layout__"]:
+        if not inspected.get("__layout__"):
             raise HnhmError(
                 f"Layout not found for entity: '{self}'."
                 " Please, specify Layout via '__layout__' attribute."
@@ -27,9 +27,9 @@ def to_core(self) -> Entity:
 
         name = layout.name
 
-        if "__doc__" not in inspected or not inspected["__doc__"]:
+        if not inspected.get("__doc__"):
             raise HnhmError(
-                f"Doc not found or empty for entity: '{name}'."
+                f"Doc not found or empty for entity: '{layout.type}.{name}'."
                 " Please, write a documentation for your entity."
             )
         doc: str = inspected["__doc__"]
@@ -39,9 +39,9 @@ def to_core(self) -> Entity:
                 keys = []
 
             case LayoutType.HNHM:
-                if "__keys__" not in inspected or not inspected["__keys__"]:
+                if not inspected.get("__keys__"):
                     raise HnhmError(
-                        f"At least one Key is required for entity '{self}' with Layout.type='{LayoutType.HNHM}'."
+                        f"At least one Key is required for entity '{layout.type}.{name}'."
                         " Please, specify entity's keys via the '__keys__' attribute."
                     )
 
@@ -50,12 +50,14 @@ def to_core(self) -> Entity:
                     if key.change_type != ChangeType.IGNORE:
                         raise HnhmError(
                             f"Change type='{key.change_type}' is not supported for Key attributes."
-                            f" Please, use 'ChangeType.IGNORE' for the key attributes in the '{self}' entity."
+                            f" Use 'ChangeType.IGNORE' for the key attributes in the '{layout.type}.{name}' entity."
                         )
 
                 keys = [key.to_core() for key in keys_hnhm]
                 if len(keys) != len(set(keys)):
-                    raise HnhmError(f"Found duplicated keys for entity: '{self}'.")
+                    raise HnhmError(
+                        f"Found duplicated keys for entity: '{layout.type}.{name}'."
+                    )
 
             case _:
                 raise HnhmError(f"Unknown LayoutType='{layout.type}'")
@@ -80,14 +82,19 @@ def to_core(self) -> Entity:
 
                 if groups[group_name].change_type != attribute.change_type:
                     raise HnhmError(
-                        f"Found conflicting ChangeType for the entity='{name}' group ='{group_name}'."
+                        f"Found conflicting ChangeType for the entity='{layout.type}.{name}' group='{group_name}'."
                         " Please, use single ChangeType for all attributes within the same group."
                     )
                 groups[group_name].attributes[attribute_name] = attribute
 
             elif attribute not in keys:
                 attributes[attribute_name] = attribute
 
+        if layout.type == LayoutType.STAGE and len(attributes) < 1:
+            raise HnhmError(
+                f"Entity='{layout.type}.{name}' should have at least 1 attribute."
+            )
+
         return Entity(
             name=name,
             layout=layout,

hnhm/hnhm_registry.py

@@ -1,4 +1,5 @@
 from .hnhm import HnHm
+from .core import HnhmError
 from .hnhm_link import HnhmLink
 from .hnhm_entity import HnhmEntity
 
@@ -11,6 +12,27 @@ def __init__(
         entities: list[HnhmEntity] | None = None,
         links: list[HnhmLink] | None = None,
     ):
+        entities_names = set()
+        for entity in entities:
+            entity_core = entity.to_core()
+            full_entity_name = f"{entity_core.layout.type}.{entity_core.name}"
+            if full_entity_name in entities_names:
+                raise HnhmError(
+                    f"Found duplicated entity: '{full_entity_name}'."
+                    " Please, use unique name for each entity and LayoutType."
+                )
+            entities_names.add(full_entity_name)
+
+        links_names = set()
+        for link in links:
+            link_core = link.to_core()
+            if link_core.name in links_names:
+                raise HnhmError(
+                    f"Found duplicated link: '{link_core.name}'."
+                    " Please, use unique name for each link."
+                )
+            links_names.add(link_core.name)
+
         self.hnhm = hnhm
         self.entities = entities
         self.links = links

hnhm/postgres/sql.py

@@ -23,6 +23,8 @@
     LoadAttribute,
     CreateAttribute,
     RemoveAttribute,
+    AddGroupAttribute,
+    RemoveGroupAttribute,
 )
 
 from .sql_templates import (
@@ -76,6 +78,9 @@ def generate_sql(mutation_or_task: Mutation | Task, jinja: jinja2.Environment) -
         case CreateAttribute(entity=entity, attribute=attribute):
             attribute_type = PG_TYPES[attribute.type]
 
+            if entity.layout.type == LayoutType.STAGE:
+                return f"ALTER TABLE stg__{entity.name} ADD COLUMN {attribute.name} {attribute_type}"
+
             time_columns = ["valid_from TIMESTAMPTZ NOT NULL"]
             if attribute.change_type == ChangeType.NEW:
                 time_columns.append("valid_to TIMESTAMPTZ")
@@ -106,6 +111,10 @@ def generate_sql(mutation_or_task: Mutation | Task, jinja: jinja2.Environment) -
                 time_columns=time_columns,
             )
 
+        case AddGroupAttribute(entity=entity, group=group, attribute=attribute):
+            attribute_type = PG_TYPES[attribute.type]
+            return f"ALTER TABLE group__{entity.name}__{group.name} ADD COLUMN {attribute.name} {attribute_type}"
+
         case CreateLink(link=link):
             entities = []
             for link_element in link.elements:
@@ -130,11 +139,17 @@ def generate_sql(mutation_or_task: Mutation | Task, jinja: jinja2.Environment) -
             return f"DROP TABLE {table_name}"
 
         case RemoveAttribute(entity=entity, attribute=attribute):
+            if entity.layout.type == LayoutType.STAGE:
+                return f"ALTER TABLE stg__{entity.name} DROP COLUMN {attribute.name}"
+
             return f"DROP TABLE attr__{entity.name}__{attribute.name}"
 
         case RemoveGroup(entity=entity, group=group):
             return f"DROP TABLE group__{entity.name}__{group.name}"
 
+        case RemoveGroupAttribute(entity=entity, group=group, attribute=attribute):
+            return f"ALTER TABLE group__{entity.name}__{group.name} DROP COLUMN {attribute.name}"
+
         case RemoveLink(link=link):
             return f"DROP TABLE link__{link.name}"
 
@@ -339,7 +354,14 @@ def execute(self, sql: str, debug: bool = False):
         if debug:
             print(dedent(sql).strip())
 
-        with self.engine.connect() as conn:
+        conn = None
+        try:
+            conn = self.engine.connect()
             conn.execute(text(sql))
             conn.commit()
-        self.engine.dispose()
+        except Exception as e:
+            raise e
+        finally:
+            if conn:
+                conn.close()
+            self.engine.dispose()

hnhm/postgres/sql_templates/create_attribute.py

@@ -5,6 +5,8 @@
     {% for time_column in time_columns -%}
         {{ time_column }},
     {% endfor -%}
+    _source VARCHAR(512) NOT NULL,
+    _loaded_at TIMESTAMPTZ NOT NULL,
     CONSTRAINT fk_{{ entity_name }}_sk
       FOREIGN KEY({{ entity_name }}_sk) 
         REFERENCES hub__{{ entity_name }}({{ entity_name }}_sk)