rendering graph example with class

Author: wassfila

README.md

@@ -2,9 +2,11 @@
 how to render a pipeline flow with graph and time chart
 
 # usage
-to run the pipeline through
+The pipeline can bun run with
 
-    python runner.py
+    run
+
+which will execute `runner.py` from venv
 
 testing containers e.g. graphviz rendering
 

cmd.cmd

@@ -0,0 +1,2 @@
+@echo off
+python containers\run.py %*

containers/docker-compose.yaml

@@ -5,3 +5,9 @@ services:
       dockerfile: graphviz.Dockerfile
     volumes:
       - ../cache:/data
+  apache:
+    image: httpd:latest
+    volumes:
+      - ../view:/usr/local/apache2/htdocs/
+    ports:
+      - 80:80

containers/run.py

@@ -3,19 +3,21 @@
 import os
 
 def graphviz(filename):
+    os.chdir(docker_compose_dir)
     command_line = f"docker compose run --rm graphviz -Tsvg {filename} -o {filename}.svg"
     subprocess.run(command_line.split(" "))
     return
 
 def run(command):
     # Path to the directory containing docker-compose.yml
-    docker_compose_dir = os.path.join(os.path.dirname(__file__))
-    # Change the working directory
-    os.chdir(docker_compose_dir)
-
     if(command == "graphviz"):
         filename = sys.argv[2]
         graphviz(filename)
+    elif(command == "server"):
+        os.chdir(docker_compose_dir)
+        subprocess.run("docker compose up apache", shell=True, check=True)
+
+docker_compose_dir = os.path.join(os.path.dirname(__file__))
 
 if __name__ == '__main__':
     # Default command if no argument is provided

graph_utils.py

@@ -0,0 +1,54 @@
+
+def int_to_alphabet(n):
+    """
+    Converts an integer to a string where numbers are mapped as follows:
+    1 -> A, 2 -> B, ..., 26 -> Z, 27 -> AA, 28 -> AB, and so on.
+    """
+    result = []
+    while n > 0:
+        n -= 1  # Decrement n by 1 to make the modulus operation zero-indexed
+        remainder = n % 26
+        result.append(chr(remainder + 65))  # 65 is the ASCII value for 'A'
+        n = n // 26  # Move to the next 'digit'
+    
+    return ''.join(reversed(result))
+
+def graph_to_dot(graph):
+    dot_string = "digraph G {\n"
+    nodes_map = {}
+    for index,node in enumerate(graph["nodes"],1):
+        nodes_map[node] = int_to_alphabet(index)
+    for node in graph["nodes"]:
+        dot_string += f'    {nodes_map[node]} [label="{node}"];\n'
+    for edge in graph["edges"]:
+        source = nodes_map[edge["source"]]
+        target = nodes_map[edge["target"]]
+        dot_string += f'    {source} -> {target};\n'
+    dot_string += "}"
+    return dot_string
+
+def add_edge(source,target):
+    global graph
+    if(source not in graph["nodes"]):
+        graph["nodes"].append(source)
+    if(target not in graph["nodes"]):
+        graph["nodes"].append(target)
+    graph["edges"].append({
+        "source":source,
+        "target":target
+    })
+    return
+
+def get_dot_graph():
+    global graph
+    dot_string = graph_to_dot(graph)
+    return dot_string
+
+def get_graph():
+    global graph
+    return graph
+
+graph = {
+    "nodes":[],
+    "edges":[]
+}

pipeline.py

@@ -13,13 +13,22 @@ def fetch():
 def calculate():
     print("- hello from calc2")
     new_data = run.get_artifact("content")
+    time.sleep(0.3)
     print(new_data)
+    run.set_artifact({"stats":5},"fetch/stats.json")
     return
 
 def compute():
     print("- hellor from comp3")
+    new_data = run.get_artifact("content")
+    time.sleep(0.4)
+    run.set_artifact({"result":6},"fetch/result.json")
     return
 
 def build():
     print("- hi build4")
+    stats = run.get_artifact("stats")
+    result = run.get_artifact("result")
+    time.sleep(0.5)
+    run.set_artifact({"web":10},"fetch/website.json")
     return

requirements.txt

@@ -0,0 +1,2 @@
+pyyaml
+networkx

run.cmd

@@ -1,2 +1,2 @@
 @echo off
-python containers\run.py %*
+venv\Scripts\python runner.py

runner.py

@@ -3,6 +3,8 @@
 from os.path import dirname,splitext,basename,join
 import state
 from datetime import datetime
+import graph_utils as gutl
+from containers import run
 
 class ArtifactError(Exception):
     """Custom exception for artifact management errors."""
@@ -34,12 +36,14 @@ def set_artifact(data,filepath,type="generic"):
     abs_filepath = join("cache",filepath)
     if(ext == ".json"):
         utl.save_json(data,abs_filepath)
+    gutl.add_edge(state.job,id)
     return
 
 def get_artifact(id):
     if(id not in state.artifacts):
         raise ArtifactError(f"Artifact with ID '{id}' does not exist")
     artifact = state.artifacts[id]
+    gutl.add_edge(id,state.job)
     if(artifact["ext"] == ".json"):
         return utl.load_json(join("cache",artifact["filepath"]))
     return None
@@ -64,7 +68,9 @@ def run_pipeline(pipeline):
         run_stage(stage, jobs)
     utl.save_json(state.artifacts,"cache/artifacts.json")
     utl.save_json(state.pipe,"cache/pipeline.json")
-    
+    utl.save_json(gutl.get_graph(),"cache/dependencies.json")
+    utl.save_text(gutl.get_dot_graph(),"cache/dependencies.dot")
+    run.graphviz("dependencies.dot")
 
 if __name__ == '__main__':
     manifest = utl.load_yaml("manifest.yaml")

test-graph.py

@@ -0,0 +1,21 @@
+import graph_utils as gutl
+import utils as utl
+from containers import run
+
+graph = {
+    "nodes": [
+        {"id": "A", "label": "Node A"},
+        {"id": "B", "label": "Node B"},
+        {"id": "C", "label": "Node C"}
+    ],
+    "edges": [
+        {"source": "A", "target": "B", "label": "A to B"},
+        {"source": "B", "target": "C", "label": "B to C"}
+    ]
+}
+
+dot = gutl.graph_to_dot(graph)
+
+utl.save_text(dot,"cache/test/gen.dot")
+
+run.graphviz("test/gen.dot")

test-manifest.yaml

@@ -23,6 +23,14 @@ def save_json(data,fileName):
     jfile.close()
     return
 
+def save_text(data,fileName):
+    path = dirname(fileName)
+    makedirs(path, exist_ok=True)
+    jfile = open(fileName, "w")
+    jfile.write(data)
+    jfile.close()
+    return
+
 def duration_text(duration: timedelta):
     # Ensure that the duration is non-negative
     if duration < timedelta(0):

utils.py

@@ -0,0 +1,18 @@
+digraph G {
+    A [label="fetch-data-1"];
+    B [label="content" class="artifact"];
+    C [label="calculate-functions"];
+    D [label="stats" class="artifact"];
+    E [label="compute-statistics"];
+    F [label="result" class="artifact"];
+    G [label="generate-website"];
+    H [label="website" class="artifact"];
+    A -> B;
+    B -> C;
+    C -> D;
+    B -> E;
+    E -> F;
+    D -> G;
+    F -> G;
+    G -> H;
+}