feat(docs): Formal Schemas for Signed Docs

.config/dictionaries/project.dic

@@ -51,6 +51,7 @@ crontabs
 crontagged
 csprng
 cstring
+cuelang
 dalek
 dashmap
 Datelike
@@ -123,6 +124,7 @@ jorm
 jormungandr
 Jörmungandr
 jsonschema
+Justfile
 kiduri
 lcov
 Leay
@@ -200,6 +202,7 @@ pubkey
 publickey
 pubspec
 pwrite
+pytest
 qpsg
 quic
 rapidoc

.gitignore

@@ -84,3 +84,7 @@ $RECYCLE.BIN/
 # Dart stuff
 /pubspec.lock
 /.dart_tool/**/*
+
+# Python stuff
+.pytest_cache
+__pycache__

.markdownlint-cli2.jsonc

@@ -10,7 +10,8 @@
   "ignores": [
     ".config/dictionaries/**",
     "**/target/**",
-    "**/.dart_tool/**"
+    "**/.dart_tool/**",
+    "**/.pytest_cache/**"
   ],
   // Set standard config options in `/.markdownlint.jsonc`
   "config": {

.vscode/extensions.json

@@ -24,5 +24,8 @@
         "dtsvet.vscode-wasm",
         "terrastruct.d2",
         "fill-labs.dependi",
+        "nefrob.vscode-just-syntax",
+        "charliermarsh.ruff",
+        "ms-python.python",
     ]
 }

Justfile

@@ -15,6 +15,19 @@ check-spelling:
     earthly +check-spelling
 
 
+# Fix and Check Markdown files
+format-python-code:
+    ruff check --select I --fix .
+    ruff format .
+
+# Fix and Check Markdown files
+lint-python:
+    ruff check .
+
+# generates specifications data
+gen_specs:
+    just specs/pre-push
+
 # Pre Push Checks - intended to be run by a git pre-push hook.
-pre-push: check-markdown check-spelling
+pre-push: gen_specs check-markdown check-spelling format-python-code lint-python
     just rust/pre-push

docs/Earthfile

@@ -4,6 +4,7 @@ IMPORT github.com/input-output-hk/catalyst-ci/earthly/docs:v3.3.0 AS docs-ci
 
 
 IMPORT .. AS repo
+IMPORT ../specs AS specs
 
 # Copy all the source we need to build the docs
 src:
@@ -13,6 +14,9 @@ src:
     # Now copy into that any artifacts we pull from the builds.
     COPY --dir repo+repo-docs/repo /docs/includes
 
+    # Copy our generated Signed Document Specification data.
+    COPY specs+src/signed_doc.json /docs/includes
+
 
 # Build the docs here.
 docs:

docs/macros/__init__.py

@@ -0,0 +1,54 @@
+from .include import inc_file
+from .signed_docs import (
+    cose_header_parameters,
+    doc_type_details,
+    doc_type_summary,
+    external_links,
+    signed_doc_details,
+)
+
+
+def define_env(env):
+    """
+    This is the hook for defining variables, macros and filters
+    """
+
+    @env.macro
+    def include_file(filename, start_line=0, end_line=None, indent=None):
+        # Provided by the base mkdocs config.
+        return inc_file(env, filename, start_line, end_line, indent)
+
+    @env.macro
+    def insert_doc_type_summary():
+        try:
+            return doc_type_summary(env)
+        except Exception as exc:
+            return f"Macro Expansion Error: {exc}"
+
+    @env.macro
+    def insert_doc_type_details():
+        try:
+            return doc_type_details(env)
+        except Exception as exc:
+            return f"Macro Expansion Error: {exc}"
+
+    @env.macro
+    def insert_cose_header_parameters():
+        try:
+            return cose_header_parameters(env)
+        except Exception as exc:
+            return f"Macro Expansion Error: {exc}"
+
+    @env.macro
+    def insert_signed_doc_details(name):
+        try:
+            return signed_doc_details(env, name)
+        except Exception as exc:
+            return f"Macro Expansion Error: {exc}"
+
+    @env.macro
+    def insert_external_links():
+        try:
+            return external_links(env)
+        except Exception as exc:
+            return f"Macro Expansion Error: {exc}"

docs/macros/signed_docs.py

@@ -0,0 +1,201 @@
+import json
+import os
+
+# import re
+# import textwrap
+
+if __name__ == "__main__":
+    SIGNED_DOCS_SPECS = "signed_doc.json"
+else:
+    SIGNED_DOCS_SPECS = "includes/signed_doc.json"
+
+
+def uuid_as_cbor(uuid):
+    return f"37(h'{uuid.replace('-', '')}')"
+
+
+def get_signed_doc_data(env):
+    """
+    Load the Signed Document Data from its json file.
+    """
+    full_filename = os.path.join(env.project_dir, SIGNED_DOCS_SPECS)
+
+    with open(full_filename, "r") as f:
+        return json.load(f)
+
+
+def doc_type_summary(env):
+    """
+    Generate a Document Base Type Summary from the Document Specifications Data
+    """
+
+    doc_data = get_signed_doc_data(env)
+    doc_types = doc_data["base_types"]
+
+    doc_type_summary = """
+| Base Type | [UUID] | [CBOR] |
+| :--- | :--- | :--- |
+"""
+
+    for k in doc_types:
+        doc_type_summary += (
+            f"| {k} | `{doc_types[k]}` | `{uuid_as_cbor(doc_types[k])}` |\n"
+        )
+
+    return doc_type_summary
+
+
+def name_for_uuid(doc_types, uuid):
+    """
+    Get the name for a document base type, given its uuid
+    """
+    for k in doc_types:
+        if doc_types[k] == uuid:
+            return k
+    return "Unknown"
+
+
+def name_to_spec_link(name, ref=None):
+    """
+    Create a link to a document type, and an optional ref inside the document.
+    """
+    link = "./../catalyst_docs/" + name.lower().replace(" ", "_") + ".md"
+    if ref is not None:
+        link += f"#{ref}"
+    return link
+
+
+def base_types(docs, doc_types, name):
+    types = docs[name]["type"]
+    type_names = ""
+    for sub_type in types:
+        type_names += name_for_uuid(doc_types, sub_type) + "/"
+    return type_names[:-1]
+
+
+def types_as_cbor(docs, name):
+    types = docs[name]["type"]
+    type_names = "["
+    for sub_type in types:
+        type_names += uuid_as_cbor(sub_type) + ",<br/>"
+    return type_names[:-6] + "]"
+
+
+def doc_type_details(env):
+    """
+    Generate a Document Type Detailed Summary from the Document Specifications Data
+    """
+
+    doc_data = get_signed_doc_data(env)
+    doc_types = doc_data["base_types"]
+    docs = doc_data["docs"]
+
+    doc_type_details = """
+| Document Type | Base Types | [CBOR] | Specification |
+| :--- | :--- | :--- | :--- |
+"""
+
+    for k in docs:
+        doc_type_details += f"| {k} | {base_types(docs, doc_types, k)} | {types_as_cbor(docs, k)} | [Specification]({name_to_spec_link(k)}) | \n"
+
+    return doc_type_details
+
+
+def external_links(env):
+    """
+    Insert External Links we might have used in descriptions.
+    """
+    doc_data = get_signed_doc_data(env)
+    links = doc_data["documentationLinks"]
+
+    link_display = ""
+    for name in links:
+        link_display += f"[{name}]: {links[name]}\n"
+
+    return link_display
+
+
+def metadata_fields(env, doc_name=None):
+    """
+    Display Metadata fields for the default set, or a specific document.
+    """
+    doc_data = get_signed_doc_data(env)
+    if doc_name is not None:
+        fields = doc_data["docs"][doc_name]["metadata"]
+        field_title_level = "###"
+    else:
+        fields = doc_data["metadata"]
+        field_title_level = "##"
+
+    order = doc_data["metadata_order"]
+
+    # make sure every field is listed in the ordering
+    for field_name in fields:
+        if field_name not in order:
+            order += field_name
+
+    field_display = ""
+    for field_name in order:
+        field = fields[field_name]
+        field_display += f"""
+{field_title_level} `{field_name}`
+
+| Parameter | Value |
+| --- | --- |
+| Required | {field["required"]} |
+"""
+        if field["required"] != "excluded":
+            field_display += f"| Format | {field['format']} |\n"
+        if "multiple" in field:
+            field_display += f"| Multiple References | {field['multiple']} |\n"
+        if "type" in field:
+            ref_heading = "Valid References"
+            ref_doc_names = field["type"]
+            if isinstance(ref_doc_names, str):
+                ref_doc_names = [ref_doc_names]
+            for ref_doc in ref_doc_names:
+                field_display += f"| {ref_heading} | {ref_doc} |\n"
+                ref_heading = ""
+
+        field_display += f"""
+{field["description"]}
+
+{field_title_level}# Validation
+
+{field["validation"]}
+"""
+    return field_display
+
+
+def signed_doc_details(env, name):
+    """
+    Generate Signed Document Detailed Documentation Page.
+    """
+    return name + "\n" + "test\n"
+
+
+# run as a program to debug the macros
+if __name__ == "__main__":
+
+    class env:
+        project_dir = "/home/steven/Development/iohk/catalyst-libs/specs"
+
+    print()
+    print("### DOC TYPE DETAILS ###")
+    print(doc_type_details(env))
+
+    print()
+    print("### DOC TYPE SUMMARY ###")
+    print(doc_type_summary(env))
+
+    print()
+    print("### COSE HEADER PARAMETERS ###")
+    print(cose_header_parameters(env))
+
+    print()
+    print("### EXTERNAL LINKS ###")
+    print(external_links(env))
+
+    print()
+    print("### GLOBAL METADATA ###")
+    print(metadata_fields(env))

docs/src/architecture/08_concepts/catalyst_docs/.pages

@@ -1,5 +1 @@
 title: Catalyst Documents
-arrange:
-- proposal.md
-- review.md
-- comment.md

docs/src/architecture/08_concepts/catalyst_docs/brand_parameters.md

@@ -0,0 +1 @@
+# {{ insert_signed_doc_details( "Brand Parameters" ) }}

docs/src/architecture/08_concepts/catalyst_docs/campaign_parameters.md

@@ -0,0 +1 @@
+# {{ insert_signed_doc_details( "Campaign Parameters" ) }}

docs/src/architecture/08_concepts/catalyst_docs/category_parameters.md

@@ -0,0 +1 @@
+# {{ insert_signed_doc_details( "Category Parameters" ) }}

docs/src/architecture/08_concepts/catalyst_docs/comment_action_document.md

@@ -0,0 +1 @@
+# {{ insert_signed_doc_details( "Comment Action Document" ) }}

docs/src/architecture/08_concepts/catalyst_docs/election_parameters.md

@@ -0,0 +1 @@
+# {{ insert_signed_doc_details( "Election Parameters" ) }}