feat(docs): Formal Schemas for Signed Docs

.config/dictionaries/project.dic

@@ -51,6 +51,7 @@ crontabs
 crontagged
 csprng
 cstring
+cuelang
 dalek
 dashmap
 Datelike
@@ -123,6 +124,7 @@ jorm
 jormungandr
 Jörmungandr
 jsonschema
+Justfile
 kiduri
 lcov
 Leay

.vscode/extensions.json

@@ -24,5 +24,6 @@
         "dtsvet.vscode-wasm",
         "terrastruct.d2",
         "fill-labs.dependi",
+        "nefrob.vscode-just-syntax",
     ]
 }

docs/Earthfile

@@ -4,6 +4,7 @@ IMPORT github.com/input-output-hk/catalyst-ci/earthly/docs:v3.3.0 AS docs-ci
 
 
 IMPORT .. AS repo
+IMPORT ../specs AS specs
 
 # Copy all the source we need to build the docs
 src:
@@ -13,6 +14,9 @@ src:
     # Now copy into that any artifacts we pull from the builds.
     COPY --dir repo+repo-docs/repo /docs/includes
 
+    # Copy our generated Signed Document Specification data.
+    COPY specs+src/signed_doc.json /docs/includes
+
 
 # Build the docs here.
 docs:

docs/macros/__init__.py

@@ -0,0 +1,20 @@
+from .include import inc_file
+from .signed_docs import (doc_type_summary, doc_type_details)
+
+def define_env(env):
+    """
+    This is the hook for defining variables, macros and filters
+    """
+
+    @env.macro
+    def include_file(filename, start_line=0, end_line=None, indent=None):
+        # Provided by the base mkdocs config.
+        return inc_file(env, filename, start_line, end_line, indent)
+
+    @env.macro
+    def insert_doc_type_summary():
+        return doc_type_summary(env)
+
+    @env.macro
+    def insert_doc_type_details():
+        return doc_type_details(env)

docs/macros/signed_docs.py

@@ -0,0 +1,106 @@
+import os
+import re
+import textwrap
+import json
+
+#SIGNED_DOCS_SPECS="signed_doc.json"
+SIGNED_DOCS_SPECS="includes/signed_doc.json"
+
+def uuid_as_cbor(uuid):
+    return f"37(h'{uuid.replace('-', '')}')"
+
+
+def get_signed_doc_data(env):
+    """
+    Load the Signed Document Data from its json file.
+    """
+    full_filename = os.path.join(env.project_dir, SIGNED_DOCS_SPECS)
+
+    with open(full_filename, "r") as f:
+        return json.load(f)
+
+def doc_type_summary(env):
+    """
+    Generate a Document Base Type Summary from the Document Specifications Data
+    """
+
+    try:
+        doc_data = get_signed_doc_data(env)
+        doc_types = doc_data["base_types"]
+
+        doc_type_summary = """
+| Base Type | [UUID] | [CBOR] |
+| :--- | :--- | :--- |
+"""
+
+        for k in doc_types:
+            doc_type_summary += f"| {k} | `{doc_types[k]}` | `{uuid_as_cbor(doc_types[k])}` |\n"
+
+        return doc_type_summary
+    except Exception as exc:
+        return f"{exc}"
+
+
+def name_for_uuid(doc_types, uuid):
+    """
+    Get the name for a document base type, given its uuid
+    """
+    for k in doc_types:
+        if doc_types[k] == uuid:
+            return k
+    return "Unknown"
+
+
+def name_to_spec_link(name,ref=None):
+    """
+    Create a link to a document type, and an optional ref inside the document.
+    """
+    link = "./../catalyst_docs/"+name.lower().replace(' ','_') + ".md"
+    if ref is not None:
+        link += f"#{ref}"
+    return link
+
+def base_types(docs, doc_types, name):
+    types = docs[name]["type"]
+    type_names = ""
+    for sub_type in types:
+        type_names += name_for_uuid(doc_types, sub_type) + "/"
+    return type_names[:-1]
+
+def types_as_cbor(docs, name):
+    types = docs[name]["type"]
+    type_names = "["
+    for sub_type in types:
+        type_names += uuid_as_cbor(sub_type) + ",<br/>"
+    return type_names[:-6] + "]"
+
+
+def doc_type_details(env):
+    """
+    Generate a Document Type Detailed Summary from the Document Specifications Data
+    """
+
+    try:
+        doc_data = get_signed_doc_data(env)
+        doc_types = doc_data["base_types"]
+        docs = doc_data["docs"]
+
+        doc_type_details = """
+| Document Type | Base Types | [CBOR] | Specification |
+| :--- | :--- | :--- | :--- |
+"""
+
+        for k in docs:
+            doc_type_details += f"| {k} | {base_types(docs,doc_types,k)} | {types_as_cbor(docs,k)} | [Specification]({name_to_spec_link(k)}) | \n"
+
+        return doc_type_details
+    except Exception as exc:
+        return f"{exc}"
+
+#class env:
+#    project_dir = "/home/steven/Development/iohk/catalyst-libs/specs"
+
+#if __name__ == '__main__':
+
+#    print(doc_type_details(env))    
+#    print(doc_type_summary(env))

docs/src/architecture/08_concepts/catalyst_docs/category_parameters.md

@@ -0,0 +1 @@
+# Category Parameters

docs/src/architecture/08_concepts/catalyst_docs/comment_action_document.md

@@ -0,0 +1 @@
+# Comment Action Document

docs/src/architecture/08_concepts/catalyst_docs/proposal_comment.md

@@ -0,0 +1 @@
+# Proposal Comment

docs/src/architecture/08_concepts/catalyst_docs/proposal_comment_meta_template.md

@@ -0,0 +1 @@
+# Comment Action Meta Template

docs/src/architecture/08_concepts/catalyst_docs/proposal_comment_template.md

@@ -0,0 +1 @@
+# Proposal Comment Template

docs/src/architecture/08_concepts/catalyst_docs/proposal_meta_template.md

@@ -0,0 +1 @@
+# Proposal Meta Template

docs/src/architecture/08_concepts/catalyst_docs/proposal_moderation_action.md

@@ -0,0 +1 @@
+# Proposal Moderation Action

docs/src/architecture/08_concepts/catalyst_docs/proposal_submission_action.md

@@ -0,0 +1 @@
+# Proposal Submission Action

docs/src/architecture/08_concepts/catalyst_docs/proposal_template.md

@@ -0,0 +1 @@
+# Proposal Template

docs/src/architecture/08_concepts/signed_doc/meta.md

@@ -10,6 +10,7 @@
 
 * [Metadata Fields List](#metadata-fields-list)
   * [`ref` Document Reference](#ref-document-reference)
+    * [Validation](#validation)
   * [`ref_hash` Secured Document Reference](#ref_hash-secured-document-reference)
   * [`template` Template Reference](#template-template-reference)
   * [`reply` Reply Reference](#reply-reply-reference)
@@ -33,6 +34,12 @@ If the `ref` is a [CBOR] array, it has the form `[<id>,<ver>]` where:
 * `<id>` - the [UUID] v7 of the referenced documents [`id`](./spec.md#id).
 * `<ver>` - the [UUID] v7 of the referenced documents [`ver`](./spec.md#ver).
 
+### Validation
+
+For any document type, `ref` can refer to only 1 other document type which must be different than
+the type of document `ref` appears in.
+For example `ref` for a Proposal Comment Document, is always a Proposal type document.
+
 ## `ref_hash` Secured Document Reference
 
 This is a cryptographically secured reference to another document.

docs/src/architecture/08_concepts/signed_doc/types.md

@@ -1,26 +1,21 @@
 # Document Types Table
 
-| [UUID]                               | [CBOR]                                    | Type Description             | Payload Type                      | Specification Link                                                             |
-| ------------------------------------ | ----------------------------------------- | ---------------------------- | --------------------------------- | ------------------------------------------------------------------------------ |
-| 7808d2ba-d511-40af-84e8-c0d1625fdfdc | `37(h'7808d2bad51140af84e8c0d1625fdfdc')` | Proposal Document            | [Brotli] Compressed [JSON]        | [Proposal Spec](./../catalyst_docs/proposal.md#proposal-document)              |
-| 0ce8ab38-9258-4fbc-a62e-7faa6e58318f | `37(h'0ce8ab3892584fbca62e7faa6e58318f')` | Proposal Template            | [Brotli] Compressed [JSON Schema] | [Proposal Template Spec](./../catalyst_docs/proposal.md#proposal-template)     |
-| b679ded3-0e7c-41ba-89f8-da62a17898ea | `37(h'b679ded30e7c41ba89f8da62a17898ea')` | Comment Document             | [Brotli] Compressed [JSON]        | [Comment Spec](./../catalyst_docs/comment.md#comment-document)                 |
-| 0b8424d4-ebfd-46e3-9577-1775a69d290c | `37(h'0b8424d4ebfd46e395771775a69d290c')` | Comment Template             | [Brotli] Compressed [JSON Schema] | [Comment Template Spec](./../catalyst_docs/comment.md#comment-template)        |
-| e4caf5f0-098b-45fd-94f3-0702a4573db5 | `37(h'e4caf5f0098b45fd94f30702a4573db5')` | Review Document              | [Brotli] Compressed [JSON]        | [Review Spec](./../catalyst_docs/review.md#review-document)                    |
-| ebe5d0bf-5d86-4577-af4d-008fddbe2edc | `37(h'ebe5d0bf5d864577af4d008fddbe2edc')` | Review Template              | [Brotli] Compressed [JSON Schema] | [Review Template Spec](./../catalyst_docs/review.md#review-template)           |
-| 48c20109-362a-4d32-9bba-e0a9cf8b45be | `37(h'48c20109362a4d329bbae0a9cf8b45be')` | Category Parameters Document | [Brotli] Compressed [JSON]        | *TBD*                                                                          |
-| 65b1e8b0-51f1-46a5-9970-72cdf26884be | `37(h'65b1e8b051f146a5997072cdf26884be')` | Category Parameters Template | [Brotli] Compressed [JSON Schema] | *TBD*                                                                          |
-| 0110ea96-a555-47ce-8408-36efe6ed6f7c | `37(h'0110ea96a55547ce840836efe6ed6f7c')` | Campaign Parameters Document | [Brotli] Compressed [JSON]        | *TBD*                                                                          |
-| 7e8f5fa2-44ce-49c8-bfd5-02af42c179a3 | `37(h'7e8f5fa244ce49c8bfd502af42c179a3')` | Campaign Parameters Template | [Brotli] Compressed [JSON Schema] | *TBD*                                                                          |
-| 3e4808cc-c86e-467b-9702-d60baa9d1fca | `37(h'3e4808ccc86e467b9702d60baa9d1fca')` | Brand Parameters Document    | [Brotli] Compressed [JSON]        | *TBD*                                                                          |
-| fd3c1735-80b1-4eea-8d63-5f436d97ea31 | `37(h'fd3c173580b14eea8d635f436d97ea31')` | Brand Parameters Template    | [Brotli] Compressed [JSON Schema] | *TBD*                                                                          |
-| 5e60e623-ad02-4a1b-a1ac-406db978ee48 | `37(h'5e60e623ad024a1ba1ac406db978ee48')` | Proposal Action Document     | *TBD*                             | *TBD*                                                                          |
-| 8de5586c-e998-4b95-8742-7be3c8592803 | `37(h'8DE5586CE9984B9587427BE3C8592803')` | Public Vote Tx V2            | [Brotli] Compressed [CBOR]        | [Public Vote Tx V2 Spec](./../catalyst_voting/v2.md#public-vote)               |
-| e78ee18d-f380-44c1-a852-80aa6ecb07fe | `37(h'E78EE18DF38044C1A85280AA6ECB07FE')` | Private Vote Tx V2           | [Brotli] Compressed [CBOR]        | [Private Vote Tx V2 Spec](./../catalyst_voting/v2.md#private-vote)             |
-| d9e7e6ce-2401-4d7d-9492-f4f7c64241c3 | `37(h'D9E7E6CE24014D7D9492F4F7C64241C3')` | Immutable Ledger Block       | [Brotli] Compressed [CBOR]        | [Immutable Ledger Block Spec](./../immutable_ledger/ledger.md#block-structure) |
-
-[JSON Schema]: https://json-schema.org/draft-07
-[JSON]: https://datatracker.ietf.org/doc/html/rfc7159
-[Brotli]: https://datatracker.ietf.org/doc/html/rfc7932
+## Document Base Types
+
+All Document Types are defined by composing these base document types:
+
+{{ insert_doc_type_summary() }}
+
+## Document Types
+
+All Defined Document Types
+
+{{ insert_doc_type_details() }}
+
+<!-- markdownlint-disable MD053 -->
+
+<!-- [JSON Schema]: https://json-schema.org/draft-07 -->
+<!-- [JSON]: https://datatracker.ietf.org/doc/html/rfc7159 -->
+<!-- [Brotli]: https://datatracker.ietf.org/doc/html/rfc7932 -->
 [CBOR]: https://datatracker.ietf.org/doc/html/rfc8610
 [UUID]: https://www.rfc-editor.org/rfc/rfc9562.html

docs/src/architecture/08_concepts/signed_doc/validation.md

@@ -0,0 +1,21 @@
+---
+Title: Catalyst Signed Documents - Validation Summary
+Category: Catalyst
+Status: Proposed
+Authors:
+    - Steven Johnson <[email protected]>
+Implementors:
+    - Catalyst Fund 14
+Discussions: []
+Created: 2024-12-29
+License: CC-BY-4.0
+---
+
+## Validation
+
+This is a list of Metadata fields that apply to each document type, and how they are to be validated.
+
+| Document Type |`ref`  | `ref_hash` | `template` | `reply` | `section` | `brand_id` | `campaign_id` | `category_id` |
+| ------------- | ----- | ---------- | --------- | ------- | --------- | ---------- | ------------ | ----------------- |
+| Proposal      | None  | `p`        | `p`       | `p`     | `p`       | `p`        | `p`          | `p`               |
+| Comment       | `c`    | `p`   | `p`        | `p`       | `p`     | `p`       | `p`        | `p`          |

specs/Earthfile

@@ -0,0 +1,37 @@
+VERSION 0.8
+
+# cspell: words cuelang
+
+# Get cue binary
+cue-bin:
+    FROM cuelang/cue:0.12.0
+    SAVE ARTIFACT /usr/bin/cue
+
+
+# builder - debian plus cue binary
+builder:
+    FROM debian:stable-20250203-slim
+    COPY +cue-bin/cue /usr/bin/cue
+
+# Copy all the source we need to build the docs
+src:
+    FROM +builder
+
+    WORKDIR /src
+    COPY . .
+    ENV CUE_EXPERIMENT="embed"
+
+    SAVE ARTIFACT signed_doc.json
+
+
+check:
+    FROM +src
+    RUN cue fmt --check --files . 
+    RUN cue vet ./signed_docs/docs:signed_docs  signed_doc.json
+
+#  Regenerate - using the builder
+regenerate:
+    FROM +src
+
+    RUN cue export -f -s ./signed_docs/docs:signed_docs --out json --outfile signed_doc.json
+    SAVE ARTIFACT --keep-ts signed_doc.json AS LOCAL .

specs/Justfile

@@ -0,0 +1,30 @@
+# use with https://github.com/casey/just
+#
+# Developer convenience functions
+
+# required to get cuelang to enable file embedding
+export  CUE_EXPERIMENT := "embed"
+default:
+    @just --list --unsorted
+
+# Fiz (where possible) and format cue files
+format:
+    cue fix ./signed_docs/docs:signed_docs
+    cue fmt --files .
+
+# Check the signed document cue files are valid.
+check: format
+    cue vet ./signed_docs/docs:signed_docs -c
+
+# Fix and Check Markdown files
+regenerate: check
+    # Make sure keys are sorted so its both reproducible, AND diffs easily.
+    cue export -f -s ./signed_docs/docs:signed_docs --out json | jq -S > signed_doc.json
+
+# Validate the generated signed_docs.json is correct against the cue schema.
+validate:
+    cue vet ./signed_docs/docs:signed_docs  signed_doc.json
+
+# Pre Push Checks - intended to be run by a git pre-push hook.
+pre-push: regenerate validate
+

specs/blueprint.cue

@@ -0,0 +1,2 @@
+version: "1.0.0"
+project: name: "catalyst-data-specifications"

specs/cue.mod/module.cue

@@ -0,0 +1,4 @@
+module: "github.com/input-output-hk/catalyst-libs/specs"
+language: {
+	version: "v0.11.2"
+}

specs/generic/optional.cue

@@ -0,0 +1,7 @@
+package optional
+
+// Is a field Required, Optional or Excluded/Unused
+#field:
+	"yes" |
+	"optional" |
+	*"excluded"

specs/generic/uuid.cue

@@ -0,0 +1,8 @@
+// UUID Definitions
+package uuid
+
+// A UUIDv4 formatted string regex
+#v4: =~"^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-4[0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$"
+
+// A uuidv7 formatted string regex
+#v7: =~"^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{7}-4[0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$"