Fill in example title, title_abbrev, and synopsis from Category templates. (#140)

* Fill in example title, title_abbrev, and synopsis from Category templates.

* Rename a couple methods.

Author: DavidSouther

README.md

@@ -62,7 +62,7 @@ python -m pip install -r requirements.txt
 python -m pip install -e .
 python -m mypy aws_doc_sdk_examples_tools
 python -m pytest -vv
-python -m black --check
+python -m black --check aws_doc_sdk_examples_tools
 ```
 
 ## Validation Extensions
@@ -87,18 +87,19 @@ There are two stages, testing and deployment.
 5. **Open a Draft PR to main branch**: Do not publish for review. Wait for checks/tests to pass on the PR.
 
 ### 2. Deployment
+
 1. **Run `stamp.sh --release` from the `main` branch to automatically perform the following actions**:
-    - Update the `setup.py` version.
-    - Create a tag in the -tools repository at the same SHA you identified earlier.
-        - stamp.sh will create the next [stamp](https://blog.aspect.build/versioning-releases-from-a-monorepo) (which is valid [semver](https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers)) number as appropriate for the changes in this release. e.g. `2024.40.2`.
-    - Push the new tag to `main`
+   - Update the `setup.py` version.
+   - Create a tag in the -tools repository at the same SHA you identified earlier.
+     - stamp.sh will create the next [stamp](https://blog.aspect.build/versioning-releases-from-a-monorepo) (which is valid [semver](https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers)) number as appropriate for the changes in this release. e.g. `2024.40.2`.
+   - Push the new tag to `main`
 1. **Update your testing PR branch**
-    - Remove SHA and add tag to [validate-doc-metadata.yml](https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/.github/workflows/validate-doc-metadata.yml)
-    - Remove the SHA from [.doc_gen/validation.yaml](https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/.doc_gen/validation.yaml)
-    - This is easily accomplished in the Github UI.
+   - Remove SHA and add tag to [validate-doc-metadata.yml](https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/.github/workflows/validate-doc-metadata.yml)
+   - Remove the SHA from [.doc_gen/validation.yaml](https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/.doc_gen/validation.yaml)
+   - This is easily accomplished in the Github UI.
 1. **Create a release**: Use the automated ["Create release from tag" button](https://github.com/awsdocs/aws-doc-sdk-examples-tools/releases/new) to create a new release with the new tag.
 1. **Perform internal update process**.
-    - See `update.sh` script in internal package.
+   - See `update.sh` script in internal package.
 
 ## Security
 

aws_doc_sdk_examples_tools/categories.py

@@ -3,8 +3,10 @@
 
 from __future__ import annotations
 
+import re
+
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 from dataclasses import dataclass, field
 
 from aws_doc_sdk_examples_tools import metadata_errors
@@ -13,6 +15,22 @@
 )
 
 
+def fake_gotmpl(tmpl: Optional[str], service: str, action: str):
+    if not tmpl:
+        return
+    values = {
+        ".ServiceEntity.Short": service,
+        ".Action": action,
+    }
+    return re.sub(
+        r"{{(?P<name>[.\w]+)}}",
+        lambda x: values[
+            x.groupdict()["name"]
+        ],  # This will be a KeyError if the replacement isn't in the values dict
+        tmpl,
+    )
+
+
 @dataclass
 class TitleInfo:
     title: Optional[str] = field(default=None)
@@ -44,6 +62,9 @@ def message(self):
         return "Category has no display value"
 
 
+empty_title_info = TitleInfo()
+
+
 @dataclass
 class Category:
     key: str
@@ -52,6 +73,23 @@ class Category:
     overrides: Optional[TitleInfo] = field(default=None)
     description: Optional[str] = field(default=None)
 
+    def evaluate(
+        self,
+        value: Optional[str],
+        field: Callable[[TitleInfo], Optional[str]],
+        service: str,
+        action: str,
+    ):
+        overrides = field(self.overrides or empty_title_info)
+        if overrides:
+            return fake_gotmpl(overrides, service, action)
+        if value:
+            return value
+        defaults = field(self.defaults or empty_title_info)
+        if defaults:
+            return fake_gotmpl(defaults, service, action)
+        return ""
+
     def validate(self, errors: MetadataErrors):
         if not self.display:
             errors.append(CategoryWithNoDisplayError(id=self.key))

aws_doc_sdk_examples_tools/doc_gen.py

@@ -150,6 +150,9 @@ def merge(self, other: "DocGen") -> MetadataErrors:
         self.snippet_files.update(other.snippet_files)
         self.cross_blocks.update(other.cross_blocks)
         self.extend_examples(other.examples.values(), warnings)
+        for name, category in other.categories.items():
+            if name not in self.categories:
+                self.categories[name] = category
 
         return warnings
 
@@ -318,6 +321,19 @@ def validate(self):
             self.root,
         )
 
+    def fill_missing_fields(self):
+        for example in self.examples.values():
+            service_id = example.service_main or next(
+                k for (k, _) in example.services.items()
+            )
+            action = (
+                next((k for k in example.services[service_id]), None)
+                or example.id.split("_", 1)[1]
+            )
+            example.fill_display_fields(
+                self.categories, self.services[service_id].short, action
+            )
+
     def stats(self):
         values = self.examples.values()
         initial = defaultdict(int)

aws_doc_sdk_examples_tools/doc_gen_cli.py

@@ -53,6 +53,9 @@ def main():
         unmerged_doc_gen = DocGen.from_root(Path(root))
         merged_doc_gen.merge(unmerged_doc_gen)
 
+    merged_doc_gen.validate()
+    merged_doc_gen.fill_missing_fields()
+
     if not args.skip_entity_expansion:
         # Replace entities
         merged_doc_gen.expand_entity_fields(merged_doc_gen)

aws_doc_sdk_examples_tools/doc_gen_cli_test.py

@@ -1,12 +1,14 @@
 import pytest
-from unittest.mock import patch, mock_open
+from unittest.mock import patch, mock_open, MagicMock
 from argparse import Namespace
 from pathlib import Path
 
+from .categories import Category
 from .doc_gen import DocGen, MetadataError, Example
 from .doc_gen_cli import main
-from .metadata import DocFilenames, Sdk, Language, SDKPageVersion, Version
-from .sdks import SdkVersion
+from .metadata import DocFilenames, Language, SDKPageVersion, Version
+from .sdks import Sdk, SdkVersion
+from .services import Service
 
 
 @pytest.fixture
@@ -46,6 +48,15 @@ def mock_doc_gen(mock_example):
         MetadataError(file="a.yaml", id="Error 1"),
         MetadataError(file="b.yaml", id="Error 2"),
     ]
+    doc_gen.categories = {"Actions": Category(key="Actions", display="Action")}
+    doc_gen.services = {
+        "medical-imaging": Service(
+            long="&AHIlong;",
+            short="&AHI;",
+            sort="HealthImaging",
+            version="medical-imaging-2023-07-19",
+        )
+    }
     doc_gen.sdks = {
         "JavaScript": Sdk(
             name="JavaScript",
@@ -60,6 +71,7 @@ def mock_doc_gen(mock_example):
 
 @pytest.fixture
 def patched_environment(mock_doc_gen):
+    mock_doc_gen.validate = MagicMock()
     with patch("argparse.ArgumentParser.parse_args") as mock_parse_args, patch(
         "aws_doc_sdk_examples_tools.doc_gen.DocGen.empty", return_value=mock_doc_gen
     ), patch("aws_doc_sdk_examples_tools.doc_gen.DocGen.from_root"), patch(

aws_doc_sdk_examples_tools/doc_gen_test.py

@@ -9,8 +9,10 @@
 from pathlib import Path
 import json
 
-from .metadata_errors import MetadataErrors, MetadataError
+from .categories import Category, TitleInfo
 from .doc_gen import DocGen, DocGenEncoder
+from .metadata import Example
+from .metadata_errors import MetadataErrors, MetadataError
 from .sdks import Sdk, SdkVersion
 from .services import Service, ServiceExpanded
 from .snippets import Snippet
@@ -115,7 +117,25 @@ def sample_doc_gen() -> DocGen:
             )
         },
         snippet_files={"test.py"},
-        examples={},
+        examples={
+            "s3_PutObject": Example(
+                "s3_PutObject",
+                file=Path("filea.txt"),
+                languages={},
+                services={"s3": set(["PutObject"])},
+            )
+        },
+        categories={
+            "Actions": Category(
+                "Actions",
+                "Actions",
+                defaults=TitleInfo(
+                    title="<code>{{.Action}}</code>",
+                    synopsis="{{.ServiceEntity.Short}} {{.Action}}",
+                ),
+                overrides=TitleInfo(title_abbrev="ExcerptPartsUsage"),
+            )
+        },
         cross_blocks={"test_block"},
     )
 
@@ -183,7 +203,29 @@ def test_doc_gen_encoder(sample_doc_gen: DocGen):
 
     # Verify examples (empty in this case)
     assert "examples" in decoded
-    assert decoded["examples"] == {}
+    assert decoded["examples"] == {
+        "s3_PutObject": {
+            "category": None,
+            "doc_filenames": None,
+            "file": "filea.txt",
+            "guide_topic": None,
+            "id": "s3_PutObject",
+            "languages": {},
+            "service_main": None,
+            "services": {
+                "s3": {
+                    "__set__": [
+                        "PutObject",
+                    ],
+                },
+            },
+            "source_key": None,
+            "synopsis": "",
+            "synopsis_list": [],
+            "title": "",
+            "title_abbrev": "",
+        },
+    }
 
 
 def test_doc_gen_load_snippets():
@@ -195,3 +237,11 @@ def test_doc_gen_load_snippets():
     doc_gen.collect_snippets()
     assert doc_gen.snippet_files == set(["snippet_file.txt"])
     assert doc_gen.snippets["snippet_file.txt"].code == "Line A\nLine C\n"
+
+
+def test_fill_fields(sample_doc_gen: DocGen):
+    sample_doc_gen.fill_missing_fields()
+    example = sample_doc_gen.examples["s3_PutObject"]
+    assert example.title == "<code>PutObject</code>"
+    assert example.title_abbrev == "ExcerptPartsUsage"
+    assert example.synopsis == "&S3; PutObject"

aws_doc_sdk_examples_tools/metadata.py

@@ -6,21 +6,19 @@
 
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, Dict, Literal, List, Optional, Set, Union, Iterable
+from typing import Dict, Literal, List, Optional, Set, Iterable
 from os.path import splitext
 from pathlib import Path
 
 from . import metadata_errors
+from .categories import Category
 from .metadata_errors import (
     MetadataErrors,
     MetadataParseError,
     ExampleMergeMismatchedId,
     ExampleMergeMismatchedLanguage,
     ExampleMergeConflict,
 )
-from .project_validator import ValidationConfig
-from .services import Service
-from .sdks import Sdk
 
 
 @dataclass
@@ -132,9 +130,9 @@ class Example:
     id: str
     file: Optional[Path]
     languages: Dict[str, Language]
-    # Human readable title. TODO: Defaults to slug-to-title of the ID if not provided.
+    # Human readable title.
     title: Optional[str] = field(default="")
-    # Used in the TOC. TODO: Defaults to slug-to-title of the ID if not provided.
+    # Used in the TOC.
     title_abbrev: Optional[str] = field(default="")
     synopsis: Optional[str] = field(default="")
     # String label categories. Categories inferred by cross-service with multiple services, and can be whatever else it wants. Controls where in the TOC it appears.
@@ -151,6 +149,30 @@ class Example:
     synopsis_list: List[str] = field(default_factory=list)
     source_key: Optional[str] = field(default=None)
 
+    def fill_display_fields(self, categories: Dict[str, Category], service, action):
+        category = self.choose_category(categories)
+        if category:
+            self.title = category.evaluate(
+                self.title, lambda x: x.title, service, action
+            )
+            self.title_abbrev = category.evaluate(
+                self.title_abbrev, lambda x: x.title_abbrev, service, action
+            )
+            self.synopsis = category.evaluate(
+                self.synopsis, lambda x: x.synopsis, service, action
+            )
+
+    def choose_category(self, categories: Dict[str, Category]) -> Optional[Category]:
+        """Find a category for an example. This logic is taken from directories and zexii.
+
+        Original Zexii code at https://code.amazon.com/packages/GoAmzn-AWSDocsCodeExampleDocBuilder/blobs/1321fffadd8ff02e6acbae4a1f42b81006cdfa72/--/zexi/zonbook/category.go#L31-L50.
+        """
+        if self.category in categories:
+            return categories[self.category]
+        if len(self.services) == 1:
+            return categories["Actions"]
+        return categories["Scenarios"]
+
     def merge(self, other: Example, errors: MetadataErrors):
         """Combine `other` Example into self example.
 
@@ -184,7 +206,7 @@ def merge(self, other: Example, errors: MetadataErrors):
                     err.id = self.id
                     err.file = self.file
                     if hasattr(err, "other_file"):
-                        err.other_file = other.file
+                        err.other_file = other.file  # type: ignore
                 errors.extend(merge_errs)
 
     def validate(self, errors: MetadataErrors, root: Path):