Add -tools build command to modularize internal build processing.

Author: DavidSouther

aws_doc_sdk_examples_tools/builder.py

@@ -0,0 +1,95 @@
+from dataclasses import dataclass, field
+import logging
+from pathlib import Path
+import re
+from typing import List, Optional, cast
+
+from aws_doc_sdk_examples_tools.doc_gen import DocGen
+from aws_doc_sdk_examples_tools.fs import Fs, PathFs
+from aws_doc_sdk_examples_tools.snippets import write_snippets
+
+logging.basicConfig(level=logging.INFO)
+
+logger = logging.getLogger(Path(__file__).name)
+
+
+IAM_PATTERN = re.compile(r'"Version"\s*:\s*"20(08|12)-10-17"\s*,')
+IAM_PATTERN_SLASHES = re.compile(r'\\"Version\\"\s*:\s*\\"20(08|12)-10-17\\"\s*,')
+# This exciting set of unicode characters is the expanded version of the &IAM-2025-waiver; entity.
+IAM_WAIVER = '"Version":"2012-10-17",\u0009\u0009\u0020\u0009\u0020\u0009\u0020'
+IAM_WAIVER_SLASHES = '\\"Version\\":\\"2012-10-17\\",\u0009\u0009\u0020\u0009\u0020\u0009\u0020'
+
+
+def _iam_replace_all(source: Optional[str]):
+    if source:
+        return IAM_PATTERN_SLASHES.subn(IAM_WAIVER_SLASHES, IAM_PATTERN.subn(IAM_WAIVER, source)[0])[0]
+    return None
+
+
+def _iam_fixup_metadata(meta_folder: Path):
+    for meta_path in meta_folder.glob("**/*_metadata.yaml"):
+        with meta_path.open("r") as meta_file:
+            contents = meta_file.read()
+        contents = cast(str, _iam_replace_all(contents))
+        with meta_path.open("w") as meta_file:
+            meta_file.write(contents)
+
+
+def _iam_fixup_docgen(doc_gen: DocGen) -> DocGen:
+    # For performance, do this mutably, but keep the signature open to making DocGen frozen
+    for snippet in doc_gen.snippets.values():
+        snippet.code = cast(str, _iam_replace_all(snippet.code))
+    return doc_gen
+
+
+@dataclass
+class Builder:
+    root: Path
+    dest: Path
+    doc_gen_folder = ".doc_gen"
+    snippet_files_folder = "snippet_files"
+    fs: Fs = field(default_factory=PathFs)
+    _doc_gen: DocGen = field(init=False)
+
+    def __post_init__(self):
+        self._doc_gen = DocGen.from_root(self.root)
+
+    def copy_doc_gen(self):
+        tmp_mirror_doc_gen = self.root / self.doc_gen_folder
+        mirror_doc_gen = self.dest / self.doc_gen_folder
+        logger.info(f"Moving cloned files into package from {tmp_mirror_doc_gen} to {mirror_doc_gen}")
+        try:
+            self.fs.copytree(tmp_mirror_doc_gen, mirror_doc_gen)
+        except Exception as e:
+            logger.error(f"Failed copy directory {tmp_mirror_doc_gen} to {mirror_doc_gen}\n{e}")
+            logger.error(e)
+            raise
+
+    def write_snippets(self):
+        write_snippets(self.dest / self.doc_gen_folder / self.snippet_files_folder, self._doc_gen.snippets)
+
+    def run(self):
+        logger.debug("Copying docgen files...")
+        self.copy_doc_gen()
+        _iam_fixup_metadata(self.dest)
+        logger.debug("Collecting snippets...")
+        self._doc_gen.collect_snippets()
+        self._doc_gen = _iam_fixup_docgen(self._doc_gen)
+        logger.debug("Writing snippets...")
+        self.write_snippets()
+
+
+def main():
+    from argparse import ArgumentParser
+    argparse = ArgumentParser()
+    argparse.add_argument('--root', type=Path, default=Path())
+    argparse.add_argument('--dest', type=Path)
+    # Load a config from somewhere to get doc_gen_folder and snippets_files_folder
+    args = argparse.parse_args()
+
+    builder = Builder(args.root, args.dest)
+    builder.run()
+
+
+if __name__ == "__main__":
+    main()

aws_doc_sdk_examples_tools/builder_test.py

@@ -3,6 +3,7 @@
 from fnmatch import fnmatch
 from os import listdir
 from pathlib import Path
+import shutil
 from stat import S_ISREG
 from typing import Dict, Generator, List
 
@@ -47,6 +48,9 @@ def mkdir(self, path: Path):
     def list(self, path: Path) -> List[Path]:
         pass
 
+    @abstractmethod
+    def copytree(self, source: Path, dest: Path):
+        pass
 
 class PathFs(Fs):
     def glob(self, path: Path, glob: str) -> Generator[Path, None, None]:
@@ -78,6 +82,9 @@ def list(self, path: Path) -> List[Path]:
         if self.stat(path).is_file:
             return []
         return [path / name for name in listdir(path)]
+    
+    def copytree(self, source: Path, dest: Path):
+        shutil.copytree(source, dest)
 
 
 class RecordFs(Fs):
@@ -137,5 +144,10 @@ def list(self, path: Path) -> List[Path]:
 
         return sorted(children)
 
+    def copytree(self, source: Path, dest: Path):
+        for child in self.list(source):
+            path = child.relative_to(dest).absolute()
+            self.fs[path] = self.fs[child]
+
 
 fs = PathFs()

aws_doc_sdk_examples_tools/fs.py

@@ -17,6 +17,7 @@
     ValidationConfig,
 )
 from . import validator_config
+from aws_doc_sdk_examples_tools import fs
 
 SNIPPET_START = "snippet-start:["
 SNIPPET_END = "snippet-end:["
@@ -309,16 +310,15 @@ def validate_snippets(
         verify_no_secret_keys(snippet.code, root / snippet.file, validation, errors)
 
 
-def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False):
+def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False, fs: Fs = fs.fs):
     errors = MetadataErrors()
     for tag in snippets:
         name = root / f"{tag}.txt"
-        if check and name.exists():
+        if check and fs.stat(name).exists:
             errors.append(SnippetAlreadyWritten(file=name))
         else:
             try:
-                with open(name, "w", encoding="utf-8") as file:
-                    file.write(snippets[tag].code)
+                fs.write(name, snippets[tag].code)
             except Exception as error:
                 errors.append(SnippetWriteError(file=name, error=error))
     return errors