Remove snippet tags from snippet files (#68)

* Strip snippet- tags from snippet-files
* Also remove SPDX headers
* Gracefully handle first few lines in SPDX header check

Author: DavidSouther

aws_doc_sdk_examples_tools/doc_gen.py

@@ -195,7 +195,7 @@ def for_root(
                 entities_config = yaml.safe_load(file)
             for entity, expanded in entities_config["expanded_override"].items():
                 self.entities[entity] = expanded
-        except Exception as e:
+        except Exception:
             pass
 
         metadata = root / ".doc_gen/metadata"

aws_doc_sdk_examples_tools/doc_gen_test.py

@@ -172,3 +172,14 @@ def test_doc_gen_encoder(sample_doc_gen: DocGen):
     # Verify examples (empty in this case)
     assert "examples" in decoded
     assert decoded["examples"] == {}
+
+
+def test_doc_gen_load_snippets():
+    errors = MetadataErrors()
+    doc_gen = DocGen(Path(), errors).for_root(
+        Path(__file__).parent / "test_resources", incremental=False
+    )
+    doc_gen.process_metadata(doc_gen.root / "valid_metadata.yaml")
+    doc_gen.collect_snippets()
+    assert doc_gen.snippet_files == set(["snippet_file.txt"])
+    assert doc_gen.snippets["snippet_file.txt"].code == "Line A\nLine C\n"

aws_doc_sdk_examples_tools/metadata_test.py

@@ -533,7 +533,7 @@ def test_verify_load_successful():
                             "php.snippet.tag.1",
                             "php.snippet.tag.2",
                         ],
-                        snippet_files=[],
+                        snippet_files=["snippet_file.txt"],
                     )
                 ],
                 add_services={},

aws_doc_sdk_examples_tools/snippets.py

@@ -15,6 +15,7 @@
     verify_no_secret_keys,
     ValidationConfig,
 )
+from . import validator_config
 
 SNIPPET_START = "snippet-start:["
 SNIPPET_END = "snippet-end:["
@@ -213,10 +214,40 @@ def collect_snippet_files(
                                 file=snippet_file,
                                 line_start=0,
                                 line_end=len(code),
-                                code="".join(code),
+                                code="".join(
+                                    strip_snippet_tags(strip_spdx_header(code))
+                                ),
                             )
 
 
+def strip_snippet_tags(lines: List[str]) -> List[str]:
+    return [line for line in lines if not has_snippet_tag_or_spdx_header(line)]
+
+
+def strip_spdx_header(file: List[str]) -> List[str]:
+    [a, b, c, *_] = file + [""] * 3
+    has_copyright = (
+        re.match(validator_config.SPDX_LEADER + validator_config.SPDX_COPYRIGHT, a)
+        is not None
+    )
+    has_license = (
+        re.match(validator_config.SPDX_LEADER + validator_config.SPDX_LICENSE, b)
+        is not None
+    )
+    has_spacer = "" == c
+    if has_copyright and has_license:
+        if has_spacer:
+            return file[3:]
+        else:
+            return file[2:]
+    else:
+        return file
+
+
+def has_snippet_tag_or_spdx_header(line: str) -> bool:
+    return "snippet-start" in line or "snippet-end" in line
+
+
 @dataclass
 class MissingSnippet(MetadataError):
     tag: Optional[str] = None

aws_doc_sdk_examples_tools/snippets_test.py

@@ -1,3 +1,6 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import pytest
 from pathlib import Path
 
@@ -65,3 +68,37 @@ def test_verify_snippet_start_end(file_contents: str, expected_error_count: int)
     _, errors = snippets.parse_snippets(file_contents.split("\n"), Path("test"), "")
     error_count = len(errors)
     assert error_count == expected_error_count
+
+
+def test_strip_snippet_tags():
+    assert ["Line A", "Line C"] == snippets.strip_snippet_tags(
+        [
+            "Line A",
+            "# snippet-start:[line b]",
+            "Line C",
+            "# snippet-end:[line d]",
+            "line E # snippet-end",
+        ]
+    )
+
+
+def test_strip_spdx_header():
+    assert ["Line A", "Line B"] == snippets.strip_spdx_header(
+        [
+            "# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.",
+            "# SPDX-License-Identifier: Apache-2.0",
+            "Line A",
+            "Line B",
+        ]
+    )
+    assert ["Line A", "Line B"] == snippets.strip_spdx_header(
+        [
+            "# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.",
+            "# SPDX-License-Identifier: Apache-2.0",
+            "",
+            "Line A",
+            "Line B",
+        ]
+    )
+
+    assert [] == snippets.strip_spdx_header([])

aws_doc_sdk_examples_tools/test_resources/snippet_file.txt

@@ -0,0 +1,5 @@
+Line A
+# snippet-start:[line b]
+Line C
+# snippet-end:[line d]
+line E # snippet-end

aws_doc_sdk_examples_tools/test_resources/valid_metadata.yaml

@@ -36,5 +36,7 @@ medical-imaging_TestExample:
               snippet_tags:
                 - php.snippet.tag.1
                 - php.snippet.tag.2
+              snippet_files:
+                - snippet_file.txt
   services:
-    medical-imaging:
+    medical-imaging: