Remove single-{ handling entirely - jinja across the board

.vscode/settings.json

@@ -1,3 +1,8 @@
 {
-    "python.defaultInterpreterPath": ".env/bin/python"
+    "python.defaultInterpreterPath": ".env/bin/python",
+    "python.testing.pytestArgs": [
+        "bin"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
 }

bin/lib/ce_install.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # coding=utf-8
+import json
 import logging
 import logging.config
 import multiprocessing
@@ -11,7 +12,7 @@
 from dataclasses import dataclass
 from functools import partial
 from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, TextIO
 
 import click
 import yaml
@@ -242,15 +243,15 @@ def cli(
 
 @cli.command(name="list")
 @click.pass_obj
-@click.option("--json", is_flag=True, help="Output in JSON format")
+@click.option("--json", "as_json", is_flag=True, help="Output in JSON format")
 @click.option("--installed-only", is_flag=True, help="Only output installed targets")
 @click.argument("filter_", metavar="FILTER", nargs=-1)
-def list_cmd(context: CliContext, filter_: List[str], json: bool, installed_only: bool):
+def list_cmd(context: CliContext, filter_: List[str], as_json: bool, installed_only: bool):
     """List installation targets matching FILTER."""
     for installable in context.get_installables(filter_):
         if installed_only and not installable.is_installed():
             continue
-        print(installable.to_json() if json else installable.name)
+        print(installable.to_json() if as_json else installable.name)
         _LOGGER.debug(installable)
 
 
@@ -538,6 +539,28 @@ def add_crate(context: CliContext, libid: str, libversion: str):
     libyaml.save()
 
 
+@cli.command()
+@click.argument("output", type=click.File("w", encoding="utf-8"), default="-")
+@click.pass_obj
+def config_dump(context: CliContext, output: TextIO):
+    """Dumps all config, expanded."""
+    for yaml_path in sorted(Path(context.installation_context.yaml_dir).glob("*.yaml")):
+        with yaml_path.open(encoding="utf-8") as yaml_file:
+            yaml_doc = yaml.load(yaml_file, Loader=ConfigSafeLoader)
+        for installer in sorted(installers_for(context.installation_context, yaml_doc, True), key=str):
+            # Read all public strings fields from installer
+            as_dict = {
+                "name": installer.name,
+                "type": str(installer),
+                "config": {
+                    field: getattr(installer, field)
+                    for field in dir(installer)
+                    if not field.startswith("_") and isinstance(getattr(installer, field), str)
+                },
+            }
+            output.write(json.dumps(as_dict) + "\n")
+
+
 def main():
     cli(prog_name="ce_install")  # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
 

bin/lib/config_expand.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any, MutableMapping
+from typing import Any, Mapping, MutableMapping
 
 import jinja2
 
@@ -28,30 +28,32 @@ def is_value_type(value: Any) -> bool:
     )
 
 
-def needs_expansion(target):
+def string_needs_expansion(value: str) -> bool:
+    return "{%" in value or "{{" in value or "{#" in value
+
+
+def needs_expansion(target: MutableMapping[str, Any]) -> bool:
     for value in target.values():
         if is_list_of_strings(value):
-            for v in value:
-                if "{" in v:
-                    return True
+            if any(string_needs_expansion(v) for v in value):
+                return True
         elif isinstance(value, str):
-            if "{" in value:
+            if string_needs_expansion(value):
                 return True
     return False
 
 
-def expand_one(template_string, configuration):
+def expand_one(template_string: str, configuration: Mapping[str, Any]) -> str:
     try:
-        jinjad = _JINJA_ENV.from_string(template_string).render(**configuration)
-        return jinjad.format(**configuration)
+        return _JINJA_ENV.from_string(template_string).render(**configuration)
     except jinja2.exceptions.TemplateError:
         # in python 3.11 we would...
         # e.add_note(f"Template '{template_string}'")
         _LOGGER.warning("Failed to expand '%s'", template_string)
         raise
 
 
-def expand_target(target: MutableMapping[str, Any], context):
+def expand_target(target: MutableMapping[str, Any], context: list[str]) -> MutableMapping[str, str]:
     iterations = 0
     while needs_expansion(target):
         iterations += 1

bin/lib/installation.py

@@ -29,7 +29,7 @@ def targets_from(node, enabled, base_config=None):
 
 
 def _check_if(enabled, node) -> bool:
-    if "if" not in node:
+    if "if" not in node or enabled is True:
         return True
     if isinstance(node["if"], list):
         condition = set(node["if"])

bin/test/config_expand_test.py

@@ -1,6 +1,35 @@
-from lib.config_expand import expand_one
+import re
+
+import pytest
+from lib.config_expand import expand_one, expand_target
 
 
 def test_expand_one():
     assert expand_one("moo", {}) == "moo"
-    assert expand_one("a{badger}b", {"badger": "moose"}) == "amooseb"
+    assert expand_one("a{{badger}}b", {"badger": "moose"}) == "amooseb"
+
+
+def test_expand_one_ignores_single_braces():
+    assert expand_one("a{badger}b", {"badger": "moose"}) == "a{badger}b"
+
+
+def test_expand_one_allows_escapes():
+    assert expand_one("a{% raw %}{badger}{% endraw %}b", {"badger": "moose"}) == "a{badger}b"
+
+
+def test_expand_target_handles_self_references():
+    assert expand_target({"sponge": "bob_{{bob}}", "bob": "robert"}, []) == {"sponge": "bob_robert", "bob": "robert"}
+
+
+def test_expand_target_handles_multiple_self_references():
+    assert expand_target({"sponge": "bob_{{bob}}", "bob": "{{ian}}", "ian": "will{{iam}}", "iam": "y"}, []) == {
+        "sponge": "bob_willy",
+        "bob": "willy",
+        "ian": "willy",
+        "iam": "y",
+    }
+
+
+def test_expand_target_handles_infinite_recursion():
+    with pytest.raises(RuntimeError, match=re.escape("Too many mutual references (in moo/shmoo)")):
+        assert expand_target({"bob": "{{ian}}", "ian": "ooh{{bob}}"}, ["moo", "shmoo"])

bin/test/installation_test.py

@@ -1,3 +1,4 @@
+import re
 from pathlib import Path
 
 import pytest
@@ -64,11 +65,11 @@ def test_codependent_configs():
         """
 compilers:
   gcc:
-    check_exe: "bin/{arch_prefix}/blah"
+    check_exe: "bin/{{arch_prefix}}/blah"
     subdir: arm
     mips:
-      arch_prefix: "{subdir}-arch"
-      check_exe: "{arch_prefix}/blah"
+      arch_prefix: "{{subdir}}-arch"
+      check_exe: "{{arch_prefix}}/blah"
       targets:
         - name: 5.4.0
           subdir: mips
@@ -78,13 +79,13 @@ def test_codependent_configs():
 
 
 def test_codependent_throws():
-    with pytest.raises(RuntimeError, match=r"Too many mutual references \(in compilers/mips\)"):
+    with pytest.raises(RuntimeError, match=re.escape("Too many mutual references (in compilers/mips)")):
         parse_targets(
             """
 compilers:
   mips:
-    x: "{y}"
-    y: "{x}"
+    x: "{{y}}"
+    y: "{{x}}"
     targets:
       - name: 5.4.0
         subdir: mips
@@ -166,7 +167,7 @@ def test_jinja_expansion_with_filters_refering_forward():
         """
 boost:
   underscore_name: "{{ name | replace('.', '_') }}"
-  url: https://dl.bintray.com/boostorg/release/{name}/source/boost_{underscore_name}.tar.bz2
+  url: https://dl.bintray.com/boostorg/release/{{name}}/source/boost_{{underscore_name}}.tar.bz2
   targets:
     - 1.64.0
       """

bin/yaml/ada.yaml

@@ -2,13 +2,13 @@ compilers:
   ada:
     gnat:
       type: tarballs
-      untar_dir: "gnat-{target}-linux64-{name}"
-      dir: "{subdir}/gnat-{target}-linux64-{name}"
-      url: https://github.com/alire-project/GNAT-FSF-builds/releases/download/gnat-{version}/gnat-{target}-linux64-{version}.tar.gz
+      untar_dir: "gnat-{{target}}-linux64-{{name}}"
+      dir: "{{subdir}}/gnat-{{target}}-linux64-{{name}}"
+      url: https://github.com/alire-project/GNAT-FSF-builds/releases/download/gnat-{{version}}/gnat-{{target}}-linux64-{{version}}.tar.gz
       compression: gz
-      check_exe: bin/{toolprefix}gcc -v
+      check_exe: bin/{{toolprefix}}gcc -v
       check_stderr_on_stdout: true
-      name: "{version}"
+      name: "{{version}}"
 
       arm:
         subdir: arm

bin/yaml/android-java.yaml

@@ -1,13 +1,13 @@
 compilers:
   android-d8:
     type: singleFile
-    dir: r8-{name}
+    dir: r8-{{name}}
     depends:
       - compilers/java 16.0.1
       - compilers/kotlin 1.9.20 # d8 runs on .class files, so kotlinc is applicable as well.
-    check_exe: "%DEP0%/bin/java -cp {dir}/{filename} com.android.tools.r8.D8 --version"
-    filename: r8-{name}.jar
-    url: https://dl.google.com/android/maven2/com/android/tools/r8/{name}/r8-{name}.jar
+    check_exe: "%DEP0%/bin/java -cp {{dir}}/{{filename}} com.android.tools.r8.D8 --version"
+    filename: r8-{{name}}.jar
+    url: https://dl.google.com/android/maven2/com/android/tools/r8/{{name}}/r8-{{name}}.jar
     targets:
       - 8.7.18
       - 8.7.14
@@ -25,12 +25,12 @@ compilers:
       - 8.1.56
   dex2oat:
     type: ziparchive
-    dir: dex2oat-{name}
+    dir: dex2oat-{{name}}
     extract_into_folder: true
     check_file: "x86_64/bin/dex2oat64"
-    url: https://dl.google.com/android/maven2/com/android/art/art/{name}/art-{name}.zip
+    url: https://dl.google.com/android/maven2/com/android/art/art/{{name}}/art-{{name}}.zip
     after_stage_script:
-      - "{yaml_dir}/android-java/create_boot_images.sh"
+      - "{{yaml_dir}}/android-java/create_boot_images.sh"
     targets:
       - "35.11"
       - "35.10"
@@ -46,20 +46,20 @@ compilers:
       # 33.10 doesn't support CMC or riscv64, so another script is used for boot images.
       - name: "33.10"
         after_stage_script:
-          - "{yaml_dir}/android-java/create_boot_images_old.sh"
+          - "{{yaml_dir}}/android-java/create_boot_images_old.sh"
 
   nightly:
     install_always: true
     if: nightly
     type: script
-    dir: "{name}"
+    dir: "{{name}}"
     script: |
-      mkdir {name}
-      cd {name}
-      {yaml_dir}/android-java/fetch_art_release_from_head.sh
+      mkdir {{name}}
+      cd {{name}}
+      {{yaml_dir}}/android-java/fetch_art_release_from_head.sh
       unzip art_release.zip
       rm art_release.zip
-      {yaml_dir}/android-java/create_boot_images.sh
+      {{yaml_dir}}/android-java/create_boot_images.sh
     dex2oat:
       targets:
         - name: dex2oat-latest