Parse YAML in pure Nix

SenchoPens · Jun 4, 2023 · 2c292df · 2c292df
1 parent 6b404cd
commit 2c292df
Show file tree

Hide file tree

Showing 5 changed files with 181 additions and 44 deletions.
diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md
@@ -1,24 +1,20 @@
 # Documentation
 
-The documentation fully explains how everything works.
-
-## Module
-_defined in [module.nix](module.nix)_
+## Module ([module.nix](module.nix))
 
 - Adds a `scheme` option to be set to whatever `mkSchemeAttrs` accepts (see below).
   When used as a value, `scheme` will be equal to `mkSchemeAttrs scheme`.
 - Sets `config.lib.base16.mkSchemeAttrs`.
 
 As you can see, it's tiny. That's because the business logic is done by the library:
 
-## Library
-_defined in [default.nix](default.nix)_
+## Library ([default.nix](default.nix))
 
 Access it as:
 - `config.lib.base16` if using `base16.nix` as a NixOS module,
 - `pkgs.callPackage inputs.base16.lib {}` otherwise.
 
-It exports just 1 function:
+It exports 3 functions:
 
 ### `mkSchemeAttrs`
 
@@ -63,7 +59,15 @@ Other cool stuff:
 Note: `∀ x . mkSchemeAttrs (mkSchemeAttrs x) == mkSchemeAttrs x`
 </blockquote></details>
 
-This function isn't exported, but it's what powers up the _scheme attrset_'s `__functor` attribute:
+### `yaml2attrs`
+Given a path to a YAML file, converts its' contents to a Nix attrset in pure Nix. May fail on complex YAMLs.
+
+### `yaml2attrs-ifd`
+Given a path to a YAML file, converts its' contents to a Nix attrset using `yaml2json` package. Causes an [IFD](https://nixos.wiki/wiki/Import_From_Derivation). Isn't used by default, but can help if you're experiencing troubles with incorrectly parsed YAML files (see README Troubleshooting section for details).
+
+---
+
+The function below isn't exported, but it's what powers up the _scheme attrset_'s `__functor` attribute:
 
 #### `mkTheme`
 <details><summary>🙃</summary><blockquote>
@@ -89,4 +93,9 @@ mkTheme = {
   # If is `null` and `templateRepo` is passed, the extension will be grabbed from there,
   # otherwise it's an empty string
   extension ? null,
+  # Whether to use [IFD](https://nixos.wiki/wiki/Import_From_Derivation) to parse yaml.
+  # Can cause problems with `nix flake check / show` (see the issue #3).
+  use-ifd ? false,
+  # Whether to check if the config.yaml was parsed correctly.
+  check-parsed-config-yaml ? true,
 }:
diff --git a/README.md b/README.md
@@ -296,17 +296,80 @@ Please feel free to list your repository above, it will make my day :)
 - [theme-base16](https://gitlab.com/rycee/nur-expressions/-/tree/master/hm-modules/theme-base16) by @rycee.
 
 
-## ☎️ Help
+## ☎️ Troubleshooting
 
-If you need any help, feel free to open an issue or
-contact me via email or telegram ([my contacts are here](https://github.com/SenchoPens)).
+<details><summary>Error / incorrect behavior after updating base16.nix or adding a new source / template</summary><blockquote>
 
+The most probable reason of such an error is either a scheme or a template YAML file.
+Since version v2.0.0 `base16.nix` parses the YAML file in pure Nix to bypass IFD issues.
+The parser work for most `base16-<scheme-name>.yaml` and templates' `config.yaml` files, but,
+as YAML can be quite complicated, sometimes they can be parsed incorrectly.
+
+The exact error depends on the point of failure.
+It probably will be cryptic if incorrect parsing caused an issue during nix evaluation.
+Otherwise, if your flake evaluates (`nix flake check` succeeds), the error will look something like this:
+```
+error: builder for '/nix/store/snbbfb43qphzfl6xr1mjs0mr8jny66x9-base16-nix-parse-check.drv' failed with exit code 1;
+       last 7 log lines:
+       > running tests
+       > Output of "jd /nix/store/9jvxabhfx9acrysknblg0r2hzvcwv6ab-fromYAML /nix/store/qwmj9cbg7fpi5fvyd2x3kywfbw7hlm8f-parsed-yaml-as-json":
+       > @ ["gotcha"]
+       > - ["1 2"]
+       > + "[ 1 2 ]"
+       > Error: /nix/store/qhdqwj0mfp8qn0gq5s95pgd2i57lb09c-source/base16-kandinsky.yaml was parsed incorrectly during nix evaluation.
+       > Please consult https://github.com/SenchoPens/base16.nix/tree/main#%EF%B8%8F-troubleshooting
+```
+This check happens by default for templates by installing a special derivation. You can do it for scheme too by adding the `config.scheme.check` derivation to your NixOS / home-manager package list. 
+
+### Fix incorrectly parsed YAML file
+
+- If the problem is with a scheme YAML file and the nix evaluates, add the `config.scheme.check` derivation to your NixOS / home-manager package list, this will indicate which part of the YAML is being parsed incorrectly.
+- If you think that it is safe to ignore this error when handling a template, turn off the check:
+  ```nix
+  home-manager.users.sencho.programs.zathura.extraConfig =
+    builtins.readFile (config.scheme {
+      check-parsed-config-yaml = false;
+      templateRepo = inputs.base16-zathura; target = "recolor";
+    });
+  ```
+- Enable IFD (but beware of a possible error described below):
+  If the problem is in the scheme YAML file, parse it with `config.lib.base16.yaml2attrs-ifd` first:
+  ```nix
+  config.scheme = config.lib.base16.yaml2attrs-ifd "${inputs.base16-schemes}/nord.yaml";
+  ```
+  If the problem is in the template `templates/config.yaml` file, turn on `use-ifd`:
+  ```nix
+  home-manager.users.sencho.programs.zathura.extraConfig =
+    builtins.readFile (config.scheme {
+      use-ifd = true;
+      templateRepo = inputs.base16-zathura; target = "recolor";
+    });
+  ```
+- Submit an issue.
+- Fix the YAML upstream. Probable causes: trailing spaces, file structure differs from typical `config.yaml` / scheme YAML files.
+- Fix the Nix parser 😈.
+
+</blockquote></details>
+
+<details><summary>Error on `nix flake check` or `nix flake show`</summary><blockquote>
+
+First, check that you have the most recent version of `base16.nix`.
+If updating doesn't help, check that you don't turn on `use-ifd` in any of the scheme calls (template instantiations)
+and that you don't invoke `yaml2attrs-ifd` function.
+
+Relevant issue: #3.
+
+If neither of the above listed solutions do not work for you, please reopen it.
+</blockquote></details>
+
+Anyhow, feel free to open an issue!
 
 ## 💙 Acknowledgments
 
 Thanks to:
 - @balsoft for [nixos-config](https://code.balsoft.ru/balsoft/nixos-config),
   which inspired this library;
+- @DavHau for [fromYaml](https://github.com/DavHau/fromYaml);
 - @cab404 for [Genix7000 — icon generator for nix projects](https://github.com/cab404/genix7000);
 - @chriskempson for creating base16
   and @belak and base16-project team for maintaining it;

diff --git a/default.nix b/default.nix
@@ -1,3 +1,4 @@
+fromYaml:
 { pkgs, lib, ... }:
 let
   #------------------#
@@ -103,14 +104,45 @@ let
 
     in based // { inherit toList withHashtag; };
 
-  writeTextFile = path: text: ''${pkgs.writeTextDir path text}/${path}'';
+  writeTextFile' = path: text: ''${pkgs.writeTextDir path text}/${path}'';
 
-  yaml2attrs = yaml:
-    builtins.fromJSON (builtins.readFile (pkgs.stdenv.mkDerivation {
-      name = "fromYAML";
-      phases = [ "buildPhase" ];
-      buildPhase = "${pkgs.yaml2json}/bin/yaml2json < ${yaml} > $out";
-    }));
+  yaml2json = yaml: pkgs.stdenv.mkDerivation {
+    name = "fromYAML";
+    phases = [ "buildPhase" ];
+    buildPhase = "${pkgs.yaml2json}/bin/yaml2json < ${yaml} > $out";
+  };
+  yaml2attrs-ifd = yaml: builtins.fromJSON (builtins.readFile (yaml2json yaml));
+  yaml2attrs = yaml: import "${fromYaml}/fromYaml.nix" { inherit lib; } (builtins.readFile yaml);
+  # Checks that the above yaml2attrs function has correctly parsed an yaml file.
+  check-parsed-yaml = yaml: let 
+    correctlyParsedYamlAsJson = yaml2json yaml;
+    yaml-filename = lib.escapeShellArg "${yaml}";
+    parsedYamlAsJson =
+      pkgs.writeText "parsed-yaml-as-json" (builtins.toJSON (yaml2attrs yaml));
+  in pkgs.stdenv.mkDerivation {
+    name = "base16-nix-parse-check";
+    nativeCheckInputs = [ pkgs.diffutils pkgs.jd-diff-patch ];
+    doCheck = true;
+    phases = [ "checkPhase" "installPhase" ];
+    checkPhase = ''
+      runHook preCheck
+      set +e
+      DIFF=$(jd ${correctlyParsedYamlAsJson} ${parsedYamlAsJson}) 
+      set -e
+      if [ "$DIFF" != "" ] 
+      then
+        echo 'Output of "jd ${correctlyParsedYamlAsJson} ${parsedYamlAsJson}":'
+        echo "$DIFF"
+        echo 'Error:' ${yaml-filename} 'was parsed incorrectly during nix evaluation.'
+        echo 'Please consult https://github.com/SenchoPens/base16.nix/tree/main#%EF%B8%8F-troubleshooting'
+        exit 1
+      fi
+      runHook postCheck
+    '';
+    installPhase = ''
+      mkdir $out
+    '';
+  };
 
   /* Builds a theme file from a scheme and a template and returns its path.
      If you do not supply `templateRepo`, then
@@ -133,14 +165,26 @@ let
     # If is `null` and `templateRepo` is passed, the extension will be grabbed from there,
     # otherwise it's an empty string
     extension ? null,
+    # Whether to use [IFD](https://nixos.wiki/wiki/Import_From_Derivation) to parse yaml.
+    # Can cause problems with `nix flake check / show` (see the issue #3).
+    use-ifd ? false,
+    # Whether to check if the config.yaml was parsed correctly.
+    check-parsed-config-yaml ? true,
   }:
     let
+      config-yaml =
+        if (extension == null && templateRepo != null) then
+          "${templateRepo}/templates/config.yaml"
+        else
+          null;
+      check-parsed-config-yaml' = config-yaml != null && check-parsed-config-yaml && !use-ifd;
       ext =
         if extension == null then
           if templateRepo == null then
             ""
-          else
-            (yaml2attrs "${templateRepo}/templates/config.yaml").${target}.extension
+          else let
+            parsed = (if use-ifd then yaml2attrs-ifd else yaml2attrs) config-yaml;
+          in parsed.${target}.extension
         else
           extension
         ;
@@ -149,24 +193,25 @@ let
         if template == null then
           "${templateRepo}/templates/${target}.mustache"
         else
-          writeTextFile "${target}.mustache" template
+          writeTextFile' "${target}.mustache" template
         ;
       # Taken from https://pablo.tools/blog/computers/nix-mustache-templates/
-      themeDerivation = pkgs.stdenv.mkDerivation rec {
+      themeDerivation = pkgs.stdenv.mkDerivation {
         name = "${builtins.unsafeDiscardStringContext scheme.scheme-slug}";
 
-        nativeBuildInpts = [ pkgs.mustache-go ];
+        nativeBuildInputs = [ pkgs.mustache-go ]
+          ++ lib.optional check-parsed-config-yaml' (check-parsed-yaml config-yaml);
 
         # Pass JSON as file to avoid escaping
         passAsFile = [ "jsonData" ];
-        jsonData = builtins.toJSON (builtins.removeAttrs scheme [ "outPath" "override" "__functor" ]);
+        jsonData = builtins.toJSON (builtins.removeAttrs scheme [ "outPath" "override" "check" "__functor" ]);
 
         # Disable phases which are not needed. In particular the unpackPhase will
         # fail, if no src attribute is set
         phases = [ "buildPhase" "installPhase" ];
 
         buildPhase = ''
-          ${pkgs.mustache-go}/bin/mustache $jsonDataPath ${templatePath} > theme
+          mustache $jsonDataPath ${templatePath} > theme
         '';
 
         installPhase = ''
@@ -197,10 +242,7 @@ let
               lib.removeSuffix ".yaml" (
                 builtins.baseNameOf (
                   builtins.unsafeDiscardStringContext "${scheme}"
-          ));}
-          //
-          (yaml2attrs scheme)
-        ;
+          )); } // yaml2attrs scheme;
 
       inputMeta = rec {
         scheme = ''${inputAttrs.scheme or "untitled"}'';
@@ -224,7 +266,7 @@ let
       magic = {
         # Lets scheme attrs be automatically coerced to string (`__str__`)
         outPath =
-          writeTextFile "${inputMeta.slug}.yaml"
+          writeTextFile' "${inputMeta.slug}.yaml"
             (builtins.concatStringsSep "\n"
               (lib.mapAttrsToList (name: value: "${name}: ${value}") inputAttrs));
         # Calling a scheme attrset will build a theme (`__call__`)
@@ -236,12 +278,25 @@ let
 
       populatedColors = colors inputAttrs;
 
-      allOther = inputMeta // builderMeta // magic // {
-        override = new: mkSchemeAttrs (inputAttrs // inputMeta // new);
+      allOther = inputMeta // builderMeta // magic // rec {
+        check =
+          if !(builtins.isAttrs scheme) then check-parsed-yaml scheme
+          else pkgs.emptyDirectory;
+        override = new: let
+          new-scheme = mkSchemeAttrs (inputAttrs // inputMeta // new);
+          new-override = new-scheme.override;
+          allOther-patch = {
+            inherit check;
+            override = new-new: (new-override new-new) // patch;
+          };
+          patch = {
+            withHashtag = new-scheme.withHashtag // allOther-patch;
+          } // allOther-patch;
+        in new-scheme // patch;
       };
 
     in populatedColors // allOther // {
       withHashtag = populatedColors.withHashtag // allOther;
     };
 
-in { inherit mkSchemeAttrs; }
+in { inherit mkSchemeAttrs yaml2attrs-ifd yaml2attrs; }
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -1,9 +1,16 @@
 {
   description = "Nix utility functions to configure base16 themes";
 
-  outputs = { self, nixpkgs, ... } @ inputs:
+  inputs = {
+    fromYaml = {
+      url = "github:SenchoPens/fromYaml";
+      flake = false;
+    };
+  };
+
+  outputs = { self, fromYaml, ... }:
     {
-      lib = import ./.;
+      lib = import ./. fromYaml;
 
       nixosModule = import ./module.nix self;