Update ghc 8.4.4 based tools to ghc 8.6.5 (#618)

Although the default ghc used by haskell.nix is ghc 8.6.5 many of
the tools used in haskell.nix are still built with the boot compiler
ghc 8.4.4.  These include

  * haskell-nix.cabal-install
  * haskell-nix.alex
  * haskell-nix.happy

This change updates those to ghc 8.6.5 and includes materializations
for the new versions.

When cabal-install is built it is careful to disable materialization
checks on the version of itself used during the build to avoid
infinite recursion.

There was a version of nix-tools built with the boot ghc which was
only used when `checkMaterialization = true`.  It was used for
the boot versions of alex, happy and hscolour.  These have been update
to use the default (ghc 8.6.5) version of nix-tools and
checkMaterialization is forced off when they are being used to build
ghc.  This means the materialization will only be checked for these
when they are built independently (they are included in the test set
via haskellNixRoots).

Three new arguments are added to `default.nix`:

* `defaultCompilerNixName` if not specified "ghc865" is used
* `checkMaterialization` makes it easier to switch on materialization checks
* `system` defaults to `builtins.currentSystem`

This change also moves the work needed for hydra eval to the eval
system using a new `evalPackages` feature.  This includes:

* Fetching from git with `fetchgit`
* Building scripts with `runCommand` and `writeTextFile`
* `git ls-files` in `cleanGit`
* running `cabal v2-configure`
* copying materialized files (we are not sure why this is necessary but
  if we do not `cp -r` the files nix will not load them on hydra)

Reduce size of `make-config-files.nix` strings by around 80%.
These are unlikely to be the cause of hydra eval time memory
issues in the GB range, but were still quite large (around 10MB for the
`cabal-simple` test case).

There was issue causing excessive builds of the `git` package when
cross compiling.  Gory details are a comment in `lib/defaults.nix`
but in short if you use `git` you need an extra `.buildPackages`
one is not enough because it depends on `gdb` and that will
be different in `buildPackages` compared to
`buildPackages.buildPackages`.

Adds missing materialization files for ghc 8.4.4 (only needed
when `checkMaterialization` is on because of other
materialiazations, but good to have).

Author: hamishmack

.gitattributes

@@ -4,3 +4,4 @@
 nix-tools/.plan.nix/*.nix linguist-generated=true
 nix-tools/pkgs.nix linguist-generated=true
 .stack-to-nix.cache linguist-generated=true
+materialized/**/* linguist-generated=true

builder/make-config-files.nix

@@ -33,28 +33,6 @@ let
        dep.components.library # Regular package dependency
     or dep;                   # or a sublib
 
-  catPkgExactDep = p: ''
-    cat ${p}/exactDep/configure-flags >> $out/configure-flags
-    cat ${p}/exactDep/cabal.config >> $out/cabal.config
-  '';
-
-  catGhcPkgExactDep = p: ''
-    if [ -e ${ghc}/exactDeps/${p} ]; then
-      cat ${ghc}/exactDeps/${p}/configure-flags >> $out/configure-flags
-      cat ${ghc}/exactDeps/${p}/cabal.config >> $out/cabal.config
-    fi
-  '';
-
-  catPkgEnvDep = p: ''
-    cat ${p}/envDep >> $out/ghc-environment
-  '';
-
-  catGhcPkgEnvDep = p: ''
-    if [ -e ${ghc}/envDeps/${p} ]; then
-      cat ${ghc}/envDeps/${p} >> $out/ghc-environment
-    fi
-  '';
-
   # Work our suitable packageCfgDir subdirectory
   isGhcjs        = ghc.isGhcjs or false;
   ghcCommand'    = if isGhcjs then "ghcjs" else "ghc";
@@ -94,10 +72,13 @@ in { identifier, component, fullName, flags ? {}, needsProfiling ? false }:
       "extra-framework-dirs" = map (p: "${p}/Library/Frameworks") component.frameworks;
     })}
 
-    # Copy over the nonReinstallablePkgs from the global package db.
-    ${lib.concatMapStringsSep "\n" (p: ''
-      find ${ghc}/lib/${ghc.name}/package.conf.d -name '${p}*.conf' -exec cp -f {} $out/${packageCfgDir} \;
-    '') nonReinstallablePkgs}
+    ghc=${ghc}
+    ${ # Copy over the nonReinstallablePkgs from the global package db.
+    ''
+      for p in ${lib.concatStringsSep " " nonReinstallablePkgs}; do
+        find $ghc/lib/${ghc.name}/package.conf.d -name $p'*.conf' -exec cp -f {} $out/${packageCfgDir} \;
+      done
+    ''}
 
     for l in "${cfgFiles}"; do
       if [ -n "$l" ]; then
@@ -110,29 +91,49 @@ in { identifier, component, fullName, flags ? {}, needsProfiling ? false }:
       fi
     done
 
-    # Note: we pass `clear` first to ensure that we never consult the implicit global package db.
-    ${flagsAndConfig "package-db" ["clear" "$out/${packageCfgDir}"]}
+    ${ # Note: we pass `clear` first to ensure that we never consult the implicit global package db.
+      flagsAndConfig "package-db" ["clear" "$out/${packageCfgDir}"]
+    }
 
     echo ${lib.concatStringsSep " " (lib.mapAttrsToList (fname: val: "--flags=${lib.optionalString (!val) "-" + fname}") flags)} >> $out/configure-flags
 
-    # Provide a cabal config without remote package repositories
-    echo "write-ghc-environment-files: never" >> $out/cabal.config
-
-    # Provide a GHC environment file
-    cat > $out/ghc-environment <<EOF
-    package-db $out/${packageCfgDir}
-    EOF
-
-    ${lib.concatMapStringsSep "\n" catPkgEnvDep libDeps}
-    ${lib.concatMapStringsSep "\n" catGhcPkgEnvDep (lib.remove "ghc" nonReinstallablePkgs)}
+    ${ # Provide a cabal config without remote package repositories
+    ''
+      echo "write-ghc-environment-files: never" >> $out/cabal.config
+    ''}
+
+    ${ # Provide a GHC environment file
+    ''
+      cat > $out/ghc-environment <<EOF
+      package-db $out/${packageCfgDir}
+      EOF
+    ''}
+
+    ${ lib.optionalString component.doExactConfig ''
+      echo "--exact-configuration" >> $out/configure-flags
+      echo "allow-newer: ${identifier.name}:*" >> $out/cabal.config
+      echo "allow-older: ${identifier.name}:*" >> $out/cabal.config
+    ''}
+
+    for p in ${lib.concatStringsSep " " libDeps}; do
+      cat $p/envDep >> $out/ghc-environment
+      ${ lib.optionalString component.doExactConfig ''
+        cat $p/exactDep/configure-flags >> $out/configure-flags
+        cat $p/exactDep/cabal.config >> $out/cabal.config
+      ''}
+    done
+    for p in ${lib.concatStringsSep " " (lib.remove "ghc" nonReinstallablePkgs)}; do
+      if [ -e $ghc/envDeps/$p ]; then
+        cat $ghc/envDeps/$p >> $out/ghc-environment
+      fi
+    done
   '' + lib.optionalString component.doExactConfig ''
-    echo "--exact-configuration" >> $out/configure-flags
-    echo "allow-newer: ${identifier.name}:*" >> $out/cabal.config
-    echo "allow-older: ${identifier.name}:*" >> $out/cabal.config
-
-    ${lib.concatMapStringsSep "\n" catPkgExactDep libDeps}
-    ${lib.concatMapStringsSep "\n" catGhcPkgExactDep nonReinstallablePkgs}
-
+    for p in ${lib.concatStringsSep " " nonReinstallablePkgs}; do
+      if [ -e $ghc/exactDeps/$p ]; then
+        cat $ghc/exactDeps/$p/configure-flags >> $out/configure-flags
+        cat $ghc/exactDeps/$p/cabal.config >> $out/cabal.config
+      fi
+    done 
   ''
   # This code originates in the `generic-builder.nix` from nixpkgs.  However GHC has been fixed
   # to drop unused libraries referneced from libraries; and this patch is usually included in the
@@ -156,12 +157,13 @@ in { identifier, component, fullName, flags ? {}, needsProfiling ? false }:
   #   's/ /\n/g ; s/\n\n*/\n/g; s/^\n//;'      Puts each field on its own line.
   #   's|/nix/store/|''${pkgroot}/../../../|'  Convert store path to pkgroot relative path
   #   's|''${pkgroot}/../../../|/nix/store/|'  Convert pkgroot relative path to store path
-  + lib.optionalString stdenv.isDarwin ''
+
     # Work around a limit in the macOS Sierra linker on the number of paths
     # referenced by any one dynamic library:
     #
     # Create a local directory with symlinks of the *.dylib (macOS shared
     # libraries) from all the dependencies.
+  + lib.optionalString stdenv.isDarwin ''
     local dynamicLinksDir="$out/lib/links"
     mkdir -p $dynamicLinksDir
     # Enumerate dynamic-library-dirs with ''${pkgroot} expanded.

ci.nix

@@ -6,7 +6,7 @@
 , restrictEval ? false }:
 let
   inherit (import ./ci-lib.nix) dimension platformFilterGeneric filterAttrsOnlyRecursive;
-  inherit (import ./default.nix {}) sources nixpkgsArgs;
+  inherit (import ./default.nix { checkMaterialization = false; }) sources nixpkgsArgs;
   nixpkgsVersions = {
     "R1909" = "nixpkgs-1909";
     "R2003" = "nixpkgs-2003";

default.nix

@@ -1,26 +1,50 @@
-let haskellNix = rec {
+let haskellNix = {
+      checkMaterialization ? false,  # Allows us to easily switch on materialization checking
+      defaultCompilerNixName ? null, # Quick way to override the default compiler e.g. "ghc883"
+      system ? builtins.currentSystem,
+      ... }: rec {
     sources = {
       inherit (import ./nixpkgs/default.nix) nixpkgs-1909 nixpkgs-2003 nixpkgs-default;
     };
 
     config   = import ./config.nix;
-    overlays = [ allOverlays.combined ];
+    overlays = [ allOverlays.combined ] ++ (
+      if checkMaterialization == true
+        then [(
+          final: prev: {
+            haskell-nix = prev.haskell-nix // {
+              checkMaterialization = true;
+            };
+          }
+        )]
+        else []
+    ) ++ (
+      if defaultCompilerNixName != null
+        then [(
+          final: prev: {
+            haskell-nix = prev.haskell-nix // {
+              inherit defaultCompilerNixName;
+            };
+          }
+        )]
+        else []
+    );
     allOverlays = import ./overlays;
-    nixpkgsArgs = { inherit config overlays; };
+    nixpkgsArgs = { inherit config overlays system; };
     pkgs = import sources.nixpkgs-default nixpkgsArgs;
   };
 
-  haskellNixV1 = haskellNix.nixpkgsArgs;
+  haskellNixV1 = (haskellNix {}).nixpkgsArgs;
   haskellNixV2 = haskellNix;
 
   v1DeprecationMessage = "Version 1 is deprecated: use version 2 (nixpkgs arguments are available as the `nixpkgsArgs` attribute of version 2)";
 # If no arguments, then you get V1
 # I'd like to make importing directly issue a warning, but I couldn't figure out a way to make it happen
 in haskellNixV1 // {
-  __functor = _: { version ? 2 }:
+  __functor = _: { version ? 2, ... }@args:
     if version == 1
     then builtins.trace v1DeprecationMessage haskellNixV1
     else if version == 2
-    then haskellNixV2
+    then haskellNixV2 args
     else builtins.throw ("haskell.nix: unknown version: " + (builtins.toString version));
 }

flake.nix

@@ -4,7 +4,10 @@
   edition = 201909;
 
   outputs = { self }: {
-    overlay = self.overlays.combined;
+    # Using the eval-on-build version here as the plan is that
+    # `builtins.currentSystem` will not be supported in flakes.
+    # https://github.com/NixOS/rfcs/pull/49/files#diff-a5a138ca225433534de8d260f225fe31R429
+    overlay = self.overlays.combined-eval-on-build;
     overlays = import ./overlays;
     config = import ./config.nix;
     sources = import ./nixpkgs;

lib/call-cabal-project-to-nix.nix

@@ -28,10 +28,6 @@
 , extra-hackage-tarballs ? []
 , ...
 }@args:
-# cabal-install versions before 2.4 will generate insufficient plan information.
-assert (if (builtins.compareVersions cabal-install.version "2.4.0.0") < 0
-         then throw "cabal-install (current version: ${cabal-install.version}) needs to be at least 2.4 for plan-to-nix to work without cabal-to-nix"
-         else true);
 
 let
   forName = pkgs.lib.optionalString (name != null) (" for " + name);
@@ -265,7 +261,7 @@ let
   '');
 
   # Dummy `ghc` that uses the captured output 
-  dummy-ghc = pkgs.writeTextFile {
+  dummy-ghc = pkgs.evalPackages.writeTextFile {
     name = "dummy-" + ghc.name;
     executable = true;
     destination = "/bin/${ghc.targetPrefix}ghc";
@@ -282,7 +278,7 @@ let
   };
 
   # Dummy `ghc-pkg` that uses the captured output 
-  dummy-ghc-pkg = pkgs.writeTextFile {
+  dummy-ghc-pkg = pkgs.evalPackages.writeTextFile {
     name = "dummy-pkg-" + ghc.name;
     executable = true;
     destination = "/bin/${ghc.targetPrefix}ghc-pkg";
@@ -306,8 +302,8 @@ let
         else null;
   } // pkgs.lib.optionalAttrs (checkMaterialization != null) {
     inherit checkMaterialization;
-  }) (runCommand (if name == null then "plan-to-nix-pkgs" else name + "-plan-to-nix-pkgs") {
-    nativeBuildInputs = [ nix-tools dummy-ghc dummy-ghc-pkg hpack cabal-install pkgs.rsync ];
+  }) (pkgs.evalPackages.runCommand (if name == null then "plan-to-nix-pkgs" else name + "-plan-to-nix-pkgs") {
+    nativeBuildInputs = [ nix-tools dummy-ghc dummy-ghc-pkg hpack cabal-install pkgs.evalPackages.rsync ];
     # Needed or stack-to-nix will die on unicode inputs
     LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.stdenv.hostPlatform.libc == "glibc") "${pkgs.glibcLocales}/lib/locale/locale-archive";
     LANG = "en_US.UTF-8";

lib/default.nix

@@ -1,8 +1,32 @@
-{ stdenv, lib, haskellLib, runCommand, git, recurseIntoAttrs, srcOnly }:
+{ pkgs, stdenv, lib, haskellLib, recurseIntoAttrs, srcOnly }:
+
 
 with haskellLib;
 
-{
+let
+  # Why `final.evalPackages.buildPackages.git`?
+  # Why not just final.evalPackages.git?
+  #
+  # A problem arises when `evalPackages` is `buildPackages`.i
+  # As may be the case in a flake.
+  #
+  # It turns out `git` depends on `gdb` in a round about way:
+  #  git -> openssh -> libfido2 -> systemd -> python libxml -> Cython -> gdb
+  # Somewhere in that chain there should perhaps be a `buildPackages` so
+  # that the `gdb` that is used is not the one for debugging code in
+  # the `final` (but instead the one for debugging code in
+  # `final.buildPackages`).
+  #
+  # Using `final.buildPackages.git` causes two problems:
+  #
+  #   * Multiple versions of `git` (and that dependency chain
+  #     to `gdb` are needed when cross compiling).
+  #   * When `gdb` does not exist for `js`, so when cross
+  #     compiling with ghcjs `final.buildPackages.git` fails
+  #     to build at all.
+  inherit (pkgs.evalPackages.buildPackages) git;
+
+in {
   # Within the package components, these are the attribute names of
   # nested attrsets.
   subComponentTypes = [
@@ -181,7 +205,8 @@ with haskellLib;
 
   # Clean git directory based on `git ls-files --recurse-submodules`
   cleanGit = import ./clean-git.nix {
-    inherit lib runCommand git cleanSourceWith;
+    inherit lib git cleanSourceWith;
+    inherit (pkgs.evalPackages) runCommand;
   };
 
   # Check a test component