Add support for static nix-tools (#2157)

* Add support for static build of nix-tools

* Add default-setup and default-setup-ghcjs to nix-tools

* Update cabal to 3.10.3

* Use nix-tools.exes.cabal when possible

Author: zeme-wana

.github/workflows/upload-artifacts.yml

@@ -0,0 +1,89 @@
+name: Wait for Hydra Build
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "nix-tools-*"
+  pull_request:
+
+env:
+  FLAKE_REF: github:${{ github.repository }}?ref=${{ github.head_ref || github.ref }}
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  wait-for-hydra:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Nix with good defaults
+        uses: input-output-hk/install-nix-action@v20
+        with:
+          extra_nix_config: |
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
+            substituters = https://cache.nixos.org/ https://cache.iog.io/ https://cache.zw3rk.com
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - name: Display flake metadata
+        id: flake-metadata
+        run: |
+          echo $PWD
+          ls -lah .
+          nix flake metadata ${{ env.FLAKE_REF }}
+          nix flake metadata ${{ env.FLAKE_REF }} --json | jq -r '"LOCKED_URL=\(.url)"' >> "$GITHUB_OUTPUT"
+
+      - name: Wait for nix-tools meta job
+        uses: input-output-hk/actions/wait-for-hydra@angerman/support-prs
+        with:
+          status: 'ci/hydra-build:nix-tools'
+
+      - name: "Pull nix-tools"
+        run: |
+          ls -lah .
+          cp $(nix build --builders "" --max-jobs 0 ${{ steps.flake-metadata.outputs.LOCKED_URL }}#hydraJobs.aarch64-darwin.nix-tools.static.zipped.nix-tools-static-no-ifd --no-link --print-out-paths)/*.zip .
+          cp $(nix build --builders "" --max-jobs 0 ${{ steps.flake-metadata.outputs.LOCKED_URL }}#hydraJobs.x86_64-darwin.nix-tools.static.zipped.nix-tools-static-no-ifd --no-link --print-out-paths)/*.zip .
+          cp $(nix build --builders "" --max-jobs 0 ${{ steps.flake-metadata.outputs.LOCKED_URL }}#hydraJobs.x86_64-linux.nix-tools.static.zipped.nix-tools-static-no-ifd --no-link --print-out-paths)/*.zip .
+          cp $(nix build --builders "" --max-jobs 0 ${{ steps.flake-metadata.outputs.LOCKED_URL }}#hydraJobs.x86_64-linux.nix-tools.static.zipped.nix-tools-static-arm64-no-ifd --no-link --print-out-paths)/*.zip .
+
+      - name: Configure Git
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          git config --global user.name 'Auto Update Bot'
+          git config --global user.email '[email protected]'
+
+      - name: "Compute nix-tools-static.nix"
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          git fetch --all
+          git switch --force static-nix-tools
+          (
+            echo "pkgs: let baseurl = \"https://github.com/input-output-hk/haskell.nix/releases/download/${GITHUB_REF_NAME}/\"; in {"
+            for arch in aarch64-darwin x86_64-darwin aarch64-linux x86_64-linux; do
+              echo "  ${arch} = pkgs.fetchurl { "
+              echo "     name = \"${arch}-nix-tools-static\";"
+              echo "     url = \"\${baseurl}${arch}-nix-tools-static.zip\";"
+              echo "     sha256 = \"$(nix-hash --sri --type sha256 --flat ${arch}-nix-tools-static.zip)\";"
+              echo "  };"
+            done
+            echo "}"
+          ) > nix-tools-static.nix
+          cat nix-tools-static.nix
+
+
+      - name: Push to nix branch
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          git add nix-tools-static.nix
+          git commit -m "update nix-tools-static.nix"
+          git push origin static-nix-tools
+
+      - name: Release
+        uses: input-output-hk/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: |
+            *-nix-tools-static.zip

build.nix

@@ -26,7 +26,7 @@ in rec {
 
   tools = pkgs.lib.optionalAttrs (ifdLevel >= 3) (
     pkgs.recurseIntoAttrs ({
-      cabal-latest = tool compiler-nix-name "cabal" { inherit evalPackages; cabalProjectLocal = builtins.readFile ./test/cabal.project.local; };
+      cabal-latest = tool compiler-nix-name "cabal" { inherit evalPackages; };
     } // pkgs.lib.optionalAttrs (__compareVersions haskell.compiler.${compiler-nix-name}.version "9.8" < 0) {
       hlint-latest = tool compiler-nix-name "hlint" {
         inherit evalPackages;
@@ -91,11 +91,7 @@ in rec {
     };
     check-hydra = pkgs.buildPackages.callPackage ./scripts/check-hydra.nix {};
     check-closure-size = pkgs.buildPackages.callPackage ./scripts/check-closure-size.nix {
-      # Includes cabal-install since this is commonly used.
-      nix-tools = pkgs.linkFarm "common-tools" [
-        { name = "nix-tools";     path = haskell.nix-tools; }
-        { name = "cabal-install"; path = haskell.cabal-install.${compiler-nix-name}; }
-      ];
+      nix-tools = haskell.nix-tools-unchecked; # includes cabal-install and default-setup
     };
     check-materialization-concurrency = pkgs.buildPackages.callPackage ./scripts/check-materialization-concurrency/check.nix {};
     check-path-support = pkgsForGitHubAction.buildPackages.callPackage ./scripts/check-path-support.nix {
@@ -114,7 +110,7 @@ in rec {
         # Some of the dependencies of the impure scripts so that they will
         # will be in the cache too for buildkite.
         inherit (pkgs.buildPackages) glibc coreutils git openssh cabal-install nix-prefetch-git;
-        inherit (haskell) nix-tools;
+        nix-tools = pkgs.haskell-nix.nix-tools-unchecked;
       })
   );
 }

builder/comp-builder.nix

@@ -110,8 +110,8 @@ let
 
   ghc = (if enableDWARF then (x: x.dwarf) else (x: x)) (
         (if smallAddressSpace then (x: x.smallAddressSpace) else (x: x)) defaults.ghc);
-  setup = (if enableDWARF then (x: x.dwarf) else (x: x)) (
-        (if smallAddressSpace then (x: x.smallAddressSpace) else (x: x)) drvArgs.setup);
+  setup = (if enableDWARF then (x: x.dwarf or x) else (x: x)) (
+          (if smallAddressSpace then (x: x.smallAddressSpace or x) else (x: x)) drvArgs.setup);
 
   # TODO fix cabal wildcard support so hpack wildcards can be mapped to cabal wildcards
   canCleanSource = !(cabal-generator == "hpack" && !(package.cleanHpack or false));
@@ -285,7 +285,7 @@ let
 
       enableParallelBuilding = true;
 
-      SETUP_HS = setup + /bin/Setup;
+      SETUP_HS = setup + "/bin/${setup.exeName}";
 
       inherit cabalFile;
       passAsFile = [ "cabalFile" ];
@@ -635,7 +635,11 @@ let
       ''))
       + (lib.optionalString doCoverage ''
         mkdir -p $out/share
-        cp -r dist/${lib.optionalString (builtins.compareVersions defaults.ghc.version "9.9" >= 0) "build/extra-compilation-artifacts/"}hpc $out/share
+        if [ -d dist/build/extra-compilation-artifacts ]; then
+          cp -r dist/build/extra-compilation-artifacts/hpc $out/share
+        else
+          cp -r dist/hpc $out/share
+        fi
         cp dist/setup-config $out/
       '')
       }

builder/hspkg-builder.nix

@@ -35,10 +35,11 @@ let
 
   setup = if package.buildType == "Simple"
     then
-      # Don't try to build default setup with DWARF enabled
-      let defaultSetup = ghc.defaultSetupFor package.identifier.name // {
-        dwarf = defaultSetup;
-      }; in defaultSetup
+      if stdenv.targetPlatform.isGhcjs # TODO probably should be hostPlatform, but only HsColour used to build ghc will change (updating will require rebuilding all the ghcjs versions)
+        then
+          buildPackages.haskell-nix.nix-tools-unchecked.exes.default-setup-ghcjs // { exeName = "default-setup-ghcjs"; }
+        else
+          buildPackages.haskell-nix.nix-tools-unchecked.exes.default-setup // { exeName = "default-setup"; }
     else setup-builder ({
       component = components.setup // {
         depends = config.setup-depends ++ components.setup.depends ++ package.setup-depends;

builder/setup-builder.nix

@@ -77,6 +77,7 @@ let
         cleanSrc = cleanSrc';
         dwarf = self (drvArgs // { enableDWARF = true; });
         smallAddressSpace = self (drvArgs // { smallAddressSpace = true; });
+        exeName = "Setup";
       };
 
       meta = {

flake.lock

@@ -39,10 +39,6 @@
       url = "github:input-output-hk/stackage.nix";
       flake = false;
     };
-    nix-tools-static = {
-      url = "github:input-output-hk/haskell-nix-example/nix";
-      flake = false;
-    };
     cabal-32 = {
       url = "github:haskell/cabal/3.2";
       flake = false;
@@ -135,7 +131,7 @@
         stripAttrsForHydra
         filterDerivations;
 
-    in traceHydraJobs ({
+      flake = {
       inherit config;
       overlay = self.overlays.combined;
       overlays = import ./overlays { sources = inputs; };
@@ -250,10 +246,6 @@
               inherit system;
               pkgs = self.legacyPackages.${system};
               src = ./nix-tools;
-              override-inputs = {
-                # Avoid downloading another `hackage.nix`.
-                inherit (inputs) hackage;
-              };
             };
             in cf.defaultNix.hydraJobs;
         in
@@ -290,7 +282,23 @@
                 "ghc901"
                 "ghc921" "ghc922" "ghc923"])
       );
-    });
+    }; in with (import nixpkgs { system = "x86_64-linux"; });
+          traceHydraJobs (lib.recursiveUpdate flake {
+            hydraJobs.nix-tools = pkgs.releaseTools.aggregate {
+              name = "nix-tools";
+              constituents = [
+                "aarch64-darwin.nix-tools.static.zipped.nix-tools-static"
+                "x86_64-darwin.nix-tools.static.zipped.nix-tools-static"
+                "x86_64-linux.nix-tools.static.zipped.nix-tools-static"
+                "x86_64-linux.nix-tools.static.zipped.nix-tools-static-arm64"
+                "aarch64-darwin.nix-tools.static.zipped.nix-tools-static-no-ifd"
+                "x86_64-darwin.nix-tools.static.zipped.nix-tools-static-no-ifd"
+                "x86_64-linux.nix-tools.static.zipped.nix-tools-static-no-ifd"
+                "x86_64-linux.nix-tools.static.zipped.nix-tools-static-arm64-no-ifd"
+                (writeText "gitrev" (self.rev or "0000000000000000000000000000000000000000"))
+              ];
+            };
+          });
 
   # --- Flake Local Nix Configuration ----------------------------
   nixConfig = {

flake.nix

@@ -128,7 +128,7 @@ let
   # This works because `cabal configure` does not include any of the `/nix/sore/`
   # paths in the `plan.json` (so materialized plan-nix will still work as expeced).
   # See tests/unit.nix for examples of input and output.
-  parseRepositoryBlock = evalPackages: _cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: block:
+  parseRepositoryBlock = evalPackages: _cabalProjectFileName: sha256map: inputMap: nix-tools: block:
     let
       lines = pkgs.lib.splitString "\n" block;
       # The first line will contain the repository name.
@@ -151,7 +151,7 @@ let
       repoContents = if inputMap ? ${attrs.url}
         # If there is an input use it to make `file:` url and create a suitable `.cabal/packages/${name}` directory
         then evalPackages.runCommand name ({
-          nativeBuildInputs = [ cabal-install ] ++ evalPackages.haskell-nix.cabal-issue-8352-workaround;
+          nativeBuildInputs = [ nix-tools.exes.cabal ] ++ evalPackages.haskell-nix.cabal-issue-8352-workaround;
           preferLocalBuild = true;
         }) ''
             HOME=$(mktemp -d)
@@ -167,7 +167,7 @@ let
             cp -r $HOME/.cabal/packages/${name} $out
         ''
         else evalPackages.runCommand name ({
-          nativeBuildInputs = [ cabal-install evalPackages.curl nix-tools ] ++ evalPackages.haskell-nix.cabal-issue-8352-workaround;
+          nativeBuildInputs = [ nix-tools.exes.cabal evalPackages.curl ] ++ evalPackages.haskell-nix.cabal-issue-8352-workaround;
           LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
           LANG = "en_US.UTF-8";
           preferLocalBuild = true;
@@ -192,7 +192,7 @@ let
       # Output of hackage-to-nix
       hackage = import (
         evalPackages.runCommand ("hackage-to-nix-" + name) {
-          nativeBuildInputs = [ cabal-install evalPackages.curl nix-tools ];
+          nativeBuildInputs = [ nix-tools.exes.hackage-to-nix ];
           LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
           LANG = "en_US.UTF-8";
           preferLocalBuild = true;
@@ -204,11 +204,11 @@ let
       repo.${name} = repoContents;
     };
 
-  parseRepositories = evalPackages: cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: projectFile:
+  parseRepositories = evalPackages: cabalProjectFileName: sha256map: inputMap: nix-tools: projectFile:
     let
       # This will leave the name of repository in the first line of each block
       blocks = pkgs.lib.splitString "\nrepository " ("\n" + projectFile);
-      repoBlocks = builtins.map (parseRepositoryBlock evalPackages cabalProjectFileName sha256map inputMap cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
+      repoBlocks = builtins.map (parseRepositoryBlock evalPackages cabalProjectFileName sha256map inputMap nix-tools) (pkgs.lib.lists.drop 1 blocks);
     in {
       extra-hackages = pkgs.lib.lists.map (block: block.hackage) repoBlocks;
       repos = pkgs.lib.lists.foldl' (x: block: x // block.repo) {} repoBlocks;

lib/cabal-project-parser.nix

@@ -70,11 +70,10 @@ let
   # These defaults are here rather than in modules/cabal-project.nix to make them
   # lazy enough to avoid infinite recursion issues.
   # Using null as the default also improves performance as they are not forced by the
-  # nix module system for `nix-tools-unchecked` and `cabal-install-unchecked`.
+  # nix module system for `nix-tools-unchecked`.
   nix-tools = if args.nix-tools or null != null
     then args.nix-tools
     else evalPackages.haskell-nix.nix-tools-unchecked;
-  cabal-install = nix-tools.exes.cabal;
   forName = pkgs.lib.optionalString (name != null) (" for " + name);
   nameAndSuffix = suffix: if name == null then suffix else name + "-" + suffix;
 
@@ -262,7 +261,7 @@ let
 
       # Parse the `repository` blocks
       repoResult = pkgs.haskell-nix.haskellLib.parseRepositories
-        evalPackages cabalProjectFileName sha256map inputMap cabal-install nix-tools sourceRepoFixedProjectFile;
+        evalPackages cabalProjectFileName sha256map inputMap nix-tools sourceRepoFixedProjectFile;
     in {
       sourceRepos = sourceReposBuild;
       inherit (repoResult) repos extra-hackages;
@@ -451,7 +450,7 @@ let
       # some packages that will be excluded by `index-state-max`
       # which is used by cabal (cached-index-state >= index-state-max).
       dotCabal {
-        inherit cabal-install nix-tools extra-hackage-tarballs;
+        inherit nix-tools extra-hackage-tarballs;
         extra-hackage-repos = fixedProject.repos;
         index-state = cached-index-state;
         sha256 = index-sha256-found;