Include all switch depexts in Nix derivation

We support this by including `all_packages` in a switch in a call to
`OpamSolution.get_depexts`, including all system packages for the packages in a
switch as `required` in a call to `OpamSysInteract.install, and including all of
these in the Nix derivation generate by the Nix depext backend.

src/client/opamClient.ml

@@ -1578,7 +1578,7 @@ let check_for_sys_packages config system_packages =
   if system_packages <> [] then
     let (missing, required, available) =
       OpamSysInteract.packages_status config
-        (OpamSysPkg.Set.of_list system_packages)
+        (OpamSysPkg.Set.of_list system_packages) ~old_packages:OpamSysPkg.Set.empty
     in
     if not (OpamSysPkg.Set.is_empty missing) then
       let vars = OpamFile.Config.global_variables config in
@@ -2272,7 +2272,8 @@ let install_t t ?ask ?(ignore_conflicts=false) ?(depext_only=false)
       in
       if depext_only then
         (OpamSolution.install_depexts ~force_depext:true ~confirm:false t
-           (OpamSolver.all_packages solution)), None
+           ~new_packages:(OpamSolver.all_packages solution)
+           ~all_packages:t.installed), None
       else
       let add_roots =
         OpamStd.Option.map (function

src/client/opamSolution.ml

@@ -1114,11 +1114,12 @@ let print_depext_msg (avail, nf) =
 
 (* Gets depexts from the state, without checking again, unless [recover] is
    true. *)
-let get_depexts ?(force=false) ?(recover=false) t packages =
+let get_depexts ?(force=false) ?(recover=false) t ~new_packages ~all_packages =
   if not force && OpamStateConfig.(!r.no_depexts) then OpamSysPkg.Set.empty, OpamSysPkg.Set.empty else
   let sys_packages =
     if recover then
-      OpamSwitchState.depexts_status_of_packages t packages
+      OpamSwitchState.depexts_status_of_packages t new_packages
+        ~old_packages:(OpamPackage.Set.diff all_packages new_packages)
     else
       let base = Lazy.force t.sys_packages in
       (* workaround: st.sys_packages is not always updated with added
@@ -1128,11 +1129,11 @@ let get_depexts ?(force=false) ?(recover=false) t packages =
             (* dirty heuristic: recompute for all non-canonical packages *)
             OpamPackage.Map.find_opt nv t.repos_package_index
             <> OpamSwitchState.opam_opt t nv)
-          packages
+          new_packages
       in
       if OpamPackage.Set.is_empty more_pkgs then base else
       OpamPackage.Map.union (fun _ x -> x) base
-        (OpamSwitchState.depexts_status_of_packages t more_pkgs)
+        (OpamSwitchState.depexts_status_of_packages t more_pkgs ~old_packages:(OpamPackage.Set.diff all_packages more_pkgs))
   in
   let avail, required, nf =
     OpamPackage.Set.fold (fun pkg (avail,req,nf) ->
@@ -1142,7 +1143,7 @@ let get_depexts ?(force=false) ?(recover=false) t packages =
           OpamSysPkg.(Set.union req sys.s_required),
           OpamSysPkg.(Set.union nf sys.s_not_found)
         | None -> avail, req, nf)
-      packages (OpamSysPkg.Set.empty, OpamSysPkg.Set.empty, OpamSysPkg.Set.empty)
+      all_packages (OpamSysPkg.Set.empty, OpamSysPkg.Set.empty, OpamSysPkg.Set.empty)
   in
   print_depext_msg (avail, nf);
   avail, required
@@ -1195,7 +1196,7 @@ let install_sys_packages ~map_sysmap ~confirm ~sys_packages ~required env config
     if OpamSysPoll.os_distribution env = Some "cygwin" then
       OpamSysInteract.Cygwin.check_setup ~update:false;
     let commands =
-      OpamSysInteract.install_packages_commands ~env (Option.map (fun t -> t.switch) t) config sys_packages
+      OpamSysInteract.install_packages_commands ~env (Option.map (fun t -> t.switch) t) config sys_packages ~required
       |> List.map (fun ((`AsAdmin c | `AsUser c), a) -> c::a)
     in
     OpamConsole.formatted_msg
@@ -1226,16 +1227,15 @@ let install_sys_packages ~map_sysmap ~confirm ~sys_packages ~required env config
     try
       if OpamSysPoll.os_distribution env = Some "cygwin" then
         OpamSysInteract.Cygwin.check_setup ~update:true;
-      let install_packages = OpamSysPkg.Set.union sys_packages required in
-      OpamSysInteract.install ~env (Option.map (fun t -> t.switch) t) config install_packages; (* handles dry_run *)
+      OpamSysInteract.install ~env (Option.map (fun t -> t.switch) t) config sys_packages ~required; (* handles dry_run *)
       map_sysmap (fun _ -> OpamSysPkg.Set.empty) t
     with Failure msg ->
       OpamConsole.error "%s" msg;
       check_again t sys_packages
   and check_again t sys_packages =
     let open OpamSysPkg.Set.Op in
     let needed, required, notfound =
-      OpamSysInteract.packages_status ~env config sys_packages
+      OpamSysInteract.packages_status ~env config sys_packages ~old_packages:required
     in
     let still_missing = needed ++ notfound in
     let installed = sys_packages -- still_missing in
@@ -1282,7 +1282,7 @@ let install_sys_packages ~map_sysmap ~confirm ~sys_packages ~required env config
     entry_point t sys_packages required
   with Sys.Break as e -> OpamStd.Exn.finalise e give_up_msg
 
-let install_depexts ?(force_depext=false) ?(confirm=true) t packages =
+let install_depexts ?(force_depext=false) ?(confirm=true) t ~new_packages ~all_packages =
   let map_sysmap f t =
     let t = Option.get t in
     let sys_packages =
@@ -1294,7 +1294,7 @@ let install_depexts ?(force_depext=false) ?(confirm=true) t packages =
                                  f status.OpamSysPkg.s_available }
               sys_map
           | None -> sys_map)
-        packages
+        new_packages
         (Lazy.force t.sys_packages)
     in
     Some ({ t with sys_packages = lazy sys_packages })
@@ -1303,7 +1303,7 @@ let install_depexts ?(force_depext=false) ?(confirm=true) t packages =
     confirm && not (OpamSysInteract.Cygwin.is_internal t.switch_global.config)
   in
   let sys_packages, required =
-    get_depexts ~force:force_depext ~recover:force_depext t packages
+    get_depexts ~force:force_depext ~recover:force_depext t ~new_packages ~all_packages
   in
   let env = t.switch_global.global_variables in
   let config = t.switch_global.config in
@@ -1333,9 +1333,9 @@ let apply ?ask t ~requested ?print_requested ?add_roots
     in
     let t =
       if OpamClientConfig.(!r.show) then
-        let _ = get_depexts t virt_inst in t
+        let _ = get_depexts t ~new_packages:virt_inst ~all_packages:t.installed in t
         (* Prints the msg about additional depexts to install *)
-      else install_depexts t virt_inst
+      else install_depexts t ~new_packages:virt_inst ~all_packages:t.installed
     in
     t, Nothing_to_do
   else (
@@ -1389,13 +1389,14 @@ let apply ?ask t ~requested ?print_requested ?add_roots
         solution0;
     );
     if OpamClientConfig.(!r.show) then
-      let _ = get_depexts t new_state0.installed in
+      let _ = get_depexts t ~new_packages:new_state0.installed ~all_packages:new_state0.installed in
       (* Prints the msg about additional depexts to install *)
       t, Aborted
     else if download_only || confirmation ?ask names solution then (
       let t =
-        install_depexts t  @@ OpamPackage.Set.inter
-          new_state0.installed (OpamSolver.all_packages solution0)
+        install_depexts t
+          ~new_packages:(OpamPackage.Set.inter new_state0.installed (OpamSolver.all_packages solution0))
+          ~all_packages:new_state0.installed
       in
       let requested =
         OpamPackage.packages_of_names new_state.installed names

src/client/opamSolution.mli

@@ -90,7 +90,7 @@ val install_sys_packages: confirm:bool -> sys_packages:OpamSysPkg.Set.t -> requi
    launched, without asking user (used by the `--depext-only` option). If
    [force_depext] is true, it overrides [OpamFile.Config.depext] value. *)
 val install_depexts: ?force_depext:bool -> ?confirm:bool -> rw switch_state ->
-  package_set -> rw switch_state
+  new_packages:package_set -> all_packages:package_set -> rw switch_state
 
 (** {2 Atoms} *)
 

src/state/opamSwitchState.ml

@@ -179,7 +179,7 @@ module Installed_cache = OpamCached.Make(struct
   end)
 
 let depexts_status_of_packages_raw
-    ~depexts ?env global_config switch_config packages =
+    ~depexts ?env global_config switch_config packages ~old_packages =
   if OpamPackage.Set.is_empty packages then OpamPackage.Map.empty else
   let open OpamSysPkg.Set.Op in
   let syspkg_set, syspkg_map =
@@ -190,14 +190,19 @@ let depexts_status_of_packages_raw
         else OpamPackage.Map.add nv s map)
       packages (OpamSysPkg.Set.empty, OpamPackage.Map.empty)
   in
+  let old_syspkg_set =
+    OpamPackage.Set.fold (fun nv set -> depexts nv ++ set)
+      old_packages OpamSysPkg.Set.empty
+  in
   let chronos = OpamConsole.timer () in
   let bypass =
     OpamFile.Config.depext_bypass global_config ++
     switch_config.OpamFile.Switch_config.depext_bypass
   in
   let syspkg_set = syspkg_set -- bypass in
   let ret =
-    match OpamSysInteract.packages_status ?env global_config syspkg_set with
+    Printf.printf "here\n\r";
+    match OpamSysInteract.packages_status ?env global_config syspkg_set ~old_packages:old_syspkg_set with
     | avail, required, not_found ->
       let avail, not_found =
         if OpamStateConfig.(!r.no_depexts) then
@@ -518,6 +523,7 @@ let load lock_kind gt rt switch =
       depexts_status_of_packages_raw gt.config switch_config
         ~env:gt.global_variables
         (Lazy.force available_packages)
+        ~old_packages:(Lazy.force available_packages)
         ~depexts:(fun package ->
             let env =
               OpamPackageVar.resolve_switch_raw ~package gt switch switch_config
@@ -1260,7 +1266,7 @@ let update_pin nv opam st =
   let sys_packages = lazy (
     OpamPackage.Map.union (fun _ n -> n)
       (Lazy.force st.sys_packages)
-      (depexts_status_of_packages st (OpamPackage.Set.singleton nv))
+      (depexts_status_of_packages st (OpamPackage.Set.singleton nv) ~old_packages:(OpamPackage.Set.singleton nv))
   ) in
   let available_packages = lazy (
     OpamPackage.Set.filter (fun nv -> depexts_unavailable st nv = None)

src/state/opamSwitchState.mli

@@ -185,7 +185,8 @@ val reverse_dependencies:
 (** Returns required system packages of each of the given packages (elements are
     not added to the map  if they don't have system dependencies) *)
 val depexts_status_of_packages:
-  'a switch_state -> package_set -> OpamSysPkg.status package_map
+  'a switch_state -> package_set -> old_packages:package_set ->
+  OpamSysPkg.status package_map
 
 (** Returns not found depexts for the package *)
 val depexts_unavailable: 'a switch_state -> package -> OpamSysPkg.Set.t option

src/state/opamSysInteract.ml

@@ -471,7 +471,7 @@ let yum_cmd = lazy begin
     raise (OpamSystem.Command_not_found "yum or dnf")
 end
 
-let packages_status ?(env=OpamVariable.Map.empty) config packages =
+let packages_status ?(env=OpamVariable.Map.empty) config packages ~old_packages =
   let (+++) pkg set = OpamSysPkg.Set.add (OpamSysPkg.of_string pkg) set in
   (* Some package managers don't permit to request on available packages. In
      this case, we consider all non installed packages as [available]. *)
@@ -969,8 +969,7 @@ let packages_status ?(env=OpamVariable.Map.empty) config packages =
          be found.' But omitting them will mean that they won't be
          added to the Nix derivation.
       *)
-      (* TODO *)
-      packages, OpamSysPkg.Set.empty, OpamSysPkg.Set.empty
+      OpamSysPkg.Set.diff packages old_packages, old_packages, OpamSysPkg.Set.empty
   | Openbsd ->
     let sys_installed =
       run_query_command "pkg_info" ["-mqP"]
@@ -1006,7 +1005,7 @@ let packages_status ?(env=OpamVariable.Map.empty) config packages =
 
 (* Install *)
 
-let install_packages_commands_t ?(env=OpamVariable.Map.empty) switch config sys_packages =
+let install_packages_commands_t ?(env=OpamVariable.Map.empty) switch config sys_packages ~required =
   let unsafe_yes = OpamCoreConfig.answer_is `unsafe_yes in
   let yes ?(no=[]) yes r =
     if unsafe_yes then
@@ -1102,7 +1101,10 @@ let install_packages_commands_t ?(env=OpamVariable.Map.empty) switch config sys_
     | Some switch ->
     let dir = OpamPath.Switch.meta OpamStateConfig.(!r.root_dir) switch in
     let drvFile = create dir (basename (raw "env.nix")) in
-    let packages = String.concat " " (OpamSysPkg.Set.fold (fun p l -> OpamSysPkg.to_string p :: l) sys_packages []) in
+    let packages = String.concat " "
+      (OpamSysPkg.Set.fold (fun p l -> OpamSysPkg.to_string p :: l)
+      OpamSysPkg.Set.Op.(sys_packages ++ required) [])
+    in
     let contents =
 {|{ pkgs ? import <nixpkgs> {} }:
 with pkgs;
@@ -1130,11 +1132,11 @@ echo "PATH	+=	$PATH	Nix" >> $out
   | Openbsd -> [`AsAdmin "pkg_add", yes ~no:["-i"] ["-I"] packages], None
   | Suse -> [`AsAdmin "zypper", yes ["--non-interactive"] ("install"::packages)], None
 
-let install_packages_commands ?env switch config sys_packages =
-  fst (install_packages_commands_t ?env switch config sys_packages)
+let install_packages_commands ?env switch config sys_packages ~required =
+  fst (install_packages_commands_t ?env switch config sys_packages ~required)
 
 let package_manager_name ?env switch config =
-  match install_packages_commands ?env switch config OpamSysPkg.Set.empty with
+  match install_packages_commands ?env switch config OpamSysPkg.Set.empty ~required:OpamSysPkg.Set.empty with
   | ((`AsAdmin pkgman | `AsUser pkgman), _) :: _ -> pkgman
   | [] -> assert false
 
@@ -1159,11 +1161,11 @@ let sudo_run_command ?(env=OpamVariable.Map.empty) ?vars cmd args =
       "failed with exit code %d at command:\n    %s"
       code (String.concat " " (cmd::args))
 
-let install ?env switch config packages =
+let install ?env switch config packages ~required =
   if OpamSysPkg.Set.is_empty packages then
     log "Nothing to install"
   else
-    let commands, vars = install_packages_commands_t ?env switch config packages in
+    let commands, vars = install_packages_commands_t ?env switch config packages ~required in
     let vars = OpamStd.Option.map (List.map (fun x -> `add, x)) vars in
     List.iter
       (fun (cmd, args) ->
@@ -1213,7 +1215,7 @@ let repo_enablers ?(env=OpamVariable.Map.empty) config =
   if family ~env () <> Centos then None else
   let (needed, _, _) =
     packages_status ~env config (OpamSysPkg.raw_set
-                       (OpamStd.String.Set.singleton "epel-release"))
+                       (OpamStd.String.Set.singleton "epel-release")) ~old_packages:OpamSysPkg.Set.empty
   in
   if OpamSysPkg.Set.is_empty needed then None
   else

src/state/opamSysInteract.mli

@@ -19,18 +19,19 @@ open OpamStateTypes
      * third one, not found set: packages not found on the defined repositories
    [env] is used to determine host specification. *)
 val packages_status:
-  ?env:gt_variables -> OpamFile.Config.t -> OpamSysPkg.Set.t
-  -> OpamSysPkg.Set.t * OpamSysPkg.Set.t * OpamSysPkg.Set.t
+  ?env:gt_variables -> OpamFile.Config.t -> OpamSysPkg.Set.t ->
+  old_packages:OpamSysPkg.Set.t -> OpamSysPkg.Set.t * OpamSysPkg.Set.t * OpamSysPkg.Set.t
 
 (* Return the commands to run to install given system packages.
    [env] is used to determine host specification. *)
 val install_packages_commands:
   ?env:gt_variables -> OpamSwitch.t option -> OpamFile.Config.t -> OpamSysPkg.Set.t ->
-  ([`AsAdmin of string | `AsUser of string] * string list) list
+  required:OpamSysPkg.Set.t -> ([`AsAdmin of string | `AsUser of string] * string list) list
 
 (* Install given system packages, by calling local system package manager.
    [env] is used to determine host specification. *)
-val install: ?env:gt_variables -> OpamSwitch.t option -> OpamFile.Config.t -> OpamSysPkg.Set.t -> unit
+val install: ?env:gt_variables -> OpamSwitch.t option -> OpamFile.Config.t ->
+  OpamSysPkg.Set.t -> required:OpamSysPkg.Set.t -> unit
 
 val update: ?env:gt_variables -> OpamFile.Config.t -> unit