Store the lock in the manifest

This is to address an issue where a _build restored
for another version of a dependency would come up as
valid.

Author: josevalim

lib/mix/lib/mix/compilers/elixir.ex

@@ -965,7 +965,8 @@ defmodule Mix.Compilers.Elixir do
     # Since Elixir is a dependency itself, we need to touch the lock
     # so the current Elixir version, used to compile the files above,
     # is properly stored.
-    Mix.Dep.ElixirSCM.update(Mix.Project.config()[:build_scm])
+    config = Mix.Project.config()
+    Mix.Dep.ElixirSCM.update(config[:build_scm], config[:deps_lock])
     :ok
   end
 

lib/mix/lib/mix/dep.ex

@@ -258,10 +258,18 @@ defmodule Mix.Dep do
       |> Keyword.put(:build_scm, scm)
       |> Keyword.put(:deps_app_path, opts[:build])
 
-    # If the dependency is not fetchable, then it is never compiled
-    # from scratch and therefore it needs the parent configuration
-    # files to know when to recompile.
-    config = [inherit_parent_config_files: not scm.fetchable?()] ++ config
+    config =
+      if scm.fetchable?() do
+        # If the dependency is fetchable, we want to store the lock in
+        # the Elixir SCM file.
+        [deps_lock: opts[:lock]] ++ config
+      else
+        # If the dependency is not fetchable, then it is never compiled
+        # from scratch and therefore it needs the parent configuration
+        # files to know when to recompile.
+        [inherit_parent_config_files: true] ++ config
+      end
+
     env = opts[:env] || :prod
     old_env = Mix.env()
 
@@ -436,23 +444,35 @@ defmodule Mix.Dep do
           %{dep | status: :lockoutdated}
 
         :ok ->
-          check_manifest(dep, opts[:build])
+          check_manifest(dep)
       end
     else
       dep
     end
   end
 
-  defp check_manifest(%{scm: scm} = dep, build_path) do
+  defp check_manifest(%{scm: scm, opts: opts} = dep) do
     vsn = {System.version(), :erlang.system_info(:otp_release)}
+    lock = opts[:lock]
 
-    case Mix.Dep.ElixirSCM.read(Path.join(build_path, ".mix")) do
-      {:ok, old_vsn, _} when old_vsn != vsn ->
+    case Mix.Dep.ElixirSCM.read(Path.join(opts[:build], ".mix")) do
+      {:ok, old_vsn, _, _} when old_vsn != vsn ->
         %{dep | status: {:vsnlock, old_vsn}}
 
-      {:ok, _, old_scm} when old_scm != scm ->
+      {:ok, _, old_scm, _} when old_scm != scm ->
         %{dep | status: {:scmlock, old_scm}}
 
+      {:ok, _, _, old_lock} when old_lock != lock ->
+        %{dep | status: :compile}
+
+      :error ->
+        if scm.fetchable?() do
+          %{dep | status: :compile}
+        else
+          dep
+        end
+
+      # If the file is missing, it is handled in the loader
       _ ->
         dep
     end

lib/mix/lib/mix/dep/elixir_scm.ex

@@ -3,20 +3,21 @@
 # SPDX-FileCopyrightText: 2012 Plataformatec
 
 # Manifest file where we treat Erlang/OTP, Elixir and SCMs as a dependency.
+# We also store the exact version this dependency was compiled for.
 defmodule Mix.Dep.ElixirSCM do
   @moduledoc false
   @manifest "compile.elixir_scm"
-  @manifest_vsn 1
+  @manifest_vsn 2
 
   def manifest(manifest_path \\ Mix.Project.manifest_path()) do
     Path.join(manifest_path, @manifest)
   end
 
-  def update(manifest_path \\ Mix.Project.manifest_path(), scm) do
+  def update(manifest_path \\ Mix.Project.manifest_path(), scm, lock) do
     File.mkdir_p!(manifest_path)
 
     manifest_data =
-      {@manifest_vsn, {System.version(), :erlang.system_info(:otp_release)}, scm}
+      {@manifest_vsn, {System.version(), :erlang.system_info(:otp_release)}, scm, lock}
       |> :erlang.term_to_binary()
 
     File.write!(manifest(manifest_path), manifest_data)
@@ -26,10 +27,10 @@ defmodule Mix.Dep.ElixirSCM do
     case File.read(manifest(manifest_path)) do
       {:ok, contents} ->
         try do
-          {@manifest_vsn, vsn, scm} = :erlang.binary_to_term(contents)
-          {:ok, vsn, scm}
+          {@manifest_vsn, vsn, scm, lock} = :erlang.binary_to_term(contents)
+          {:ok, vsn, scm, lock}
         rescue
-          _ -> {:ok, {"1.0.0", ~c"17"}, nil}
+          _ -> {:ok, {"1.0.0", ~c"17"}, nil, nil}
         end
 
       _ ->

lib/mix/lib/mix/dep/fetcher.ex

@@ -63,8 +63,8 @@ defmodule Mix.Dep.Fetcher do
 
       # If the dependency is not available or we have a lock mismatch
       out_of_date?(dep) ->
-        # Mark the dependency as fetched upfront, in case updating
-        # fails, get interrupted, or corrupted.
+        # Mark the dependency as fetched upfront, in case updating fails,
+        # gets interrupted, or corrupted.
         mark_as_fetched([dep])
 
         new =

lib/mix/lib/mix/dep/loader.ex

@@ -389,9 +389,6 @@ defmodule Mix.Dep.Loader do
       not ok?(dep) ->
         dep
 
-      recently_fetched?(dep) ->
-        %{dep | status: :compile}
-
       opts_app == false ->
         dep
 
@@ -406,10 +403,6 @@ defmodule Mix.Dep.Loader do
     end
   end
 
-  defp recently_fetched?(%Mix.Dep{opts: opts, scm: scm}) do
-    scm.fetchable?() and not File.exists?(Path.join(opts[:build], ".mix/compile.elixir_scm"))
-  end
-
   defp app_status(app_path, app, req) do
     case Mix.AppLoader.read_app(app, app_path) do
       {:ok, properties} ->

lib/mix/lib/mix/tasks/compile.app.ex

@@ -188,6 +188,7 @@ defmodule Mix.Tasks.Compile.App do
         |> add_modules(modules, compile_path)
 
       contents = to_erl_term({:application, app, properties})
+      :application.unload(app)
       :application.load({:application, app, properties})
 
       Mix.Project.ensure_structure()

lib/mix/lib/mix/tasks/deps.compile.ex

@@ -104,17 +104,18 @@ defmodule Mix.Tasks.Deps.Compile do
 
     # If all dependencies are local and ok, do not bother starting
     # processes as it will likely slow everything down.
-    compiled? =
-      if count > 1 and match?([_, _ | _], deps) and
-           not Enum.all?(deps, &(Mix.Dep.ok?(&1) and not &1.scm.fetchable?())) do
-        Mix.shell().info("mix deps.compile running across #{count} OS processes")
-        Mix.Tasks.Deps.Partition.server(deps, count, force?)
-      else
-        config = Mix.Project.deps_config()
-        true in Enum.map(deps, &compile_single(&1, force?, config))
-      end
+    if count > 1 and match?([_, _ | _], deps) and
+         not Enum.all?(deps, &(Mix.Dep.ok?(&1) and not &1.scm.fetchable?())) do
+      Mix.shell().info("mix deps.compile running across #{count} OS processes")
 
-    if compiled?, do: Mix.Task.run("will_recompile"), else: :ok
+      if Mix.Tasks.Deps.Partition.server(deps, count, force?) do
+        # Each partition will trigger will_recompile but we need to trigger it here too
+        Mix.Task.run("will_recompile")
+      end
+    else
+      config = Mix.Project.deps_config()
+      Enum.each(deps, &compile_single(&1, force?, config))
+    end
   end
 
   @doc false
@@ -169,8 +170,14 @@ defmodule Mix.Tasks.Deps.Compile do
     # We should touch fetchable dependencies even if they
     # did not compile otherwise they will always be marked
     # as stale, even when there is nothing to do.
-    fetchable? = touch_fetchable(scm, opts[:build])
-    compiled? and fetchable?
+    fetchable? = touch_fetchable(scm, opts)
+
+    if compiled? and fetchable? do
+      Mix.Task.run("will_recompile")
+      true
+    else
+      false
+    end
   end
 
   defp check_unavailable!(app, scm, {:unavailable, path}) do
@@ -192,9 +199,9 @@ defmodule Mix.Tasks.Deps.Compile do
     :ok
   end
 
-  defp touch_fetchable(scm, path) do
+  defp touch_fetchable(scm, opts) do
     if scm.fetchable?() do
-      Mix.Dep.ElixirSCM.update(Path.join(path, ".mix"), scm)
+      Mix.Dep.ElixirSCM.update(Path.join(opts[:build], ".mix"), scm, opts[:lock])
       true
     else
       false

lib/mix/lib/mix/tasks/loadpaths.ex

@@ -59,7 +59,7 @@ defmodule Mix.Tasks.Loadpaths do
     # any of SCM or Elixir version changes. Applies
     # to dependencies and the main project alike.
     case Mix.Dep.ElixirSCM.read() do
-      {:ok, old_vsn, old_scm} when old_vsn != vsn or old_scm != scm ->
+      {:ok, old_vsn, old_scm, _deps_lock} when old_vsn != vsn or old_scm != scm ->
         if Mix.debug?() do
           Mix.shell().info(
             "-- Recompiling #{inspect(config[:app])} because Erlang/OTP, Elixir, or SCM version changed"

lib/mix/test/mix/tasks/compile.elixir_test.exs

@@ -608,7 +608,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
 
       assert File.exists?("_build/dev/lib/sample")
       assert File.exists?("_build/dev/lib/sample/consolidated")
-      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path}
+      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path, nil}
 
       Mix.Task.clear()
       File.write!("_build/dev/lib/sample/consolidated/.to_be_removed", "")
@@ -617,7 +617,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
       File.touch!("_build/dev/lib/sample/.mix/compile.elixir_scm", @old_time)
 
       Mix.Tasks.Compile.run([])
-      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path}
+      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path, nil}
 
       assert mtime("_build/dev/lib/sample/.mix/compile.elixir_scm") > @old_time
       refute File.exists?("_build/dev/lib/sample/consolidated/.to_be_removed")
@@ -631,15 +631,15 @@ defmodule Mix.Tasks.Compile.ElixirTest do
       purge([A, B])
 
       assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
-      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path}
+      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path, nil}
 
       Mix.Task.clear()
       manifest_data = :erlang.term_to_binary({1, @elixir_otp_version, :another})
       File.write!("_build/dev/lib/sample/.mix/compile.elixir_scm", manifest_data)
       File.touch!("_build/dev/lib/sample/.mix/compile.elixir_scm", @old_time)
 
       Mix.Tasks.Compile.run([])
-      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path}
+      assert Mix.Dep.ElixirSCM.read() == {:ok, @elixir_otp_version, Mix.SCM.Path, nil}
 
       assert mtime("_build/dev/lib/sample/.mix/compile.elixir_scm") > @old_time
     end)
@@ -704,9 +704,6 @@ defmodule Mix.Tasks.Compile.ElixirTest do
       Mix.State.clear_cache()
       purge([GitRepo.MixProject, PathOnGitRepo.MixProject, PathOnGitRepo.Hello])
 
-      # Unload the git repo application so we can pick the new modules definition
-      :ok = Application.unload(:git_repo)
-
       Mix.Tasks.Deps.Update.run(["--all"])
       assert File.read!("mix.lock") =~ last
 

lib/mix/test/mix/tasks/deps_test.exs

@@ -764,7 +764,9 @@ defmodule Mix.Tasks.DepsTest do
       File.mkdir_p!("_build/dev/lib/ok/.mix")
 
       manifest_data =
-        :erlang.term_to_binary({1, {System.version(), :erlang.system_info(:otp_release)}, :scm})
+        :erlang.term_to_binary(
+          {2, {System.version(), :erlang.system_info(:otp_release)}, :scm, nil}
+        )
 
       File.write!("_build/dev/lib/ok/.mix/compile.elixir_scm", manifest_data)
       Mix.Task.clear()