set journal mode at storage_up time. this allows us to avoid problems… (#25)

* set journal mode at storage_up time. this allows us to avoid problems with setting it in parallel across all connections at connection time, which can hit database locked errors

* format

* bump exqlite, add changelog

Author: kevinlang

.github/workflows/ci.yml

@@ -45,6 +45,6 @@ jobs:
       # - run: mix compile --warnings-as-errors
       #   if: ${{ matrix.lint }}
 
-      - run: mix test
+      - run: mix test --trace
 
-      - run: EXQLITE_INTEGRATION=true mix test
+      - run: EXQLITE_INTEGRATION=true mix test --trace

CHANGELOG.md

@@ -5,6 +5,9 @@ All notable changes will be documented in this file.
 The format is based on [Keep a Changelog][keepachangelog], and this project
 adheres to [Semantic Versioning][semver].
 
+## [0.5.5] - Unreleased
+- Fix "database is locked" issue by setting `journal_mode` at `storage_up` time.
+
 
 ## [0.5.4] - 2021-04-06
 - Upgrade `ecto_sql` dependency to 3.6.0

lib/ecto/adapters/sqlite3.ex

@@ -157,9 +157,10 @@ defmodule Ecto.Adapters.SQLite3 do
 
   @impl Ecto.Adapter.Storage
   def storage_up(options) do
-    options
-    |> Keyword.get(:database)
-    |> storage_up_with_path()
+    storage_up_with_path(
+      Keyword.get(options, :database),
+      Keyword.get(options, :journal_mode, :wal)
+    )
   end
 
   @impl Ecto.Adapter.Migration
@@ -324,7 +325,7 @@ defmodule Ecto.Adapters.SQLite3 do
   ## HELPERS
   ##
 
-  defp storage_up_with_path(nil) do
+  defp storage_up_with_path(nil, _) do
     raise ArgumentError,
           """
           No SQLite database path specified. Please check the configuration for your Repo.
@@ -336,12 +337,13 @@ defmodule Ecto.Adapters.SQLite3 do
           """
   end
 
-  defp storage_up_with_path(db_path) do
+  defp storage_up_with_path(db_path, journal_mode) do
     if File.exists?(db_path) do
       {:error, :already_up}
     else
       db_path |> Path.dirname() |> File.mkdir_p!()
       {:ok, db} = Exqlite.Sqlite3.open(db_path)
+      :ok = Exqlite.Sqlite3.execute(db, "PRAGMA JOURNAL_MODE = #{journal_mode}")
       :ok = Exqlite.Sqlite3.close(db)
     end
   end

lib/ecto/adapters/sqlite3/connection.ex

@@ -15,32 +15,13 @@ defmodule Ecto.Adapters.SQLite3.Connection do
   import Ecto.Adapters.SQLite3.DataType
 
   @parent_as __MODULE__
-  @idx_connect_buffer 5
-  @rand_connect_buffer 50
-
-  # db_connection eagerly obtains and holds the configured
-  # pool_size of connections, which can cause us to hit
-  # "SQLite Busy" issues. we solve this by making sure to wait
-  # a bit by hooking into the :configure callback opt and
-  # sleeping an amount of time which is a function of the pool_index
-  def handle_thundering_herd(opts) do
-    case Keyword.get(opts, :pool_index) do
-      idx when is_integer(idx) -> :timer.sleep(idx * @idx_connect_buffer)
-      nil -> :timer.sleep(:rand.uniform(@rand_connect_buffer))
-    end
-
-    opts
-  end
 
   defp default_opts(opts) do
-    # todo: we may want to consider wrapping any provided :configure
-    # with our custom connection buffering logic
     opts
     |> Keyword.put_new(:journal_mode, :wal)
     |> Keyword.put_new(:cache_size, -64000)
     |> Keyword.put_new(:temp_store, :memory)
     |> Keyword.put_new(:pool_size, 5)
-    |> Keyword.put_new(:configure, &handle_thundering_herd/1)
   end
 
   def start_link(opts) do

mix.lock

@@ -10,7 +10,7 @@
   "ecto_sql": {:hex, :ecto_sql, "3.6.0", "5cb277b086618a644f2c5450316202a885716bb7726b9f13b74cb0708bea3a8f", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.6.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3486d6e29ee4a0e7a381390c9c289bfbbaf5dc1971e269c579799d2300e5bd5"},
   "elixir_make": {:hex, :elixir_make, "0.6.2", "7dffacd77dec4c37b39af867cedaabb0b59f6a871f89722c25b28fcd4bd70530", [:mix], [], "hexpm", "03e49eadda22526a7e5279d53321d1cced6552f344ba4e03e619063de75348d9"},
   "ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"},
-  "exqlite": {:hex, :exqlite, "0.5.8", "b1942f9f51b27608917ec2f035e69f7ff6d846eccff5157e27c90c14a5d1a168", [:make, :mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "5bed331bf406c27c7dae42040b835f1371018595a9dc218f9b92927e8a7c47c5"},
+  "exqlite": {:hex, :exqlite, "0.5.10", "678f1237e877b87dd59e56ee13d63a0479640aa99094ead946bcd8451b492868", [:make, :mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "d30baa1fca4d87b1bc80c2cc591fe398d81457e1b5975d2e5963d233b1275c1a"},
   "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
   "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"},