feat(stores): Splitting Sled and Indexeddb Store into separate crates

More details in pull request #498 .

Author: gnunicorn

.github/workflows/appservice.yml

@@ -5,12 +5,18 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
   test-appservice:
+    if: github.event_name == 'push' || !github.event.pull_request.draft
     name: ${{ matrix.name }}
 
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}

.github/workflows/ci.yml

@@ -1,10 +1,16 @@
 name: CI
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
   pull_request:
     branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
 
 env:
   CARGO_TERM_COLOR: always
@@ -13,6 +19,7 @@ jobs:
   style:
     name: Check style
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     steps:
     - name: Checkout the repo
@@ -36,6 +43,7 @@ jobs:
     name: Spell Check with Typos
     needs: [style]
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     steps:
     - name: Checkout Actions Repository
@@ -48,6 +56,7 @@ jobs:
     name: Run clippy
     needs: [style]
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     steps:
     - name: Checkout the repo
@@ -72,6 +81,7 @@ jobs:
 
   test-features:
     name: ${{ matrix.name }}
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
     strategy:
@@ -141,6 +151,7 @@ jobs:
 
   test:
     name: ${{ matrix.name }}
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
     strategy:

.github/workflows/docs.yml

@@ -9,6 +9,7 @@ jobs:
   docs:
     name: Docs
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     steps:
       - name: Checkout repository

.github/workflows/wasm.yml

@@ -5,6 +5,11 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
 
 env:
   CARGO_TERM_COLOR: always
@@ -13,25 +18,24 @@ jobs:
   check-wasm:
     name: Build test
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.draft
 
     strategy:
-#      fail-fast: true
-
+      fail-fast: true
       matrix:
         name:
           - matrix-qrcode
           - matrix-sdk (no-default, wasm-flags)
           - matrix-sdk-base
           - matrix-sdk-common
           - matrix-sdk-crypto
-          - matrix-sdk-base / indexeddb_state_store
-          - matrix-sdk-crypto / indexeddb_cryptostore
+          - matrix-sdk / indexeddb_stores
 
         include:
           - name: matrix-qrcode
             cargo_args: --package matrix-qrcode
           - name: matrix-sdk (no-default, wasm-flags)
-            cargo_args: --no-default-features --features qrcode,encryption,indexeddb_stores,rustls-tls --package matrix-sdk
+            cargo_args: --no-default-features --features qrcode,encryption,indexeddb_stores,rustls-tls --package matrix-sdk --lib
           - name: matrix-sdk-base
             cargo_args: --package matrix-sdk-base
           - name: matrix-sdk-common
@@ -40,10 +44,8 @@ jobs:
             cargo_args: --package matrix-sdk-crypto
 
           # special check for specific features
-          - name: matrix-sdk-base / indexeddb_state_store
-            cargo_args: --package matrix-sdk-base --features indexeddb_state_store
-          - name: matrix-sdk-crypto / indexeddb_cryptostore
-            cargo_args: --package matrix-sdk-crypto --features indexeddb_cryptostore
+          - name: matrix-sdk / indexeddb_stores
+            cargo_args: --package matrix-sdk --no-default-features --features indexeddb_stores,encryption,rustls-tls --lib
 
     steps:
     - name: Checkout the repo
@@ -59,7 +61,6 @@ jobs:
 
     - name: Load cache
       uses: Swatinem/rust-cache@v1
-
     # needed for libolm-sys compilation
     - name: Install emscripten
       uses: mymindstorm/setup-emsdk@v11
@@ -76,26 +77,26 @@ jobs:
     # building wasm is not enough, we've seen runtime errors before,
     # hence the tests
     name: ${{ matrix.name }} test
+    if: github.event_name == 'push' || !github.event.pull_request.draft
     runs-on: ubuntu-latest
     continue-on-error: ${{ matrix.experimental }}
 
     strategy:
       matrix:
         include:
-          - name: "matrix-sdk-crypto @Node14 emcc2.0"
-            experimental: false
-            emcc_version: 2.0.27
-            base_dir: matrix-sdk-crypto
-            cargo_args: --features indexeddb_cryptostore
-          - name: matrix-sdk-base
-            experimental: false
-            base_dir: matrix-sdk-base
-            cargo_args: --features indexeddb_state_store,indexeddb_cryptostore
           - name: matrix-sdk
             experimental: false
             base_dir: matrix-sdk
-            # FIXME: we have to skip all examples b/c of https://github.com/rustwasm/wasm-pack/issues/666
-            cargo_args:  --no-default-features --features indexeddb_stores,rustls-tls --lib
+            cargo_args: --no-default-features --features indexeddb_stores,rustls-tls --lib
+          - name: matrix-sdk-encryption
+            experimental: false
+            emcc_version: 2.0.27
+            node_version: '14'
+            base_dir: matrix-sdk
+            cargo_args: --no-default-features --features indexeddb_stores,rustls-tls,encryption --lib
+          - name: "matrix-sdk-indexeddb"
+            base_dir: matrix-sdk-indexeddb
+            experimental: false
           - name: "matrix-sdk-example-wasm_command_bot @Node14"
             experimental: false
             emcc_version: 2.0.27
@@ -106,11 +107,6 @@ jobs:
               npm install
               npm test
               wasm-pack test --firefox --headless
-
-          # these are _known_ broken on latest node
-          # - name: matrix-sdk-crypto
-          #   experimental: true
-          #   base_dir: matrix-sdk-crypto
           - name: matrix-sdk-example-wasm_command_bot
             base_dir: matrix-sdk/examples/wasm_command_bot
             # this might fail
@@ -160,14 +156,14 @@ jobs:
         emcc -v
 
     - name: Default wasm-pack tests
-      if: ${{!matrix.cmd}}
+      if: ${{ !matrix.cmd }}
       run: |
         cd crates/${{matrix.base_dir}}
         wasm-pack test --node -- ${{ matrix.cargo_args }}
-        wasm-pack test --firefox --headless -- --features ${{ matrix.cargo_args }}
+        wasm-pack test --firefox --headless -- ${{ matrix.cargo_args }}
 
     - name: Testing with custom command
-      if: ${{matrix.cmd}}
+      if: ${{ matrix.cmd }}
       run: |
         cd crates/${{matrix.base_dir}}
         ${{matrix.cmd}}

crates/matrix-sdk-base/Cargo.toml

@@ -17,21 +17,19 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [features]
 default = []
-encryption = ["matrix-sdk-crypto"]
+encryption = ["matrix-sdk-crypto", "store_key"]
 qrcode = ["matrix-sdk-crypto/qrcode"]
-sled_state_store = [
-    "sled",
-    "tokio",
+
+# Store Key is a helper feature for state store implementations
+store_key = [
     "pbkdf2",
     "hmac",
     "sha2",
     "rand",
     "chacha20poly1305",
 ]
-sled_cryptostore = ["matrix-sdk-crypto/sled_cryptostore"]
-
-indexeddb_state_store = ["indexed_db_futures", "wasm-bindgen", "pbkdf2", "hmac", "sha2", "rand", "chacha20poly1305"]
-indexeddb_cryptostore = ["matrix-sdk-crypto/indexeddb_cryptostore"]
+# helpers for testing features build upon this
+testing = [ "http" ]
 
 [dependencies]
 async-stream = "0.3.2"
@@ -50,38 +48,17 @@ ruma = { version = "0.5.0", features = ["client-api-c", "signatures", "unstable-
 serde = { version = "1.0.126", features = ["rc"] }
 serde_json = "1.0.64"
 sha2 = { version = "0.10.1", optional = true }
-sled = { version = "0.34.6", optional = true }
 thiserror = "1.0.25"
 tracing = "0.1.26"
 zeroize = { version = "1.3.0", features = ["zeroize_derive"] }
-
-## Feature sled_state_store
-tokio = { version = "1.7.1", optional = true, default-features = false, features = ["sync", "fs"] }
-
-## Feature indexeddb-state-store
-indexed_db_futures = { version = "0.2.0", optional = true }
-wasm-bindgen = { version = "0.2.74", features = ["serde-serialize"], optional = true }
+anyhow = "1"
+http = { version = "0.2.4", optional = true }
 
 [dev-dependencies]
 futures = { version = "0.3.15", default-features = false, features = ["executor"] }
 http = "0.2.4"
 matrix-sdk-test = { version = "0.4.0", path = "../matrix-sdk-test" }
-
-[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
-atty = "0.2.14"
-clap = "3.1.0"
-rustyline = "9.0.0"
-rustyline-derive = "0.6.0"
-syntect = "4.5.0"
 tokio = { version = "1.7.1", default-features = false, features = [
     "rt-multi-thread",
     "macros",
 ] }
-tempfile = "3.2.0"
-
-[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
-wasm-bindgen-test = "0.3.24"
-
-[[example]]
-name = "state_inspector"
-required-features = ["sled_state_store"]

crates/matrix-sdk-base/README.md

@@ -6,9 +6,6 @@ library.
 The following crate feature flags are available:
 
 * `encryption`: Enables end-to-end encryption support in the library.
-* `sled_cryptostore`: Enables a Sled based store for the encryption
-  keys. If this is disabled and `encryption` support is enabled the keys will
-  by default be stored only in memory and thus lost after the client is
-  destroyed.
-* `sled_state_store`: Enables a Sled based store for the storage of the local
-  client state.
+* `qrcode`: Enbles QRcode generation and reading code
+* `store_key`: Provides extra facilities for StateStores to manage store keys 
+* `testing`: provides facilities and functions for tests, in particular for integration testing store implementations. ATTENTION: do not ever use outside of tests, we do not provide any stability warantees on these, these are merely helpers. If you find you _need_ any function provided here outside of tests, please open a Github Issue and inform us about your use case for us to consider.