Skip to content

Protocol changes for sync streams #307

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 21 commits into
base: main
Choose a base branch
from
Open

Protocol changes for sync streams #307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
1561867
Start with new unified syntax
simolus3 Jun 17, 2025
626358a
Support customizing subscriptions
simolus3 Jun 18, 2025
0d575b4
Include query streams in checkpoint message
simolus3 Jun 18, 2025
8af7af4
Avoid subscription terminology
simolus3 Jun 19, 2025
bd2962c
Rename more stream subscription references
simolus3 Jun 19, 2025
5534033
Translate resolved buckets
simolus3 Jul 21, 2025
bc77408
Deduplicate buckets
simolus3 Jul 21, 2025
5823e24
Introduce BucketSource interface
simolus3 Jul 30, 2025
62f8027
Fix sync rule tests
simolus3 Jul 30, 2025
4926033
Revert more sync-rule changes
simolus3 Jul 31, 2025
55df061
Unit tests for bucket checksum state
simolus3 Jul 31, 2025
bfd62d4
Fix more tsts
simolus3 Jul 31, 2025
3f5e9c3
Build correct index
simolus3 Jul 31, 2025
c4b8496
Review feedback
simolus3 Aug 5, 2025
38f6195
Update test snapshots
simolus3 Aug 5, 2025
7fb3be1
Include potential errors
simolus3 Aug 6, 2025
7a280ba
Remove core tests again
simolus3 Aug 6, 2025
aa21126
Complete docs sentence
simolus3 Aug 7, 2025
d2be184
Add changeset
simolus3 Aug 11, 2025
cc1adcd
Fix definition of override_priority
simolus3 Aug 13, 2025
c77b754
Fix tests
simolus3 Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/great-rabbits-exercise.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@powersync/service-sync-rules': minor
'@powersync/service-core': patch
'@powersync/service-image': patch
---

Refactor interface between service and sync rule bindings in preparation for sync streams.
197 changes: 197 additions & 0 deletions modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,21 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
"checksum": -93886621,
"count": 2,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"last_op_id": "2",
"streams": [
{
"errors": [],
"is_default": true,
"name": "mybucket",
},
],
"write_checkpoint": undefined,
},
},
Expand Down Expand Up @@ -46,6 +58,11 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
"checksum": 499012468,
"count": 4,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"write_checkpoint": undefined,
Expand Down Expand Up @@ -105,9 +122,21 @@ exports[`sync - mongodb > expiring token 1`] = `
"checksum": 0,
"count": 0,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"last_op_id": "0",
"streams": [
{
"errors": [],
"is_default": true,
"name": "mybucket",
},
],
"write_checkpoint": undefined,
},
},
Expand Down Expand Up @@ -137,9 +166,21 @@ exports[`sync - mongodb > sends checkpoint complete line for empty checkpoint 1`
"checksum": -1221282404,
"count": 1,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"last_op_id": "1",
"streams": [
{
"errors": [],
"is_default": true,
"name": "mybucket",
},
],
"write_checkpoint": undefined,
},
},
Expand Down Expand Up @@ -193,15 +234,37 @@ exports[`sync - mongodb > sync buckets in order 1`] = `
"checksum": 920318466,
"count": 1,
"priority": 2,
"subscriptions": [
{
"default": 0,
},
],
},
{
"bucket": "b1[]",
"checksum": -1382098757,
"count": 1,
"priority": 1,
"subscriptions": [
{
"default": 1,
},
],
},
],
"last_op_id": "2",
"streams": [
{
"errors": [],
"is_default": true,
"name": "b0",
},
{
"errors": [],
"is_default": true,
"name": "b1",
},
],
"write_checkpoint": undefined,
},
},
Expand Down Expand Up @@ -267,9 +330,21 @@ exports[`sync - mongodb > sync global data 1`] = `
"checksum": -93886621,
"count": 2,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"last_op_id": "2",
"streams": [
{
"errors": [],
"is_default": true,
"name": "mybucket",
},
],
"write_checkpoint": undefined,
},
},
Expand Down Expand Up @@ -319,21 +394,53 @@ exports[`sync - mongodb > sync interrupts low-priority buckets on new checkpoint
"checksum": -659831575,
"count": 2000,
"priority": 2,
"subscriptions": [
{
"default": 0,
},
],
},
{
"bucket": "b0b[]",
"checksum": -659831575,
"count": 2000,
"priority": 2,
"subscriptions": [
{
"default": 1,
},
],
},
{
"bucket": "b1[]",
"checksum": -1096116670,
"count": 1,
"priority": 1,
"subscriptions": [
{
"default": 2,
},
],
},
],
"last_op_id": "4001",
"streams": [
{
"errors": [],
"is_default": true,
"name": "b0a",
},
{
"errors": [],
"is_default": true,
"name": "b0b",
},
{
"errors": [],
"is_default": true,
"name": "b1",
},
],
"write_checkpoint": undefined,
},
},
Expand Down Expand Up @@ -380,18 +487,33 @@ exports[`sync - mongodb > sync interrupts low-priority buckets on new checkpoint
"checksum": 883076828,
"count": 2001,
"priority": 2,
"subscriptions": [
{
"default": 0,
},
],
},
{
"bucket": "b0b[]",
"checksum": 883076828,
"count": 2001,
"priority": 2,
"subscriptions": [
{
"default": 1,
},
],
},
{
"bucket": "b1[]",
"checksum": 1841937527,
"count": 2,
"priority": 1,
"subscriptions": [
{
"default": 2,
},
],
},
],
"write_checkpoint": undefined,
Expand Down Expand Up @@ -466,9 +588,21 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
"checksum": -852817836,
"count": 1,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"last_op_id": "1",
"streams": [
{
"errors": [],
"is_default": true,
"name": "mybucket",
},
],
"write_checkpoint": undefined,
},
},
Expand Down Expand Up @@ -514,9 +648,21 @@ exports[`sync - mongodb > sync updates to data query only 1`] = `
"checksum": 0,
"count": 0,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"last_op_id": "1",
"streams": [
{
"errors": [],
"is_default": true,
"name": "by_user",
},
],
"write_checkpoint": undefined,
},
},
Expand All @@ -540,6 +686,11 @@ exports[`sync - mongodb > sync updates to data query only 2`] = `
"checksum": 1418351250,
"count": 1,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"write_checkpoint": undefined,
Expand Down Expand Up @@ -582,9 +733,21 @@ exports[`sync - mongodb > sync updates to global data 1`] = `
"checksum": 0,
"count": 0,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"last_op_id": "0",
"streams": [
{
"errors": [],
"is_default": true,
"name": "mybucket",
},
],
"write_checkpoint": undefined,
},
},
Expand All @@ -608,6 +771,11 @@ exports[`sync - mongodb > sync updates to global data 2`] = `
"checksum": 920318466,
"count": 1,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"write_checkpoint": undefined,
Expand Down Expand Up @@ -652,6 +820,11 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
"checksum": -93886621,
"count": 2,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"write_checkpoint": undefined,
Expand Down Expand Up @@ -690,6 +863,13 @@ exports[`sync - mongodb > sync updates to parameter query + data 1`] = `
"checkpoint": {
"buckets": [],
"last_op_id": "0",
"streams": [
{
"errors": [],
"is_default": true,
"name": "by_user",
},
],
"write_checkpoint": undefined,
},
},
Expand All @@ -713,6 +893,11 @@ exports[`sync - mongodb > sync updates to parameter query + data 2`] = `
"checksum": 1418351250,
"count": 1,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"write_checkpoint": undefined,
Expand Down Expand Up @@ -751,6 +936,13 @@ exports[`sync - mongodb > sync updates to parameter query only 1`] = `
"checkpoint": {
"buckets": [],
"last_op_id": "0",
"streams": [
{
"errors": [],
"is_default": true,
"name": "by_user",
},
],
"write_checkpoint": undefined,
},
},
Expand All @@ -774,6 +966,11 @@ exports[`sync - mongodb > sync updates to parameter query only 2`] = `
"checksum": 0,
"count": 0,
"priority": 3,
"subscriptions": [
{
"default": 0,
},
],
},
],
"write_checkpoint": undefined,
Expand Down
Loading