Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ingest: Add support for CAP-63 #5615

Merged
merged 4 commits into from
Feb 27, 2025
Merged

ingest: Add support for CAP-63 #5615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9299162
Add support for CAP-63
tamirms Feb 26, 2025
dc49ed1
code review feedback
tamirms Feb 26, 2025
39ebc15
update etl processor
tamirms Feb 27, 2025
77cfaea
add test case
tamirms Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions ingest/change_compactor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,14 @@ func (c *ChangeCompactor) getLedgerKey(ledgerEntry *xdr.LedgerEntry) ([]byte, er
return ledgerKey, nil
}

// maxTTL returns the ttl entry with the highest LiveUntilLedgerSeq
func maxTTL(a, b xdr.TtlEntry) xdr.TtlEntry {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Acknowledged for the maxTTL

if a.LiveUntilLedgerSeq > b.LiveUntilLedgerSeq {
return a
}
return b
}

// addUpdatedChange adds a change to the cache, but returns an error if update
// change is unexpected.
func (c *ChangeCompactor) addUpdatedChange(change Change) error {
Expand All @@ -179,22 +187,19 @@ func (c *ChangeCompactor) addUpdatedChange(change Change) error {
}

switch existingChange.ChangeType {
case xdr.LedgerEntryChangeTypeLedgerEntryCreated:
// If existing type is created it means that this entry does not
// exist in a DB so we update entry change.
c.cache[ledgerKeyString] = Change{
Type: change.Type,
Pre: existingChange.Pre, // = nil
Post: change.Post,
ChangeType: existingChange.ChangeType,
case xdr.LedgerEntryChangeTypeLedgerEntryCreated,
xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LedgerEntryUpdated's are now cached into the compactor all the time, before it was dropped, don't see any changes in existing tests that may have broken related to this, I do see the new coverage on TTL's, is there potential edge case where LedgerEntryUpdated should still be dropped or are TTL's the only source of an LedgerEntryUpdated?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LedgerEntryUpdated's are now cached into the compactor all the time, before it was dropped

the behavior is actually the same. The fallthrough keyword means it should match the case below. In other words,

	case xdr.LedgerEntryChangeTypeLedgerEntryUpdated:
		fallthrough
	case xdr.LedgerEntryChangeTypeLedgerEntryRestored:

is equivalent to:

	case xdr.LedgerEntryChangeTypeLedgerEntryUpdated, xdr.LedgerEntryChangeTypeLedgerEntryRestored:

xdr.LedgerEntryChangeTypeLedgerEntryRestored:
post := change.Post
if change.Type == xdr.LedgerEntryTypeTtl {
// CAP-63 introduces special update semantics for TTL entries, see
// https://github.com/stellar/stellar-protocol/blob/master/core/cap-0063.md#ttl-ledger-change-semantics
*post.Data.Ttl = maxTTL(*existingChange.Post.Data.Ttl, *post.Data.Ttl)
}
case xdr.LedgerEntryChangeTypeLedgerEntryUpdated:
fallthrough
case xdr.LedgerEntryChangeTypeLedgerEntryRestored:
c.cache[ledgerKeyString] = Change{
Type: change.Type,
Pre: existingChange.Pre,
Post: change.Post,
Post: post,
ChangeType: existingChange.ChangeType, //keep the existing change type
}
case xdr.LedgerEntryChangeTypeLedgerEntryRemoved:
Expand Down
219 changes: 219 additions & 0 deletions ingest/change_compactor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,3 +601,222 @@ func TestChangeCompactorSquashMultiplePayments(t *testing.T) {
}
}
}

func TestCompactTTLUpdates(t *testing.T) {
cache := NewChangeCompactor(ChangeCompactorConfig{})
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Pre: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 10,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 11,
},
},
},
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 13,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
}))
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Pre: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 10,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 11,
},
},
},
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 15,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
}))
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Pre: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 10,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 11,
},
},
},
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 12,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
}))

changes := cache.GetChanges()
assert.Len(t, changes, 1)
assert.Equal(t, xdr.Uint32(15), changes[0].Post.Data.MustTtl().LiveUntilLedgerSeq)
assert.Equal(t, xdr.Hash{1}, changes[0].Post.Data.MustTtl().KeyHash)

cache = NewChangeCompactor(ChangeCompactorConfig{})
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 15,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryCreated,
}))
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Pre: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 15,
},
},
},
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 30,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
}))
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Pre: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 15,
},
},
},
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 22,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
}))

changes = cache.GetChanges()
assert.Len(t, changes, 1)
assert.Equal(t, xdr.Uint32(30), changes[0].Post.Data.MustTtl().LiveUntilLedgerSeq)
assert.Equal(t, xdr.Hash{1}, changes[0].Post.Data.MustTtl().KeyHash)

cache = NewChangeCompactor(ChangeCompactorConfig{})
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 15,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryRestored,
}))
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Pre: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 15,
},
},
},
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 30,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
}))
assert.NoError(t, cache.AddChange(Change{
Type: xdr.LedgerEntryTypeTtl,
Pre: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 15,
},
},
},
Post: &xdr.LedgerEntry{
LastModifiedLedgerSeq: 12,
Data: xdr.LedgerEntryData{
Type: xdr.LedgerEntryTypeTtl,
Ttl: &xdr.TtlEntry{
KeyHash: xdr.Hash{1},
LiveUntilLedgerSeq: 22,
},
},
},
ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
}))

changes = cache.GetChanges()
assert.Len(t, changes, 1)
assert.Equal(t, xdr.Uint32(30), changes[0].Post.Data.MustTtl().LiveUntilLedgerSeq)
assert.Equal(t, xdr.Hash{1}, changes[0].Post.Data.MustTtl().KeyHash)
}
Loading
Loading