Skip to content

Commit e239fcf

Browse files
urvisavlaurvisavla
committed
Rework Restore change type handling based on meta changes in core
1 parent e79cd01 commit e239fcf

7 files changed

+399
-744
lines changed

‎ingest/change.go

+13-19
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ func (c Change) LedgerKey() (xdr.LedgerKey, error) {
137137
// - for create, pre is null and post is a new entry,
138138
// - for update, pre is previous state and post is the current state,
139139
// - for removed, pre is previous state and post is null.
140+
// - for restored, pre is null and post is a new/restored entry
140141
//
141142
// stellar-core source:
142143
// https://github.com/stellar/stellar-core/blob/e584b43/src/ledger/LedgerTxn.cpp#L582
@@ -153,16 +154,26 @@ func GetChangesFromLedgerEntryChanges(ledgerEntryChanges xdr.LedgerEntryChanges)
153154
ChangeType: entryChange.Type,
154155
})
155156
case xdr.LedgerEntryChangeTypeLedgerEntryUpdated:
156-
state := ledgerEntryChanges[i-1].MustState()
157157
updated := entryChange.MustUpdated()
158+
var state xdr.LedgerEntry
159+
if _, ok := ledgerEntryChanges[i-1].GetState(); ok {
160+
state = ledgerEntryChanges[i-1].MustState()
161+
} else {
162+
state = ledgerEntryChanges[i-1].MustRestored()
163+
}
158164
changes = append(changes, Change{
159165
Type: state.Data.Type,
160166
Pre: &state,
161167
Post: &updated,
162168
ChangeType: entryChange.Type,
163169
})
164170
case xdr.LedgerEntryChangeTypeLedgerEntryRemoved:
165-
state := ledgerEntryChanges[i-1].MustState()
171+
var state xdr.LedgerEntry
172+
if _, ok := ledgerEntryChanges[i-1].GetState(); ok {
173+
state = ledgerEntryChanges[i-1].MustState()
174+
} else {
175+
state = ledgerEntryChanges[i-1].MustRestored()
176+
}
166177
changes = append(changes, Change{
167178
Type: state.Data.Type,
168179
Pre: &state,
@@ -171,23 +182,6 @@ func GetChangesFromLedgerEntryChanges(ledgerEntryChanges xdr.LedgerEntryChanges)
171182
})
172183
case xdr.LedgerEntryChangeTypeLedgerEntryRestored:
173184
restored := entryChange.MustRestored()
174-
175-
// "restore" entries of type TTL can appear in two ways:
176-
// - "restore" alone, meaning an evicted item is being restored.
177-
// - "restore" with a preceding "state" entry, meaning an archived item is being restored.
178-
// Both are restore types, but the way the "Pre" field is populated differs.
179-
// For more details, see https://github.com/stellar/stellar-protocol/blob/master/core/cap-0062.md#meta
180-
if i > 0 {
181-
if state, ok := ledgerEntryChanges[i-1].GetState(); ok {
182-
changes = append(changes, Change{
183-
Type: restored.Data.Type,
184-
Pre: &state,
185-
Post: &restored,
186-
ChangeType: entryChange.Type,
187-
})
188-
continue
189-
}
190-
}
191185
changes = append(changes, Change{
192186
Type: restored.Data.Type,
193187
Pre: nil,

‎ingest/change_compactor.go

+50-45
Original file line numberDiff line numberDiff line change
@@ -25,55 +25,57 @@ import (
2525
//
2626
// 1. If the change is CREATED it checks if any change connected to given entry
2727
// is already in the cache. If not, it adds CREATED change. Otherwise, if
28-
// existing change is
29-
// a. CREATED: return an error because we can't add an entry that already exists.
30-
// b. UPDATED: return an error because we can't add an entry that already exists.
31-
// c. REMOVED: entry exists in the DB but was marked for removal; change the type
32-
// to UPDATED and update the new value.
33-
// d. RESTORED: return an error as the RESTORED change indicates the entry already
28+
// existing change is:
29+
// a. CREATED it returns error because we can't add an entry that already
3430
// exists.
35-
//
31+
// b. UPDATED it returns error because we can't add an entry that already
32+
// exists.
33+
// c. REMOVED it means that due to previous transitions we want to remove
34+
// this from a DB what means that it already exists in a DB so we need to
35+
// update the type of change to UPDATED.
36+
// d. RESTORED it returns an error as the RESTORED change indicates the
37+
// entry already exists.
3638
// 2. If the change is UPDATE it checks if any change connected to given entry
3739
// is already in the cache. If not, it adds UPDATE change. Otherwise, if
38-
// existing change is
39-
// a. CREATED: We want to create this in a DB which means that it doesn't exist
40-
// in a DB so we need to update the entry but stay with CREATED type.
41-
// b. UPDATED: update it with the new value.
40+
// existing change is:
41+
// a. CREATED it means that due to previous transitions we want to create
42+
// this in a DB what means that it doesn't exist in a DB so we need to
43+
// update the entry but stay with CREATED type.
44+
// b. UPDATED we simply update it with the new value.
4245
// c. REMOVED it means that at this point in the ledger the entry is removed
4346
// so updating it returns an error.
44-
// d. RESTORED: update it with the new value but keep the type as RESTORED.
45-
//
46-
// 3. If the change is REMOVED, it checks if any change related to the given entry
47-
// already exists in the cache. If not, it adds the `REMOVED` change. Otherwise,
48-
// if existing change is
49-
// a. CREATED: due to previous transitions we want to create
50-
// this in a DB which means that it doesn't exist in a DB. If it was created and
51-
// removed in the same ledger it's a noop so we remove the entry from the cache.
52-
// b. UPDATED: update it to be a REMOVE change because the UPDATE change means
53-
// the entry exists in a DB.
54-
// c. REMOVED: return an error because we can't remove an entry that was already
55-
// removed.
56-
// d. RESTORED: if the item was previously restored from an archived state, it means
57-
// it already exists in the DB, so change it to REMOVED type. If the restored item
58-
// was evicted, it doesn't exist in the DB, so it's a noop so remove the entry from
59-
// the cache.
60-
//
47+
// d. RESTORED we update it with the new value but keep the change type as RESTORED.
48+
// 3. If the change is REMOVE it checks if any change connected to given entry
49+
// is already in the cache. If not, it adds REMOVE change. Otherwise, if
50+
// existing change is:
51+
// a. CREATED it means that due to previous transitions we want to create
52+
// this in a DB what means that it doesn't exist in a DB. If it was
53+
// created and removed in the same ledger it's a noop so we remove entry
54+
// from the cache.
55+
// b. UPDATED we simply update it to be a REMOVE change because the UPDATE
56+
// change means the entry exists in a DB.
57+
// c. REMOVED it returns error because we can't remove an entry that was
58+
// already removed.
59+
// d. RESTORED it means the entry doesn't exist in the DB, so it's a noop
60+
// so remove the entry from the cache.
6161
// 4. If the change is RESTORED it checks if any change related to the given entry
6262
// already exists in the cache. If not, it adds the RESTORED change. Otherwise,
6363
// returns an error since restoration is only possible for previously archived/evicted
6464
// entries. If the entry was created, updated or removed within the same ledger, restoration
6565
// is not possible.
6666
type ChangeCompactor struct {
6767
// ledger key => Change
68-
cache map[string]Change
69-
encodingBuffer *xdr.EncodingBuffer
68+
cache map[string]Change
69+
encodingBuffer *xdr.EncodingBuffer
70+
emitExpiredEntriesRemovedChange bool
7071
}
7172

7273
// NewChangeCompactor returns a new ChangeCompactor.
73-
func NewChangeCompactor() *ChangeCompactor {
74+
func NewChangeCompactor(emitExpiredEntriesRemovedChange bool) *ChangeCompactor {
7475
return &ChangeCompactor{
75-
cache: make(map[string]Change),
76-
encodingBuffer: xdr.NewEncodingBuffer(),
76+
cache: make(map[string]Change),
77+
encodingBuffer: xdr.NewEncodingBuffer(),
78+
emitExpiredEntriesRemovedChange: emitExpiredEntriesRemovedChange,
7779
}
7880
}
7981

@@ -169,17 +171,21 @@ func (c *ChangeCompactor) addUpdatedChange(change Change) error {
169171

170172
switch existingChange.ChangeType {
171173
case xdr.LedgerEntryChangeTypeLedgerEntryCreated:
172-
fallthrough
174+
// If existing type is created it means that this entry does not
175+
// exist in a DB so we update entry change.
176+
c.cache[ledgerKeyString] = Change{
177+
Type: key.Type,
178+
Pre: existingChange.Pre, // = nil
179+
Post: change.Post,
180+
}
173181
case xdr.LedgerEntryChangeTypeLedgerEntryUpdated:
174182
fallthrough
175183
case xdr.LedgerEntryChangeTypeLedgerEntryRestored:
176-
// If existing type is created it means that this entry does not
177-
// exist in a DB so we update entry change.
178184
c.cache[ledgerKeyString] = Change{
179185
Type: key.Type,
180-
Pre: existingChange.Pre, // = nil for created type
186+
Pre: existingChange.Pre,
181187
Post: change.Post,
182-
ChangeType: existingChange.ChangeType,
188+
ChangeType: existingChange.ChangeType, //keep the existing change type
183189
}
184190
case xdr.LedgerEntryChangeTypeLedgerEntryRemoved:
185191
return NewStateError(errors.Errorf(
@@ -232,17 +238,16 @@ func (c *ChangeCompactor) addRemovedChange(change Change) error {
232238
base64.StdEncoding.EncodeToString(ledgerKey),
233239
))
234240
case xdr.LedgerEntryChangeTypeLedgerEntryRestored:
235-
if existingChange.Pre == nil {
236-
// Entry was created and removed in the same ledger; deleting it is effectively a noop.
237-
delete(c.cache, ledgerKeyString)
238-
} else {
239-
// If the entry exists, we mark it as removed by setting Post to nil.
241+
if c.emitExpiredEntriesRemovedChange {
240242
c.cache[ledgerKeyString] = Change{
241-
Type: existingChange.Type,
242-
Pre: existingChange.Pre,
243+
Type: key.Type,
244+
Pre: change.Pre,
243245
Post: nil,
244246
ChangeType: change.ChangeType,
245247
}
248+
} else {
249+
// Entry was restored and removed in the same ledger; deleting it is effectively a noop.
250+
delete(c.cache, ledgerKeyString)
246251
}
247252
default:
248253
return errors.Errorf("Unknown LedgerEntryChangeType: %d", existingChange.ChangeType)

0 commit comments

Comments
 (0)