Skip to content

Commit 574f14f

Browse files
SirTysonSirTyson
committed
Cleanup
1 parent ce031d1 commit 574f14f

File tree

5 files changed

+73
-67
lines changed

5 files changed

+73
-67
lines changed

‎src/bucket/Bucket.cpp

+2-3
Original file line numberDiff line numberDiff line change
@@ -141,15 +141,14 @@ Bucket::apply(Application& app) const
141141
{
142142
ZoneScoped;
143143

144-
UnorderedSet<LedgerKey> emptySet;
144+
std::unordered_set<LedgerKey> emptySet;
145145
BucketApplicator applicator(
146146
app, app.getConfig().LEDGER_PROTOCOL_VERSION,
147147
0 /*set to 0 so we always load from the parent to check state*/,
148148
0 /*set to a level that's not the bottom so we don't treat live entries
149149
as init*/
150150
,
151-
shared_from_this(), [](LedgerEntryType) { return true; },
152-
/*newOffersOnly=*/app.getConfig().isUsingBucketListDB(), emptySet);
151+
shared_from_this(), [](LedgerEntryType) { return true; }, emptySet);
153152
BucketApplicator::Counters counters(app.getClock().now());
154153
while (applicator)
155154
{

‎src/bucket/BucketApplicator.cpp

+12-9
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,13 @@ BucketApplicator::BucketApplicator(Application& app,
2222
uint32_t level,
2323
std::shared_ptr<Bucket const> bucket,
2424
std::function<bool(LedgerEntryType)> filter,
25-
bool newOffersOnly,
26-
UnorderedSet<LedgerKey>& seenKeys)
25+
std::unordered_set<LedgerKey>& seenKeys)
2726
: mApp(app)
2827
, mMaxProtocolVersion(maxProtocolVersion)
2928
, mMinProtocolVersionSeen(minProtocolVersionSeen)
3029
, mLevel(level)
3130
, mBucketIter(bucket)
3231
, mEntryTypeFilter(filter)
33-
, mNewOffersOnly(newOffersOnly)
3432
, mSeenKeys(seenKeys)
3533
{
3634
auto protocolVersion = mBucketIter.getMetadata().ledgerVersion;
@@ -42,9 +40,9 @@ BucketApplicator::BucketApplicator(Application& app,
4240
protocolVersion, mMaxProtocolVersion));
4341
}
4442

45-
if (newOffersOnly && !bucket->isEmpty())
43+
// Only apply offers if BucketListDB is enabled
44+
if (mApp.getConfig().isUsingBucketListDB() && !bucket->isEmpty())
4645
{
47-
releaseAssertOrThrow(mApp.getConfig().isUsingBucketListDB());
4846
auto offsetOp = bucket->getOfferRange();
4947
if (offsetOp)
5048
{
@@ -62,8 +60,12 @@ BucketApplicator::BucketApplicator(Application& app,
6260

6361
BucketApplicator::operator bool() const
6462
{
63+
// There is more work to do (i.e. (bool) *this == true) iff:
64+
// 1. The underlying bucket iterator is not EOF and
65+
// 2. Either BucketListDB is not enabled (so we must apply all entry types)
66+
// or BucketListDB is enabled and we have offers still remaining.
6567
return static_cast<bool>(mBucketIter) &&
66-
(!mNewOffersOnly || mOffersRemaining);
68+
(!mApp.getConfig().isUsingBucketListDB() || mOffersRemaining);
6769
}
6870

6971
size_t
@@ -125,7 +127,8 @@ BucketApplicator::advance(BucketApplicator::Counters& counters)
125127
// returns the file offset at the end of the currently loaded entry.
126128
// This means we must read until pos is strictly greater than the upper
127129
// bound so that we don't skip the last offer in the range.
128-
if (mNewOffersOnly && mBucketIter.pos() > mUpperBoundOffset)
130+
auto isUsingBucketListDB = mApp.getConfig().isUsingBucketListDB();
131+
if (isUsingBucketListDB && mBucketIter.pos() > mUpperBoundOffset)
129132
{
130133
mOffersRemaining = false;
131134
break;
@@ -136,7 +139,7 @@ BucketApplicator::advance(BucketApplicator::Counters& counters)
136139

137140
if (shouldApplyEntry(mEntryTypeFilter, e))
138141
{
139-
if (mNewOffersOnly)
142+
if (isUsingBucketListDB)
140143
{
141144
if (e.type() == LIVEENTRY || e.type() == INITENTRY)
142145
{
@@ -201,7 +204,7 @@ BucketApplicator::advance(BucketApplicator::Counters& counters)
201204
}
202205
else
203206
{
204-
releaseAssertOrThrow(!mNewOffersOnly);
207+
releaseAssertOrThrow(!isUsingBucketListDB);
205208
if (protocolVersionIsBefore(
206209
mMinProtocolVersionSeen,
207210
Bucket::

‎src/bucket/BucketApplicator.h

+3-3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include "bucket/Bucket.h"
88
#include "bucket/BucketInputIterator.h"
9+
#include "ledger/LedgerHashUtils.h"
910
#include "util/Timer.h"
1011
#include "util/XDRStream.h"
1112
#include <memory>
@@ -28,8 +29,7 @@ class BucketApplicator
2829
BucketInputIterator mBucketIter;
2930
size_t mCount{0};
3031
std::function<bool(LedgerEntryType)> mEntryTypeFilter;
31-
bool const mNewOffersOnly;
32-
UnorderedSet<LedgerKey>& mSeenKeys;
32+
std::unordered_set<LedgerKey>& mSeenKeys;
3333
std::streamoff mUpperBoundOffset;
3434
bool mOffersRemaining{true};
3535

@@ -76,7 +76,7 @@ class BucketApplicator
7676
uint32_t minProtocolVersionSeen, uint32_t level,
7777
std::shared_ptr<Bucket const> bucket,
7878
std::function<bool(LedgerEntryType)> filter,
79-
bool newOffersOnly, UnorderedSet<LedgerKey>& seenKeys);
79+
std::unordered_set<LedgerKey>& seenKeys);
8080
operator bool() const;
8181
size_t advance(Counters& counters);
8282

‎src/catchup/ApplyBucketsWork.cpp

+49-47
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,16 @@ class TempLedgerVersionSetter : NonMovableOrCopyable
5252
};
5353

5454
uint32_t
55-
ApplyBucketsWork::startingLevel(bool offersOnly)
55+
ApplyBucketsWork::startingLevel()
5656
{
57-
return offersOnly ? 0 : BucketList::kNumLevels - 1;
57+
return mApp.getConfig().isUsingBucketListDB() ? 0
58+
: BucketList::kNumLevels - 1;
5859
}
5960

6061
bool
6162
ApplyBucketsWork::appliedAllLevels() const
6263
{
63-
if (mOffersOnly)
64+
if (mApp.getConfig().isUsingBucketListDB())
6465
{
6566
return mLevel == BucketList::kNumLevels - 1;
6667
}
@@ -73,7 +74,7 @@ ApplyBucketsWork::appliedAllLevels() const
7374
uint32_t
7475
ApplyBucketsWork::nextLevel() const
7576
{
76-
return mOffersOnly ? mLevel + 1 : mLevel - 1;
77+
return mApp.getConfig().isUsingBucketListDB() ? mLevel + 1 : mLevel - 1;
7778
}
7879

7980
ApplyBucketsWork::ApplyBucketsWork(
@@ -87,8 +88,7 @@ ApplyBucketsWork::ApplyBucketsWork(
8788
, mEntryTypeFilter(onlyApply)
8889
, mApplying(false)
8990
, mTotalSize(0)
90-
, mOffersOnly(app.getConfig().isUsingBucketListDB())
91-
, mLevel(startingLevel(mOffersOnly))
91+
, mLevel(startingLevel())
9292
, mMaxProtocolVersion(maxProtocolVersion)
9393
, mCounters(app.getClock().now())
9494
{
@@ -139,7 +139,7 @@ ApplyBucketsWork::doReset()
139139

140140
if (!isAborting())
141141
{
142-
if (mOffersOnly)
142+
if (mApp.getConfig().isUsingBucketListDB())
143143
{
144144
// The current size of this set is 1.6 million during BucketApply
145145
// (as of 12/20/23). There's not a great way to estimate this, so
@@ -173,7 +173,7 @@ ApplyBucketsWork::doReset()
173173
mTotalSize += bucket->getSize();
174174
}
175175

176-
if (mOffersOnly)
176+
if (mApp.getConfig().isUsingBucketListDB())
177177
{
178178
mBucketsToIndex.emplace_back(bucket);
179179
}
@@ -195,10 +195,12 @@ ApplyBucketsWork::doReset()
195195
mApp.getLedgerTxnRoot().prepareNewObjects(totalLECount);
196196
}
197197

198-
mLevel = startingLevel(mOffersOnly);
198+
mLevel = startingLevel();
199199
mApplying = false;
200200
mDelayChecked = false;
201-
mSpawnedAssumeStateWork = false;
201+
202+
mIndexBucketsWork.reset();
203+
mAssumeStateWork.reset();
202204

203205
mFirstBucket.reset();
204206
mSecondBucket.reset();
@@ -229,24 +231,25 @@ ApplyBucketsWork::startLevel()
229231
CLOG_DEBUG(History, "ApplyBuckets : starting level {}", mLevel);
230232
auto& level = getBucketLevel(mLevel);
231233
HistoryStateBucket const& i = mApplyState.currentBuckets.at(mLevel);
234+
bool isUsingBucketListDB = mApp.getConfig().isUsingBucketListDB();
232235

233-
bool applyFirst = mOffersOnly
236+
bool applyFirst = isUsingBucketListDB
234237
? (i.curr != binToHex(level.getCurr()->getHash()))
235238
: (i.snap != binToHex(level.getSnap()->getHash()));
236-
bool applySecond = mOffersOnly
239+
bool applySecond = isUsingBucketListDB
237240
? (i.snap != binToHex(level.getSnap()->getHash()))
238241
: (i.curr != binToHex(level.getCurr()->getHash()));
239242

240243
if (mApplying || applyFirst)
241244
{
242-
mFirstBucket = getBucket(mOffersOnly ? i.curr : i.snap);
245+
mFirstBucket = getBucket(isUsingBucketListDB ? i.curr : i.snap);
243246
mMinProtocolVersionSeen = std::min(
244247
mMinProtocolVersionSeen, Bucket::getBucketVersion(mFirstBucket));
245248
mFirstBucketApplicator = std::make_unique<BucketApplicator>(
246249
mApp, mMaxProtocolVersion, mMinProtocolVersionSeen, mLevel,
247-
mFirstBucket, mEntryTypeFilter, mOffersOnly, mSeenKeys);
250+
mFirstBucket, mEntryTypeFilter, mSeenKeys);
248251

249-
if (mOffersOnly)
252+
if (isUsingBucketListDB)
250253
{
251254
CLOG_DEBUG(History, "ApplyBuckets : starting level[{}].curr = {}",
252255
mLevel, i.curr);
@@ -261,14 +264,14 @@ ApplyBucketsWork::startLevel()
261264
}
262265
if (mApplying || applySecond)
263266
{
264-
mSecondBucket = getBucket(mOffersOnly ? i.snap : i.curr);
267+
mSecondBucket = getBucket(isUsingBucketListDB ? i.snap : i.curr);
265268
mMinProtocolVersionSeen = std::min(
266269
mMinProtocolVersionSeen, Bucket::getBucketVersion(mSecondBucket));
267270
mSecondBucketApplicator = std::make_unique<BucketApplicator>(
268271
mApp, mMaxProtocolVersion, mMinProtocolVersionSeen, mLevel,
269-
mSecondBucket, mEntryTypeFilter, mOffersOnly, mSeenKeys);
272+
mSecondBucket, mEntryTypeFilter, mSeenKeys);
270273

271-
if (mOffersOnly)
274+
if (isUsingBucketListDB)
272275
{
273276
CLOG_DEBUG(History, "ApplyBuckets : starting level[{}].snap = {}",
274277
mLevel, i.snap);
@@ -288,34 +291,26 @@ ApplyBucketsWork::doWork()
288291
ZoneScoped;
289292

290293
// Step 1: index buckets. Step 2: apply buckets. Step 3: assume state
291-
if (!mSpawnedIndexBucketsWork)
294+
bool isUsingBucketListDB = mApp.getConfig().isUsingBucketListDB();
295+
if (isUsingBucketListDB)
292296
{
293-
if (mOffersOnly)
294-
{
295-
addWork<IndexBucketsWork>(mBucketsToIndex);
296-
}
297-
else
297+
if (!mIndexBucketsWork)
298298
{
299-
mFinishedIndexBucketsWork = true;
299+
mIndexBucketsWork = addWork<IndexBucketsWork>(mBucketsToIndex);
300300
}
301301

302-
mSpawnedIndexBucketsWork = true;
303-
}
304-
305-
if (!mFinishedIndexBucketsWork)
306-
{
302+
// If indexing has not finished or finished and failed, return result
303+
// status. If indexing finished and succeeded, move on and spawn assume
304+
// state work.
307305
auto status = checkChildrenStatus();
308-
if (status == BasicWork::State::WORK_SUCCESS)
309-
{
310-
mFinishedIndexBucketsWork = true;
311-
}
312-
else
306+
if (!mIndexBucketsWork->isDone() ||
307+
status != BasicWork::State::WORK_SUCCESS)
313308
{
314309
return status;
315310
}
316311
}
317312

318-
if (!mSpawnedAssumeStateWork)
313+
if (!mAssumeStateWork)
319314
{
320315
if (mApp.getLedgerManager().rebuildingInMemoryState() && !mDelayChecked)
321316
{
@@ -347,30 +342,35 @@ ApplyBucketsWork::doWork()
347342
if (mFirstBucketApplicator)
348343
{
349344
TempLedgerVersionSetter tlvs(mApp, mMaxProtocolVersion);
345+
346+
// When BucketListDB is enabled, we apply in order starting with
347+
// curr. If BucketListDB is not enabled, we iterate in reverse
348+
// starting with snap.
349+
bool isCurr = isUsingBucketListDB;
350350
if (*mFirstBucketApplicator)
351351
{
352-
advance(mOffersOnly ? "curr" : "snap", *mFirstBucketApplicator);
352+
advance(isCurr ? "curr" : "snap", *mFirstBucketApplicator);
353353
return State::WORK_RUNNING;
354354
}
355355
mApp.getInvariantManager().checkOnBucketApply(
356-
mFirstBucket, mApplyState.currentLedger, mLevel,
357-
/*isCurr=*/mOffersOnly, mEntryTypeFilter);
356+
mFirstBucket, mApplyState.currentLedger, mLevel, isCurr,
357+
mEntryTypeFilter);
358358
mFirstBucketApplicator.reset();
359359
mFirstBucket.reset();
360360
mApp.getCatchupManager().bucketsApplied();
361361
}
362362
if (mSecondBucketApplicator)
363363
{
364+
bool isCurr = !isUsingBucketListDB;
364365
TempLedgerVersionSetter tlvs(mApp, mMaxProtocolVersion);
365366
if (*mSecondBucketApplicator)
366367
{
367-
advance(mOffersOnly ? "snap" : "curr",
368-
*mSecondBucketApplicator);
368+
advance(isCurr ? "curr" : "snap", *mSecondBucketApplicator);
369369
return State::WORK_RUNNING;
370370
}
371371
mApp.getInvariantManager().checkOnBucketApply(
372-
mSecondBucket, mApplyState.currentLedger, mLevel,
373-
/*isCurr=*/!mOffersOnly, mEntryTypeFilter);
372+
mSecondBucket, mApplyState.currentLedger, mLevel, isCurr,
373+
mEntryTypeFilter);
374374
mSecondBucketApplicator.reset();
375375
mSecondBucket.reset();
376376
mApp.getCatchupManager().bucketsApplied();
@@ -387,9 +387,9 @@ ApplyBucketsWork::doWork()
387387
CLOG_INFO(History, "ApplyBuckets : done, assuming state");
388388

389389
// After all buckets applied, spawn assumeState work
390-
addWork<AssumeStateWork>(mApplyState, mMaxProtocolVersion,
391-
/* restartMerges */ true);
392-
mSpawnedAssumeStateWork = true;
390+
mAssumeStateWork =
391+
addWork<AssumeStateWork>(mApplyState, mMaxProtocolVersion,
392+
/* restartMerges */ true);
393393
}
394394

395395
return checkChildrenStatus();
@@ -449,7 +449,9 @@ ApplyBucketsWork::getStatus() const
449449
{
450450
// This status string only applies to step 2 when we actually apply the
451451
// buckets.
452-
if (mFinishedIndexBucketsWork && !mSpawnedAssumeStateWork)
452+
bool doneIndexing = !mApp.getConfig().isUsingBucketListDB() ||
453+
(mIndexBucketsWork && mIndexBucketsWork->isDone());
454+
if (doneIndexing && !mSpawnedAssumeStateWork)
453455
{
454456
auto size = mTotalSize == 0 ? 0 : (100 * mAppliedSize / mTotalSize);
455457
return fmt::format(

0 commit comments

Comments
 (0)