wip

Author: SirTyson

src/bucket/BucketManager.h

@@ -92,13 +92,16 @@ struct BucketListEvictionCounters
 // Per-LedgerKey metrics used for dumping archival state
 struct StateArchivalMetric
 {
-    // Newest version of LedgerEntry in the BucketList, or nullopt if newest
-    // version is a DEADENTRY
-    std::optional<LedgerEntry> le{};
+    // True if the newest version of the entry is a DEADENTRY
+    bool isDead{};
 
-    // Number of bytes that the given LedgerKey occupies in the BucketList,
-    // including shadowed entries
-    uint64_t bytes{};
+    // Number of bytes that the newest version of the entry occupies in the
+    // BucketList
+    uint64_t newestBytes{};
+
+    // Number of bytes that all outdated versions of the entry occupy in the
+    // BucketList
+    uint64_t outdatedBytes{};
 };
 
 /**

src/bucket/BucketManagerImpl.cpp

@@ -1163,7 +1163,7 @@ loadEntriesFromBucket(std::shared_ptr<Bucket> b, std::string const& name,
 }
 
 static std::vector<std::pair<Hash, std::string>>
-getBucketHashes(HistoryArchiveState const& has)
+getBucketHashes(HistoryArchiveState const& has, bool inOrder)
 {
     std::vector<std::pair<Hash, std::string>> hashes;
     for (uint32_t i = BucketList::kNumLevels; i > 0; --i)
@@ -1228,8 +1228,9 @@ static void
 processArchivalMetrics(
     std::shared_ptr<Bucket const> const b,
     UnorderedMap<LedgerKey, StateArchivalMetric>& ledgerEntries,
-    UnorderedMap<LedgerKey, StateArchivalMetric>& ttls)
+    UnorderedMap<LedgerKey, std::pair<StateArchivalMetric, uint32_t>>& ttls)
 {
+    CLOG_FATAL(Bucket, "Size {}", b->getSize());
     for (BucketInputIterator in(b); in; ++in)
     {
         auto const& be = *in;
@@ -1242,29 +1243,47 @@ processArchivalMetrics(
             continue;
         }
 
-        auto proccessEntry = [&](auto& map) {
-            auto iter = map.find(k);
-            if (iter == map.end())
+        if (isTTL)
+        {
+            auto iter = ttls.find(k);
+            if (iter == ttls.end())
             {
+                releaseAssert(!isDead);
                 StateArchivalMetric metric;
-                metric.le =
-                    isDead ? std::nullopt : std::make_optional(be.liveEntry());
-                metric.bytes = xdr::xdr_size(be);
-                map.emplace(k, metric);
+                metric.isDead = isDead;
+                metric.newestBytes = xdr::xdr_size(be);
+                if (isDead)
+                {
+                    releaseAssert(false);
+                    ttls.emplace(k, std::make_pair(metric, 0));
+                }
+                else
+                {
+                    ttls.emplace(
+                        k, std::make_pair(
+                               metric,
+                               be.liveEntry().data.ttl().liveUntilLedgerSeq));
+                }
             }
             else
             {
-                iter->second.bytes += xdr::xdr_size(be);
+                iter->second.first.outdatedBytes += xdr::xdr_size(be);
             }
-        };
-
-        if (isTTL)
-        {
-            proccessEntry(ttls);
         }
         else
         {
-            proccessEntry(ledgerEntries);
+            auto iter = ledgerEntries.find(k);
+            if (iter == ledgerEntries.end())
+            {
+                StateArchivalMetric metric;
+                metric.isDead = isDead;
+                metric.newestBytes = xdr::xdr_size(be);
+                ledgerEntries.emplace(k, metric);
+            }
+            else
+            {
+                iter->second.outdatedBytes += xdr::xdr_size(be);
+            }
         }
     }
 }
@@ -1275,7 +1294,9 @@ BucketManagerImpl::dumpStateArchivalStatistics(HistoryArchiveState const& has)
     ZoneScoped;
     auto hashes = getBucketHashes(has);
     UnorderedMap<LedgerKey, StateArchivalMetric> ledgerEntries;
-    UnorderedMap<LedgerKey, StateArchivalMetric> ttls;
+
+    // key -> (metric, liveUntilLedger)
+    UnorderedMap<LedgerKey, std::pair<StateArchivalMetric, uint32_t>> ttls;
     float blSize = 0;
     for (auto const& pair : hashes)
     {
@@ -1293,65 +1314,63 @@ BucketManagerImpl::dumpStateArchivalStatistics(HistoryArchiveState const& has)
         blSize += b->getSize();
     }
 
-    // *BytesNoShadow == bytes consumed only by newest version of BucketEntry
-    // *BytesShadows == bytes consumed only by shadows, does not count newest
-    //                  version of BucketEntry
-    // live -> liveUntilLedger > ledgerSeq
+    // *BytesNewest == bytes consumed only by newest version of BucketEntry
+    // *BytesOutdated == bytes consumed only by outdated version of BucketEntry
+    // live -> liveUntilLedger >= ledgerSeq
     // expired -> liveUntilLedger < ledgerSeq, but not yet evicted
-    uint64_t liveBytesNoShadow{};
-    uint64_t liveBytesShadows{};
-    uint64_t expiredBytesNoShadow{};
-    uint64_t expiredBytesShadows{};
-    uint64_t evictedBytes{}; // All evicted bytes considered shadows
+    uint64_t liveBytesNewest{};
+    uint64_t liveBytesOutdated{};
+    uint64_t expiredBytesNewest{};
+    uint64_t expiredBytesOutdated{};
+    uint64_t evictedBytes{}; // All evicted bytes considered "outdated"
 
     for (auto const& [k, leMetric] : ledgerEntries)
     {
         auto ttlIter = ttls.find(getTTLKey(k));
         releaseAssertOrThrow(ttlIter != ttls.end());
-        auto const& ttlMetric = ttlIter->second;
+        auto const& [ttlMetric, liveUntilLedger] = ttlIter->second;
 
-        if (ttlMetric.le.has_value())
+        auto newestBytes = ttlMetric.newestBytes + leMetric.newestBytes;
+        auto outdatedBytes = ttlMetric.outdatedBytes + leMetric.outdatedBytes;
+
+        if (ttlMetric.isDead)
         {
-            releaseAssertOrThrow(leMetric.le.has_value());
-            auto ttlSize = xdr::xdr_size(*ttlMetric.le);
-            auto leSize = xdr::xdr_size(*leMetric.le);
-            if (isLive(*ttlMetric.le,
-                       mApp.getLedgerManager().getLastClosedLedgerNum()))
-            {
-                liveBytesNoShadow += ttlSize + leSize;
+            releaseAssertOrThrow(leMetric.isDead);
 
-                // Don't count most recent version in shadow bytes
-                liveBytesShadows +=
-                    (ttlMetric.bytes - ttlSize) + (leMetric.bytes - leSize);
+            // All bytes considred outdated for evicted entries
+            evictedBytes += newestBytes + outdatedBytes;
+        }
+        else
+        {
+            releaseAssertOrThrow(!leMetric.isDead);
+
+            // If entry is live
+            if (liveUntilLedger >=
+                mApp.getLedgerManager().getLastClosedLedgerNum())
+            {
+                liveBytesNewest += newestBytes;
+                liveBytesOutdated += outdatedBytes;
             }
             else
             {
-                expiredBytesNoShadow += ttlSize + leSize;
-
-                // Don't count most recent version in shadow bytes
-                expiredBytesShadows +=
-                    (ttlMetric.bytes - ttlSize) + (leMetric.bytes - leSize);
+                expiredBytesNewest += newestBytes;
+                expiredBytesOutdated += outdatedBytes;
             }
         }
-        else
-        {
-            releaseAssertOrThrow(!leMetric.le.has_value());
-            evictedBytes += leMetric.bytes + ttlMetric.bytes;
-        }
     }
 
     CLOG_INFO(Bucket, "BucketList total bytes: {}", blSize);
     CLOG_INFO(Bucket,
-              "Live Temporary Entries: Non-shadows {} bytes ({}%), Shadowed {} "
+              "Live Temporary Entries: Newest {} bytes ({}%), Outdated {} "
               "bytes ({}%)",
-              liveBytesNoShadow, (liveBytesNoShadow / blSize) * 100,
-              liveBytesShadows, (liveBytesShadows / blSize) * 100);
+              liveBytesNewest, (liveBytesNewest / blSize) * 100,
+              liveBytesOutdated, (liveBytesOutdated / blSize) * 100);
     CLOG_INFO(Bucket,
-              "Expired but not evicted Temporary Entries: Non-shadows {} bytes "
-              "({}%), Shadowed {} bytes ({}%)",
-              expiredBytesNoShadow, (expiredBytesNoShadow / blSize) * 100,
-              expiredBytesShadows, (expiredBytesShadows / blSize) * 100);
-    CLOG_INFO(Bucket, "Evicted Temporary Entries: Shadowed {} bytes ({}%)",
+              "Expired but not evicted Temporary Newest {} bytes "
+              "({}%), Outdated {} bytes ({}%)",
+              expiredBytesNewest, (expiredBytesNewest / blSize) * 100,
+              expiredBytesOutdated, (expiredBytesOutdated / blSize) * 100);
+    CLOG_INFO(Bucket, "Evicted Temporary Entries: Outdated {} bytes ({}%)",
               evictedBytes, (evictedBytes / blSize) * 100);
 }
 

src/main/ApplicationUtils.cpp

@@ -561,7 +561,7 @@ dumpStateArchivalStatistics(Config cfg)
     VirtualClock clock;
     cfg.setNoListen();
     Application::pointer app = Application::create(clock, cfg, false);
-    app->getLedgerManager().loadLastKnownLedger(nullptr);
+    app->getLedgerManager().loadLastKnownLedger(false, false);
     auto& lm = app->getLedgerManager();
     auto& bm = app->getBucketManager();
     HistoryArchiveState has = lm.getLastClosedLedgerHAS();