Merge pull request #4154 from SirTyson/archival-stats

Adds CLI tool to print BucketList archival stats

Reviewed-by: marta-lokhova

Author: latobarita

docs/software/commands.md

@@ -84,6 +84,7 @@ Command options can only by placed after command.
    See more examples in [ledger_query_examples.md](ledger_query_examples.md).
 
 * **dump-xdr <FILE-NAME>**:  Dumps the given XDR file and then exits.
+* **dump-archival-stats**:  Logs state archival statistics about the BucketList.
 * **encode-asset**: Prints a base-64 encoded asset built from  `--code <CODE>` and `--issuer <ISSUER>`. Prints the native asset if neither `--code` nor `--issuer` is given.
 * **fuzz <FILE-NAME>**: Run a single fuzz input and exit.
 * **gen-fuzz <FILE-NAME>**:  Generate a random fuzzer input file.

src/main/ApplicationUtils.cpp

@@ -4,6 +4,7 @@
 
 #include "main/ApplicationUtils.h"
 #include "bucket/Bucket.h"
+#include "bucket/BucketList.h"
 #include "bucket/BucketManager.h"
 #include "catchup/ApplyBucketsWork.h"
 #include "catchup/CatchupConfiguration.h"
@@ -555,6 +556,186 @@ mergeBucketList(Config cfg, std::string const& outputDir)
     }
 }
 
+// Per-LedgerKey metrics used for dumping archival state
+struct StateArchivalMetric
+{
+    // True if the newest version of the entry is a DEADENTRY
+    bool isDead{};
+
+    // Number of bytes that the newest version of the entry occupies in the
+    // BucketList
+    uint64_t newestBytes{};
+
+    // Number of bytes that all outdated versions of the entry occupy in the
+    // BucketList
+    uint64_t outdatedBytes{};
+};
+
+static void
+processArchivalMetrics(
+    std::shared_ptr<Bucket const> const b,
+    UnorderedMap<LedgerKey, StateArchivalMetric>& ledgerEntries,
+    UnorderedMap<LedgerKey, std::pair<StateArchivalMetric, uint32_t>>& ttls)
+{
+    for (BucketInputIterator in(b); in; ++in)
+    {
+        auto const& be = *in;
+        bool isDead = be.type() == DEADENTRY;
+        LedgerKey k = isDead ? be.deadEntry() : LedgerEntryKey(be.liveEntry());
+        bool isTTL = k.type() == TTL;
+
+        if (!isTemporaryEntry(k) && !isTTL)
+        {
+            continue;
+        }
+
+        if (isTTL)
+        {
+            auto iter = ttls.find(k);
+            if (iter == ttls.end())
+            {
+                StateArchivalMetric metric;
+                metric.isDead = isDead;
+                metric.newestBytes = xdr::xdr_size(be);
+                if (isDead)
+                {
+                    ttls.emplace(k, std::make_pair(metric, 0));
+                }
+                else
+                {
+                    ttls.emplace(
+                        k, std::make_pair(
+                               metric,
+                               be.liveEntry().data.ttl().liveUntilLedgerSeq));
+                }
+            }
+            else
+            {
+                iter->second.first.outdatedBytes += xdr::xdr_size(be);
+            }
+        }
+        else
+        {
+            auto iter = ledgerEntries.find(k);
+            if (iter == ledgerEntries.end())
+            {
+                StateArchivalMetric metric;
+                metric.isDead = isDead;
+                metric.newestBytes = xdr::xdr_size(be);
+                ledgerEntries.emplace(k, metric);
+            }
+            else
+            {
+                iter->second.outdatedBytes += xdr::xdr_size(be);
+            }
+        }
+    }
+}
+
+int
+dumpStateArchivalStatistics(Config cfg)
+{
+    ZoneScoped;
+    VirtualClock clock;
+    cfg.setNoListen();
+    Application::pointer app = Application::create(clock, cfg, false);
+    app->getLedgerManager().loadLastKnownLedger(/* restoreBucketlist */ false,
+                                                /* isLedgerStateReady */ true);
+    auto& lm = app->getLedgerManager();
+    auto& bm = app->getBucketManager();
+    HistoryArchiveState has = lm.getLastClosedLedgerHAS();
+
+    std::vector<Hash> hashes;
+    for (uint32_t i = 0; i < BucketList::kNumLevels; ++i)
+    {
+        HistoryStateBucket const& hsb = has.currentBuckets.at(i);
+        hashes.emplace_back(hexToBin256(hsb.curr));
+        hashes.emplace_back(hexToBin256(hsb.snap));
+    }
+
+    UnorderedMap<LedgerKey, StateArchivalMetric> ledgerEntries;
+
+    // key -> (metric, liveUntilLedger)
+    UnorderedMap<LedgerKey, std::pair<StateArchivalMetric, uint32_t>> ttls;
+    float blSize = 0;
+    for (auto const& hash : hashes)
+    {
+        if (isZero(hash))
+        {
+            continue;
+        }
+        auto b = bm.getBucketByHash(hash);
+        if (!b)
+        {
+            throw std::runtime_error(std::string("missing bucket: ") +
+                                     binToHex(hash));
+        }
+        processArchivalMetrics(b, ledgerEntries, ttls);
+        blSize += b->getSize();
+    }
+
+    // *BytesNewest == bytes consumed only by newest version of BucketEntry
+    // *BytesOutdated == bytes consumed only by outdated version of BucketEntry
+    // live -> liveUntilLedger >= ledgerSeq
+    // expired -> liveUntilLedger < ledgerSeq, but not yet evicted
+    uint64_t liveBytesNewest{};
+    uint64_t liveBytesOutdated{};
+    uint64_t expiredBytesNewest{};
+    uint64_t expiredBytesOutdated{};
+    uint64_t evictedBytes{}; // All evicted bytes considered "outdated"
+
+    for (auto const& [k, leMetric] : ledgerEntries)
+    {
+        auto ttlIter = ttls.find(getTTLKey(k));
+        releaseAssertOrThrow(ttlIter != ttls.end());
+        auto const& [ttlMetric, liveUntilLedger] = ttlIter->second;
+
+        auto newestBytes = ttlMetric.newestBytes + leMetric.newestBytes;
+        auto outdatedBytes = ttlMetric.outdatedBytes + leMetric.outdatedBytes;
+
+        if (ttlMetric.isDead)
+        {
+            releaseAssertOrThrow(leMetric.isDead);
+
+            // All bytes considered outdated for evicted entries
+            evictedBytes += newestBytes + outdatedBytes;
+        }
+        else
+        {
+            releaseAssertOrThrow(!leMetric.isDead);
+
+            // If entry is live
+            if (liveUntilLedger >=
+                app->getLedgerManager().getLastClosedLedgerNum())
+            {
+                liveBytesNewest += newestBytes;
+                liveBytesOutdated += outdatedBytes;
+            }
+            else
+            {
+                expiredBytesNewest += newestBytes;
+                expiredBytesOutdated += outdatedBytes;
+            }
+        }
+    }
+
+    CLOG_INFO(Bucket, "BucketList total bytes: {}", blSize);
+    CLOG_INFO(Bucket,
+              "Live Temporary Entries: Newest bytes {} ({}%), Outdated bytes "
+              "{} ({}%)",
+              liveBytesNewest, (liveBytesNewest / blSize) * 100,
+              liveBytesOutdated, (liveBytesOutdated / blSize) * 100);
+    CLOG_INFO(Bucket,
+              "Expired but not evicted Temporary: Newest bytes {} ({}%), "
+              "Outdated bytes {} ({}%)",
+              expiredBytesNewest, (expiredBytesNewest / blSize) * 100,
+              expiredBytesOutdated, (expiredBytesOutdated / blSize) * 100);
+    CLOG_INFO(Bucket, "Evicted Temporary Entries: Outdated bytes {} ({}%)",
+              evictedBytes, (evictedBytes / blSize) * 100);
+
+    return 0;
+}
+
 int
 dumpLedger(Config cfg, std::string const& outputFile,
            std::optional<std::string> filterQuery,

src/main/ApplicationUtils.h

@@ -24,6 +24,11 @@ void initializeDatabase(Config cfg);
 void httpCommand(std::string const& command, unsigned short port);
 int selfCheck(Config cfg);
 int mergeBucketList(Config cfg, std::string const& outputDir);
+
+// Logs state archival statistics, such as the number of expired entries
+// currently in the BucketList, number of bytes of evicted entries, etc.
+int dumpStateArchivalStatistics(Config cfg);
+
 int dumpLedger(Config cfg, std::string const& outputFile,
                std::optional<std::string> filterQuery,
                std::optional<uint32_t> lastModifiedLedgerCount,

src/main/CommandLine.cpp

@@ -1168,6 +1168,15 @@ runMergeBucketList(CommandLineArgs const& args)
         [&] { return mergeBucketList(configOption.getConfig(), outputDir); });
 }
 
+int
+runDumpStateArchivalStatistics(CommandLineArgs const& args)
+{
+    CommandLine::ConfigOption configOption;
+    return runWithHelp(args, {configurationParser(configOption)}, [&] {
+        return dumpStateArchivalStatistics(configOption.getConfig());
+    });
+}
+
 int
 runDumpLedger(CommandLineArgs const& args)
 {
@@ -1836,6 +1845,9 @@ handleCommandLine(int argc, char* const* argv)
          {"self-check", "performs diagnostic checks", runSelfCheck},
          {"merge-bucketlist", "writes diagnostic merged bucket list",
           runMergeBucketList},
+         {"dump-archival-stats",
+          "prints statistics about expired/evicted entries in the BucketList",
+          runDumpStateArchivalStatistics},
          {"new-db", "creates or restores the DB to the genesis ledger",
           runNewDB},
          {"new-hist", "initialize history archives", runNewHist},