WIP bucket entry counts in meta

src/bucket/Bucket.cpp

@@ -854,4 +854,28 @@ Bucket::getBucketVersion(std::shared_ptr<Bucket const> const& bucket)
     BucketInputIterator it(bucket);
     return it.getMetadata().ledgerVersion;
 }
+
+BucketEntryCounters
+Bucket::getBucketEntryCounters() const
+{
+    if (mIndex)
+    {
+        return mIndex->getBucketEntryCounters();
+    }
+    return {};
+}
+
+BucketEntryCounters&
+BucketEntryCounters::operator+=(BucketEntryCounters const& other)
+{
+    for (auto [type, count] : other.mEntryTypeCounts)
+    {
+        this->mEntryTypeCounts[type] += count;
+    }
+    for (auto [type, size] : other.mEntryTypeSizes)
+    {
+        this->mEntryTypeSizes[type] += size;
+    }
+    return *this;
+}
 }

src/bucket/Bucket.h

@@ -9,6 +9,7 @@
 #include "util/ProtocolVersion.h"
 #include "xdr/Stellar-ledger.h"
 #include <list>
+#include <map>
 #include <optional>
 #include <string>
 
@@ -179,7 +180,37 @@ class Bucket : public std::enable_shared_from_this<Bucket>,
     static uint32_t getBucketVersion(std::shared_ptr<Bucket> const& bucket);
     static uint32_t
     getBucketVersion(std::shared_ptr<Bucket const> const& bucket);
-
+    BucketEntryCounters getBucketEntryCounters() const;
     friend class BucketSnapshot;
 };
+
+enum class LedgerEntryTypeAndDurability : uint32_t
+{
+    ACCOUNT = 0,
+    TRUSTLINE = 1,
+    OFFER = 2,
+    DATA = 3,
+    CLAIMABLE_BALANCE = 4,
+    LIQUIDITY_POOL = 5,
+    TEMPORARY_CONTRACT_DATA = 6,
+    PERSISTENT_CONTRACT_DATA = 7,
+    CONTRACT_CODE = 8,
+    CONFIG_SETTING = 9,
+    TTL = 10
+};
+
+struct BucketEntryCounters
+{
+    std::map<LedgerEntryTypeAndDurability, size_t> mEntryTypeCounts;
+    std::map<LedgerEntryTypeAndDurability, size_t> mEntryTypeSizes;
+
+    BucketEntryCounters& operator+=(BucketEntryCounters const& other);
+
+    template <class Archive>
+    void
+    serialize(Archive& ar) const
+    {
+        ar(mEntryTypeCounts, mEntryTypeSizes);
+    }
+};
 }

src/bucket/BucketIndex.h

@@ -33,6 +33,7 @@ namespace stellar
  */
 
 class BucketManager;
+struct BucketEntryCounters;
 
 // BucketIndex abstract interface
 class BucketIndex : public NonMovableOrCopyable
@@ -74,7 +75,7 @@ class BucketIndex : public NonMovableOrCopyable
                                   IndividualIndex::const_iterator>;
 
     inline static const std::string DB_BACKEND_STATE = "bl";
-    inline static const uint32_t BUCKET_INDEX_VERSION = 3;
+    inline static const uint32_t BUCKET_INDEX_VERSION = 4;
 
     // Returns true if LedgerEntryType not supported by BucketListDB
     static bool typeNotSupported(LedgerEntryType t);
@@ -132,7 +133,7 @@ class BucketIndex : public NonMovableOrCopyable
 
     virtual void markBloomMiss() const = 0;
     virtual void markBloomLookup() const = 0;
-
+    virtual BucketEntryCounters const& getBucketEntryCounters() const = 0;
 #ifdef BUILD_TESTS
     virtual bool operator==(BucketIndex const& inRaw) const = 0;
 #endif

src/bucket/BucketIndexImpl.cpp

@@ -7,6 +7,7 @@
 #include "bucket/BucketManager.h"
 #include "crypto/Hex.h"
 #include "crypto/ShortHash.h"
+#include "ledger/LedgerTypeUtils.h"
 #include "main/Config.h"
 #include "util/BinaryFuseFilter.h"
 #include "util/Fs.h"
@@ -100,6 +101,79 @@ BucketIndexImpl<IndexT>::BucketIndexImpl(BucketManager& bm,
         std::vector<uint64_t> keyHashes;
         auto seed = shortHash::getShortHashInitKey();
 
+        auto countEntry = [&](BucketEntry const& be) {
+            auto ledgerEntryToLedgerEntryAndDurabilityType =
+                [&](LedgerEntryType let, bool isTemporaryEntry) {
+                    switch (let)
+                    {
+                    case ACCOUNT:
+                        return LedgerEntryTypeAndDurability::ACCOUNT;
+                        break;
+                    case TRUSTLINE:
+                        return LedgerEntryTypeAndDurability::TRUSTLINE;
+                        break;
+                    case OFFER:
+                        return LedgerEntryTypeAndDurability::OFFER;
+                        break;
+                    case DATA:
+                        return LedgerEntryTypeAndDurability::DATA;
+                        break;
+                    case CLAIMABLE_BALANCE:
+                        return LedgerEntryTypeAndDurability::CLAIMABLE_BALANCE;
+                        break;
+                    case LIQUIDITY_POOL:
+                        return LedgerEntryTypeAndDurability::LIQUIDITY_POOL;
+                        break;
+                    case CONTRACT_DATA:
+                        return isTemporaryEntry ? LedgerEntryTypeAndDurability::
+                                                      TEMPORARY_CONTRACT_DATA
+                                                : LedgerEntryTypeAndDurability::
+                                                      PERSISTENT_CONTRACT_DATA;
+                        break;
+                    case CONTRACT_CODE:
+                        return LedgerEntryTypeAndDurability::CONTRACT_CODE;
+                        break;
+                    case CONFIG_SETTING:
+                        return LedgerEntryTypeAndDurability::CONFIG_SETTING;
+                        break;
+                    case TTL:
+                        return LedgerEntryTypeAndDurability::TTL;
+                        break;
+                    default:
+                        throw std::runtime_error(fmt::format(
+                            FMT_STRING("Unknown LedgerEntryType {}"), let));
+                        break;
+                    }
+                };
+            auto bet = be.type();
+            LedgerEntryType let;
+            bool isTemp = false;
+            if (bet == INITENTRY || bet == LIVEENTRY)
+            {
+                let = be.liveEntry().data.type();
+                if (let == CONTRACT_DATA)
+                {
+                    isTemp = isTemporaryEntry(be.liveEntry().data);
+                }
+            }
+            else if (bet == DEADENTRY)
+            {
+                let = be.deadEntry().type();
+                if (let == CONTRACT_DATA)
+                {
+                    isTemp = isTemporaryEntry(be.deadEntry());
+                }
+            }
+            else
+            {
+                // Do not count meta entries.
+                return;
+            }
+            auto ledt = ledgerEntryToLedgerEntryAndDurabilityType(let, isTemp);
+            mData.counters.mEntryTypeCounts[ledt]++;
+            mData.counters.mEntryTypeSizes[ledt] += xdr::xdr_size(be);
+        };
+
         while (in && in.readOne(be))
         {
             // peridocially check if bucket manager is exiting to stop indexing
@@ -170,6 +244,7 @@ BucketIndexImpl<IndexT>::BucketIndexImpl(BucketManager& bm,
                 {
                     mData.keysToOffset.emplace_back(key, pos);
                 }
+                countEntry(be);
             }
 
             pos = in.pos();
@@ -580,4 +655,11 @@ BucketIndexImpl<BucketIndex::RangeIndex>::markBloomLookup() const
 {
     mBloomLookupMeter.Mark();
 }
-}
+
+template <class IndexT>
+BucketEntryCounters const&
+BucketIndexImpl<IndexT>::getBucketEntryCounters() const
+{
+    return mData.counters;
+}
+}

src/bucket/BucketIndexImpl.h

@@ -4,6 +4,7 @@
 // under the Apache License, Version 2.0. See the COPYING file at the root
 // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
 
+#include "bucket/Bucket.h"
 #include "bucket/BucketIndex.h"
 #include "medida/meter.h"
 #include "util/BinaryFuseFilter.h"
@@ -15,7 +16,6 @@
 
 namespace stellar
 {
-
 // Index maps either individual keys or a key range of BucketEntry's to the
 // associated offset within the bucket file. Index stored as vector of pairs:
 // First: LedgerKey/Key ranges sorted in the same scheme as LedgerEntryCmp
@@ -33,13 +33,15 @@ template <class IndexT> class BucketIndexImpl : public BucketIndex
         std::streamoff pageSize{};
         std::unique_ptr<BinaryFuseFilter16> filter{};
         std::map<Asset, std::vector<PoolID>> assetToPoolID{};
+        BucketEntryCounters counters{};
 
         template <class Archive>
         void
         save(Archive& ar) const
         {
             auto version = BUCKET_INDEX_VERSION;
-            ar(version, pageSize, assetToPoolID, keysToOffset, filter);
+            ar(version, pageSize, assetToPoolID, keysToOffset, filter,
+               counters);
         }
 
         // Note: version and pageSize must be loaded before this function is
@@ -111,6 +113,7 @@ template <class IndexT> class BucketIndexImpl : public BucketIndex
 
     virtual void markBloomMiss() const override;
     virtual void markBloomLookup() const override;
+    virtual BucketEntryCounters const& getBucketEntryCounters() const override;
 
 #ifdef BUILD_TESTS
     virtual bool operator==(BucketIndex const& inRaw) const override;

src/bucket/BucketList.cpp

@@ -4,6 +4,7 @@
 
 #include "BucketList.h"
 #include "bucket/Bucket.h"
+#include "bucket/BucketIndexImpl.h"
 #include "bucket/BucketInputIterator.h"
 #include "bucket/BucketManager.h"
 #include "bucket/BucketSnapshot.h"
@@ -631,6 +632,21 @@ BucketList::addBatch(Application& app, uint32_t currLedger,
     }
 }
 
+BucketEntryCounters
+BucketList::sumBucketEntryCounters() const
+{
+    BucketEntryCounters counters;
+    for (auto const& lev : mLevels)
+    {
+        for (auto const& b : {lev.getCurr(), lev.getSnap()})
+        {
+            auto c = b->getBucketEntryCounters();
+            counters += c;
+        }
+    }
+    return counters;
+}
+
 void
 BucketList::updateStartingEvictionIterator(EvictionIterator& iter,
                                            uint32_t firstScanLevel,

src/bucket/BucketList.h

@@ -345,6 +345,7 @@ namespace stellar
 class AbstractLedgerTxn;
 class Application;
 class Bucket;
+struct BucketEntryCounters;
 class Config;
 struct EvictionCounters;
 struct InflationWinner;
@@ -523,5 +524,6 @@ class BucketList
                   std::vector<LedgerEntry> const& initEntries,
                   std::vector<LedgerEntry> const& liveEntries,
                   std::vector<LedgerKey> const& deadEntries);
+    BucketEntryCounters sumBucketEntryCounters() const;
 };
 }

src/bucket/BucketManager.h

@@ -29,6 +29,7 @@ class BasicWork;
 class BucketList;
 class BucketSnapshotManager;
 class Config;
+class LedgerCloseMetaFrame;
 class SearchableBucketListSnapshot;
 class TmpDirManager;
 struct HistoryArchiveState;
@@ -271,7 +272,8 @@ class BucketManager : NonMovableOrCopyable
                           uint32_t currLedgerProtocol,
                           std::vector<LedgerEntry> const& initEntries,
                           std::vector<LedgerEntry> const& liveEntries,
-                          std::vector<LedgerKey> const& deadEntries) = 0;
+                          std::vector<LedgerKey> const& deadEntries,
+                          LedgerCloseMetaFrame* ledgerCloseMeta = nullptr) = 0;
 
     // Update the given LedgerHeader's bucketListHash to reflect the current
     // state of the bucket list.
@@ -382,5 +384,7 @@ class BucketManager : NonMovableOrCopyable
     scheduleVerifyReferencedBucketsWork() = 0;
 
     virtual Config const& getConfig() const = 0;
+    virtual void
+    reportBucketEntryCountMetrics(LedgerCloseMetaFrame* ledgerCloseMeta) = 0;
 };
 }