Add new throttling metrics

Author: marta-lokhova

src/overlay/FlowControl.cpp

@@ -111,6 +111,13 @@ FlowControl::maybeReleaseCapacityAndTriggerSend(StellarMessage const& msg)
 
     if (msg.type() == SEND_MORE || msg.type() == SEND_MORE_EXTENDED)
     {
+        if (mNoOutboundCapacity)
+        {
+            mApp.getOverlayManager()
+                .getOverlayMetrics()
+                .mConnectionFloodThrottle.Update(mApp.getClock().now() -
+                                                 *mNoOutboundCapacity);
+        }
         mNoOutboundCapacity.reset();
 
         mFlowControlCapacity->releaseOutboundCapacity(msg);

src/overlay/OverlayMetrics.cpp

@@ -32,6 +32,10 @@ OverlayMetrics::OverlayMetrics(Application& app)
           {"overlay", "timeout", "straggler"}, "timeout"))
     , mConnectionLatencyTimer(
           app.getMetrics().NewTimer({"overlay", "connection", "latency"}))
+    , mConnectionReadThrottle(
+          app.getMetrics().NewTimer({"overlay", "connection", "read-throttle"}))
+    , mConnectionFloodThrottle(app.getMetrics().NewTimer(
+          {"overlay", "connection", "flood-throttle"}))
 
     , mItemFetcherNextPeer(app.getMetrics().NewMeter(
           {"overlay", "item-fetcher", "next-peer"}, "item-fetcher"))

src/overlay/OverlayMetrics.h

@@ -34,6 +34,8 @@ struct OverlayMetrics
     medida::Meter& mTimeoutIdle;
     medida::Meter& mTimeoutStraggler;
     medida::Timer& mConnectionLatencyTimer;
+    medida::Timer& mConnectionReadThrottle;
+    medida::Timer& mConnectionFloodThrottle;
 
     medida::Meter& mItemFetcherNextPeer;
 

src/overlay/Peer.cpp

@@ -170,11 +170,13 @@ Peer::endMessageProcessing(StellarMessage const& msg)
     mFlowControl->endMessageProcessing(msg, shared_from_this());
 
     releaseAssert(canRead());
-    if (mIsPeerThrottled)
+    if (mLastThrottle)
     {
         CLOG_DEBUG(Overlay, "Stop throttling reading from peer {}",
                    mApp.getConfig().toShortString(getPeerID()));
-        mIsPeerThrottled = false;
+        getOverlayMetrics().mConnectionReadThrottle.Update(
+            mApp.getClock().now() - *mLastThrottle);
+        mLastThrottle.reset();
         scheduleRead();
     }
 }

src/overlay/Peer.h

@@ -201,7 +201,7 @@ class Peer : public std::enable_shared_from_this<Peer>,
     };
 
     // Is this peer currently throttled due to lack of capacity
-    bool mIsPeerThrottled{false};
+    std::optional<VirtualClock::time_point> mLastThrottle;
 
     // Does local node have capacity to read from this peer
     bool canRead() const;

src/overlay/TCPPeer.cpp

@@ -461,7 +461,7 @@ TCPPeer::scheduleRead()
     // this will be throttled to try to balance input rates across peers.
     ZoneScoped;
 
-    if (mIsPeerThrottled)
+    if (mLastThrottle)
     {
         return;
     }
@@ -543,7 +543,7 @@ TCPPeer::startRead()
                     // Break and wait until more capacity frees up
                     CLOG_DEBUG(Overlay, "Throttle reading from peer {}!",
                                mApp.getConfig().toShortString(getPeerID()));
-                    mIsPeerThrottled = true;
+                    mLastThrottle = mApp.getClock().now();
                     return;
                 }
                 if (mApp.getClock().shouldYield())
@@ -689,7 +689,7 @@ TCPPeer::readBodyHandler(asio::error_code const& error,
             CLOG_DEBUG(Overlay,
                        "TCPPeer::readBodyHandler: throttle reading from {}",
                        mApp.getConfig().toShortString(getPeerID()));
-            mIsPeerThrottled = true;
+            mLastThrottle = mApp.getClock().now();
             return;
         }
 

src/overlay/test/LoopbackPeer.cpp

@@ -231,7 +231,7 @@ LoopbackPeer::processInQueue()
 {
     if (!canRead())
     {
-        mIsPeerThrottled = true;
+        mLastThrottle = mApp.getClock().now();
         return;
     }