Use exponential backoff for connection retries

Calls to socket.connect() are non-blocking, hence all subsequent calls
to socket.sendall() will fail if the target KDC service is temporarily
or indefinitely unreachable. Since the kdcproxy task uses busy-looping,
it results in the journal to be flooded with warning logs.

This commit introduces a per-socket reactivation delay which increases
exponentially as the number of reties is incremented, until timeout is
reached (i.e. 100ms, 200ms, 400ms, 800ms, 1.6s, 3.2s, ...).

Signed-off-by: Julien Rische <[email protected]>

Author: jrisc

kdcproxy/__init__.py

@@ -74,6 +74,7 @@ def __init__(self):
     def __await_reply(self, pr, rsocks, wsocks, timeout):
         extra = 0
         read_buffers = {}
+        reactivations = {}
         while (timeout + extra) > time.time():
             if not wsocks and not rsocks:
                 break
@@ -92,6 +93,9 @@ def __await_reply(self, pr, rsocks, wsocks, timeout):
                     pass
 
             for sock in w:
+                (react_n, react_t) = reactivations.get(sock, (-1, 0.0))
+                if react_t > time.time():
+                    continue
                 try:
                     if self.sock_type(sock) == socket.SOCK_DGRAM:
                         # If we proxy over UDP, remove the 4-byte length
@@ -101,8 +105,12 @@ def __await_reply(self, pr, rsocks, wsocks, timeout):
                         sock.sendall(pr.request)
                         extra = 10  # New connections get 10 extra seconds
                 except Exception as e:
-                    logging.warning("Conection broken while writing (%s)", e)
+                    logging.warning("Connection broken while writing (%s)", e)
+                    reactivations[sock] = (react_n + 1,
+                                           time.time() + 2.0**(react_n + 1) / 10)
                     continue
+                if sock in reactivations:
+                    del reactivations[sock]
                 rsocks.append(sock)
                 wsocks.remove(sock)