add exponential backoff, and random offset (#84)

* add exponential backoff, and random offset

* jitter is multiplicative, backoff actually exponential

Author: RokasEl

paraffin/stage.py

@@ -3,6 +3,7 @@
 import dataclasses
 import json
 import logging
+import random
 import subprocess
 import threading
 import time
@@ -44,7 +45,7 @@ def cmd(self) -> str:
         return self.stage.cmd
 
 
-def retry(times, exceptions, delay: float = 0):
+def retry(times, exceptions, delay: float = 0, exponential: bool = True):
     """
     Retry Decorator
     Retries the wrapped function/method `times` times if the exceptions listed
@@ -64,7 +65,9 @@ def newfn(*args, **kwargs):
                 except exceptions as e:
                     attempt += 1
                     log.warning(f"Caught exception {e} - retrying {attempt}/{times}")
-                    time.sleep(delay)
+                    sleep_time = delay * (2.0**attempt) if exponential else delay
+                    sleep_time *= random.uniform(0, 1.0)
+                    time.sleep(sleep_time)
             return func(*args, **kwargs)
 
         return newfn