Merge pull request #2007 from seleniumbase/optimize-requests-timeouts

Optimize timeouts in requests

Author: mdmintz

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.17.6"
+__version__ = "4.17.7"

seleniumbase/console_scripts/sb_install.py

@@ -132,12 +132,12 @@ def requests_get(url):
     if use_proxy:
         proxies = {protocol: proxy_string}
     try:
-        response = requests.get(url, proxies=proxies, timeout=3)
+        response = requests.get(url, proxies=proxies, timeout=1.25)
     except Exception:
         # Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
         url = url.replace("https://", "http://")
         time.sleep(0.04)
-        response = requests.get(url, proxies=proxies, timeout=4)
+        response = requests.get(url, proxies=proxies, timeout=2.75)
     return response
 
 
@@ -148,14 +148,14 @@ def requests_get_with_retry(url):
     if use_proxy:
         proxies = {protocol: proxy_string}
     try:
-        response = requests.get(url, proxies=proxies, timeout=3)
+        response = requests.get(url, proxies=proxies, timeout=1.35)
     except Exception:
         time.sleep(1)
         try:
-            response = requests.get(url, proxies=proxies, timeout=4)
+            response = requests.get(url, proxies=proxies, timeout=2.45)
         except Exception:
             time.sleep(1)
-            response = requests.get(url, proxies=proxies, timeout=4)
+            response = requests.get(url, proxies=proxies, timeout=3.55)
     return response
 
 

seleniumbase/core/browser_launcher.py

@@ -100,31 +100,23 @@ def requests_get(url, proxy_string=None):
     import requests
 
     protocol = "http"
+    proxies = None
     if proxy_string:
         if proxy_string.endswith(":443"):
             protocol = "https"
         elif "socks4" in proxy_string:
             protocol = "socks4"
         elif "socks5" in proxy_string:
             protocol = "socks5"
-    response = None
-    if proxy_string:
         proxies = {protocol: proxy_string}
-        try:
-            response = requests.get(url, proxies=proxies, timeout=3)
-        except Exception:
-            # Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
-            url = url.replace("https://", "http://")
-            time.sleep(0.04)
-            response = requests.get(url, proxies=proxies, timeout=4)
-    else:
-        try:
-            response = requests.get(url, timeout=3)
-        except Exception:
-            # Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
-            url = url.replace("https://", "http://")
-            time.sleep(0.04)
-            response = requests.get(url, timeout=4)
+    response = None
+    try:
+        response = requests.get(url, proxies=proxies, timeout=1.25)
+    except Exception:
+        # Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
+        url = url.replace("https://", "http://")
+        time.sleep(0.04)
+        response = requests.get(url, proxies=proxies, timeout=2.75)
     return response