Skip to content

Commit b83631d

Browse files
mdmintzmdmintz
authored
Merge pull request #2005 from seleniumbase/requests-get-with-timeout
Add "timeout" to "requests.get()"
2 parents 89de9e8 + 3c633b0 commit b83631d

File tree

12 files changed

+50
-48
lines changed

12 files changed

+50
-48
lines changed

‎examples/boilerplates/samples/google_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@
66
class GoogleTests(BaseCase):
77
def test_google_dot_com(self):
88
self.open("https://google.com/ncr")
9-
self.sleep(0.5)
9+
self.sleep(0.4)
1010
self.save_screenshot_to_logs() # ("./latest_logs" folder)
11-
self.sleep(0.5)
11+
self.sleep(0.2)
1212
self.type(HomePage.search_box, "github.com")
1313
self.assert_element(HomePage.search_button)
1414
self.assert_element(HomePage.feeling_lucky_button)

‎examples/boilerplates/samples/test_page_objects.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@
66
class GooglePage:
77
def go_to_google(self, sb):
88
sb.open("https://google.com/ncr")
9-
sb.sleep(0.2)
10-
sb.hide_elements('iframe[name="callout"]') # Hide "Sign in" pop-up
119
sb.sleep(0.1)
10+
sb.hide_elements('iframe') # Hide "Sign in" pop-up
11+
sb.sleep(0.2)
1212

1313
def do_search(self, sb, search_term):
1414
sb.click('[title="Search"]')

‎examples/old_wordle_script.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class WordleTests(BaseCase):
1717

1818
def initialize_word_list(self):
1919
txt_file = "https://seleniumbase.github.io/cdn/txt/wordle_words.txt"
20-
word_string = requests.get(txt_file).text
20+
word_string = requests.get(txt_file, timeout=3).text
2121
self.word_list = ast.literal_eval(word_string)
2222

2323
def modify_word_list(self, word, letter_status):

‎examples/wordle_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ class WordleTests(BaseCase):
1010

1111
def initialize_word_list(self):
1212
txt_file = "https://seleniumbase.github.io/cdn/txt/wordle_words.txt"
13-
word_string = requests.get(txt_file).text
13+
word_string = requests.get(txt_file, timeout=3).text
1414
self.word_list = ast.literal_eval(word_string)
1515

1616
def modify_word_list(self, word, letter_status):

‎requirements.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ pip>=23.2.1;python_version>="3.7"
33
packaging>=21.3;python_version<"3.7"
44
packaging>=23.1;python_version>="3.7"
55
setuptools>=59.6.0;python_version<"3.7"
6-
setuptools>=68.0.0;python_version>="3.7"
6+
setuptools>=68.0.0;python_version>="3.7" and python_version<"3.8"
7+
setuptools>=68.1.0;python_version>="3.8"
78
wheel>=0.37.1;python_version<"3.7"
89
wheel>=0.41.1;python_version>="3.7"
910
attrs==22.1.0;python_version<"3.7"

‎seleniumbase/__version__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
# seleniumbase package
2-
__version__ = "4.17.5"
2+
__version__ = "4.17.6"

‎seleniumbase/console_scripts/sb_install.py

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -90,24 +90,13 @@ def make_executable(file_path):
9090
os.chmod(file_path, mode)
9191

9292

93-
def requests_get(url):
94-
response = None
95-
try:
96-
response = requests.get(url)
97-
except Exception:
98-
# Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
99-
url = url.replace("https://", "http://")
100-
response = requests.get(url)
101-
return response
102-
103-
104-
def requests_get_with_retry(url):
93+
def get_proxy_info():
10594
use_proxy = None
10695
protocol = "http"
96+
proxy_string = None
10797
user_and_pass = None
10898
if " --proxy=" in " ".join(sys.argv):
10999
from seleniumbase.core import proxy_helper
110-
111100
for arg in sys.argv:
112101
if arg.startswith("--proxy="):
113102
proxy_string = arg.split("--proxy=")[1]
@@ -133,30 +122,41 @@ def requests_get_with_retry(url):
133122
proxy_string = "%s@%s" % (user_and_pass, proxy_string)
134123
use_proxy = True
135124
break
125+
return (use_proxy, protocol, proxy_string)
126+
127+
128+
def requests_get(url):
129+
use_proxy, protocol, proxy_string = get_proxy_info()
130+
proxies = None
136131
response = None
137132
if use_proxy:
138133
proxies = {protocol: proxy_string}
134+
try:
135+
response = requests.get(url, proxies=proxies, timeout=3)
136+
except Exception:
137+
# Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
138+
url = url.replace("https://", "http://")
139+
time.sleep(0.04)
140+
response = requests.get(url, proxies=proxies, timeout=4)
141+
return response
142+
143+
144+
def requests_get_with_retry(url):
145+
use_proxy, protocol, proxy_string = get_proxy_info()
146+
proxies = None
147+
response = None
148+
if use_proxy:
149+
proxies = {protocol: proxy_string}
150+
try:
151+
response = requests.get(url, proxies=proxies, timeout=3)
152+
except Exception:
153+
time.sleep(1)
139154
try:
140-
response = requests.get(url, proxies=proxies)
141-
except Exception:
142-
time.sleep(1.1)
143-
try:
144-
response = requests.get(url, proxies=proxies)
145-
except Exception:
146-
time.sleep(1.2)
147-
response = requests.get(url, proxies=proxies)
148-
return response
149-
else:
150-
try:
151-
response = requests.get(url)
155+
response = requests.get(url, proxies=proxies, timeout=4)
152156
except Exception:
153-
time.sleep(1.1)
154-
try:
155-
response = requests.get(url)
156-
except Exception:
157-
time.sleep(1.2)
158-
response = requests.get(url)
159-
return response
157+
time.sleep(1)
158+
response = requests.get(url, proxies=proxies, timeout=4)
159+
return response
160160

161161

162162
def main(override=None, intel_for_uc=None):

‎seleniumbase/core/browser_launcher.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,20 +111,20 @@ def requests_get(url, proxy_string=None):
111111
if proxy_string:
112112
proxies = {protocol: proxy_string}
113113
try:
114-
response = requests.get(url, proxies=proxies)
114+
response = requests.get(url, proxies=proxies, timeout=3)
115115
except Exception:
116116
# Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
117117
url = url.replace("https://", "http://")
118118
time.sleep(0.04)
119-
response = requests.get(url, proxies=proxies)
119+
response = requests.get(url, proxies=proxies, timeout=4)
120120
else:
121121
try:
122-
response = requests.get(url)
122+
response = requests.get(url, timeout=3)
123123
except Exception:
124124
# Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED
125125
url = url.replace("https://", "http://")
126126
time.sleep(0.04)
127-
response = requests.get(url)
127+
response = requests.get(url, timeout=4)
128128
return response
129129

130130

‎seleniumbase/fixtures/js_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ def add_css_style(driver, css_style):
531531
def add_js_code_from_link(driver, js_link):
532532
if js_link.startswith("//"):
533533
js_link = "http:" + js_link
534-
js_code = requests.get(js_link).text
534+
js_code = requests.get(js_link, timeout=5).text
535535
add_js_code_script = (
536536
"""var body_tag=document.getElementsByTagName('body').item(0);"""
537537
"""var script_tag=document.createElement("script");"""

‎seleniumbase/fixtures/page_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ def _download_file_to(file_url, destination_folder, new_file_name=None):
245245
file_name = new_file_name
246246
else:
247247
file_name = file_url.split("/")[-1]
248-
r = requests.get(file_url)
248+
r = requests.get(file_url, timeout=5)
249249
file_path = os.path.join(destination_folder, file_name)
250250
download_file_lock = fasteners.InterProcessLock(
251251
constants.MultiBrowser.DOWNLOAD_FILE_LOCK

0 commit comments

Comments
 (0)