Skip to content

Commit 8a2d5d5

Browse files
j-mendezj-mendez
committedFeb 25, 2025
chore(proxy): add random connect wave prevent
1 parent be49edd commit 8a2d5d5

13 files changed

+381
-36
lines changed
 

‎.github/workflows/benches.yml

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ jobs:
5252
CHROME_PATH: /out/latest/headless-shell/chrome-headless-shell-linux64/chrome-headless-shell
5353
RUST_LOG: info,error,warn
5454
TEST_NO_ARGS: false
55+
SAMPLES: 10
5556

5657
- name: Commit and Push Changes
5758
run: |

‎Cargo.lock

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎benches/basic.rs

+7-2
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,16 @@ const LOG_FILE_NAME: &str = "benchmark_logs.txt";
99
async fn main() {
1010
set_var("CHROME_INIT", "ignore"); // Ignore the auto start.
1111
set_var("HOSTNAME", "localhost");
12+
let samples = std::env::var("SAMPLES")
13+
.unwrap_or("10".into())
14+
.parse::<u32>()
15+
.unwrap_or(10);
16+
1217
headless_browser_lib::fork(Some(*headless_browser_lib::conf::DEFAULT_PORT)).await;
1318
let task = tokio::spawn(headless_browser_lib::run_main());
1419
tokio::time::sleep(std::time::Duration::from_millis(1000)).await; // Wait for the server to load.
15-
run::run(LOG_FILE_NAME).await;
16-
run_concurrent::run(LOG_FILE_NAME).await;
20+
run::run(LOG_FILE_NAME, samples).await;
21+
run_concurrent::run(LOG_FILE_NAME, samples).await;
1722
headless_browser_lib::shutdown_instances().await;
1823
task.abort();
1924
}

‎benches/basic_no_args.rs

+7-2
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,16 @@ async fn main() {
99
set_var("CHROME_INIT", "ignore"); // Ignore the auto start.
1010
set_var("TEST_NO_ARGS", "true");
1111
set_var("HOSTNAME", "localhost");
12+
let samples = std::env::var("SAMPLES")
13+
.unwrap_or("10".into())
14+
.parse::<u32>()
15+
.unwrap_or(10);
16+
1217
headless_browser_lib::fork(Some(*headless_browser_lib::conf::DEFAULT_PORT)).await;
1318
let task = tokio::spawn(headless_browser_lib::run_main());
1419
tokio::time::sleep(std::time::Duration::from_millis(1000)).await; // Wait for the server to load.
15-
run::run(LOG_FILE_NAME).await;
16-
run_concurrent::run(LOG_FILE_NAME).await;
20+
run::run(LOG_FILE_NAME, samples).await;
21+
run_concurrent::run(LOG_FILE_NAME, samples).await;
1722
headless_browser_lib::shutdown_instances().await;
1823
task.abort();
1924
}

‎benches/logs/Darwin_v10cpu_benchmark_logs.txt

+136
Large diffs are not rendered by default.

‎benches/logs/Darwin_v10cpu_benchmark_noargs_logs.txt

+128
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,131 @@ DATE: 2025-02-25 13:16:24
2121
Total Duration: 1.372718833s
2222
Average Duration: 137.257075ms
2323

24+
<http://spider.cloud> - 5 SAMPLES
25+
CHROME_PATH: headless_shell
26+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
27+
MACHINE: Darwin/v10cpu
28+
DATE: 2025-02-25 13:29:55
29+
Total Duration: 834.222292ms
30+
Average Duration: 166.822649ms
31+
32+
<http://spider.cloud> - 2 SAMPLES
33+
CHROME_PATH: headless_shell
34+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
35+
MACHINE: Darwin/v10cpu
36+
DATE: 2025-02-25 13:32:14
37+
Total Duration: 410.434625ms
38+
Average Duration: 205.164521ms
39+
40+
<http://spider.cloud> - 4 SAMPLES
41+
CHROME_PATH: headless_shell
42+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
43+
MACHINE: Darwin/v10cpu
44+
DATE: 2025-02-25 13:32:57
45+
Total Duration: 655.835292ms
46+
Average Duration: 163.909187ms
47+
48+
<http://spider.cloud> - 20 SAMPLES
49+
CHROME_PATH: headless_shell
50+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
51+
MACHINE: Darwin/v10cpu
52+
DATE: 2025-02-25 13:33:44
53+
Total Duration: 2.676222959s
54+
Average Duration: 133.7862ms
55+
56+
<http://spider.cloud> - 15 SAMPLES
57+
CHROME_PATH: headless_shell
58+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
59+
MACHINE: Darwin/v10cpu
60+
DATE: 2025-02-25 13:35:43
61+
Total Duration: 2.397807791s
62+
Average Duration: 159.828138ms
63+
64+
<http://spider.cloud> - 50 SAMPLES
65+
CHROME_PATH: headless_shell
66+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
67+
MACHINE: Darwin/v10cpu
68+
DATE: 2025-02-25 13:36:48
69+
Total Duration: 6.484034583s
70+
Average Duration: 129.662054ms
71+
72+
<http://spider.cloud> - 50 SAMPLES
73+
CHROME_PATH: headless_shell
74+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
75+
MACHINE: Darwin/v10cpu
76+
DATE: 2025-02-25 13:39:26
77+
Total Duration: 6.713125291s
78+
Average Duration: 134.24218ms
79+
80+
<http://spider.cloud> - 20 SAMPLES
81+
CHROME_PATH: headless_shell
82+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
83+
MACHINE: Darwin/v10cpu
84+
DATE: 2025-02-25 14:51:21
85+
Total Duration: 2.614171791s
86+
Average Duration: 130.690579ms
87+
88+
<http://spider.cloud> - 20 SAMPLES
89+
CHROME_PATH: headless_shell
90+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
91+
MACHINE: Darwin/v10cpu
92+
DATE: 2025-02-25 14:58:31
93+
Total Duration: 2.55098625s
94+
Average Duration: 127.531591ms
95+
96+
<http://spider.cloud> - 20 SAMPLES
97+
CHROME_PATH: headless_shell
98+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
99+
MACHINE: Darwin/v10cpu
100+
DATE: 2025-02-25 14:59:52
101+
Total Duration: 2.762563667s
102+
Average Duration: 138.107618ms
103+
104+
<http://spider.cloud> - 20 SAMPLES
105+
CHROME_PATH: headless_shell
106+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
107+
MACHINE: Darwin/v10cpu
108+
DATE: 2025-02-25 15:02:15
109+
Total Duration: 2.666630334s
110+
Average Duration: 133.312329ms
111+
112+
<http://spider.cloud> - 50 SAMPLES
113+
CHROME_PATH: headless_shell
114+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
115+
MACHINE: Darwin/v10cpu
116+
DATE: 2025-02-25 15:03:19
117+
Total Duration: 6.518802417s
118+
Average Duration: 130.358351ms
119+
120+
<http://spider.cloud> - 50 SAMPLES
121+
CHROME_PATH: headless_shell
122+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
123+
MACHINE: Darwin/v10cpu
124+
DATE: 2025-02-25 15:04:39
125+
Total Duration: 6.622225458s
126+
Average Duration: 132.424491ms
127+
128+
<http://spider.cloud> - 25 SAMPLES
129+
CHROME_PATH: headless_shell
130+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
131+
MACHINE: Darwin/v10cpu
132+
DATE: 2025-02-25 15:06:23
133+
Total Duration: 3.417184042s
134+
Average Duration: 136.668676ms
135+
136+
<http://spider.cloud> - 15 SAMPLES
137+
CHROME_PATH: headless_shell
138+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
139+
MACHINE: Darwin/v10cpu
140+
DATE: 2025-02-25 15:07:04
141+
Total Duration: 2.123479708s
142+
Average Duration: 141.54543ms
143+
144+
<http://spider.cloud> - 6 SAMPLES
145+
CHROME_PATH: headless_shell
146+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
147+
MACHINE: Darwin/v10cpu
148+
DATE: 2025-02-25 15:07:24
149+
Total Duration: 865.587958ms
150+
Average Duration: 144.236333ms
151+

‎benches/logs_concurrent/Darwin_v10cpu_benchmark_logs.txt

+34-3
Large diffs are not rendered by default.

‎benches/logs_concurrent/Darwin_v10cpu_benchmark_noargs_logs.txt

+34-3
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,38 @@
22
CHROME_PATH: headless_shell
33
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
44
MACHINE: Darwin/v10cpu
5-
DATE: 2025-02-25 13:16:24
6-
Total Duration: 609.04825ms
7-
Average Duration: 394.105729ms
5+
DATE: 2025-02-25 09:49:33
6+
Average Duration: 170.837966ms
7+
8+
<http://spider.cloud> - 50 SAMPLES
9+
CHROME_PATH: headless_shell
10+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
11+
MACHINE: Darwin/v10cpu
12+
DATE: 2025-02-25 15:04:43
13+
Total Duration: 4.126866167s
14+
Average Duration: 3.754067105s
15+
16+
<http://spider.cloud> - 25 SAMPLES
17+
CHROME_PATH: headless_shell
18+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
19+
MACHINE: Darwin/v10cpu
20+
DATE: 2025-02-25 15:06:25
21+
Total Duration: 1.688262667s
22+
Average Duration: 1.151223173s
23+
24+
<http://spider.cloud> - 15 SAMPLES
25+
CHROME_PATH: headless_shell
26+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
27+
MACHINE: Darwin/v10cpu
28+
DATE: 2025-02-25 15:07:05
29+
Total Duration: 839.65475ms
30+
Average Duration: 532.685644ms
31+
32+
<http://spider.cloud> - 6 SAMPLES
33+
CHROME_PATH: headless_shell
34+
CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
35+
MACHINE: Darwin/v10cpu
36+
DATE: 2025-02-25 15:07:24
37+
Total Duration: 513.7985ms
38+
Average Duration: 401.752354ms
839

‎benches/run.rs

+8-8
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,16 @@ use std::{
1010
};
1111

1212
const LOG_DIR: &str = "logs";
13-
const SAMPLE_COUNT: u32 = 10;
1413

1514
/// Run the benchmarks to the env BENCH_URL.
16-
pub async fn run(log_file_name: &str) {
15+
pub async fn run(log_file_name: &str, samples: u32) {
1716
ensure_log_directory_exists(LOG_DIR).expect("Failed to create log directory");
1817
let query = env::var("BENCH_URL").unwrap_or_else(|_| "http://spider.cloud".into());
1918
let mut total_duration = Duration::new(0, 0);
2019
let current_time = Instant::now();
2120

22-
for i in 0..SAMPLE_COUNT {
23-
println!("Running sample {} of {}", i + 1, SAMPLE_COUNT);
21+
for i in 0..samples {
22+
println!("Running sample {} of {}", i + 1, samples);
2423

2524
let start_time = Instant::now();
2625
let result = navigate_extract_and_close(&query).await;
@@ -35,15 +34,15 @@ pub async fn run(log_file_name: &str) {
3534
total_duration += duration;
3635
}
3736

38-
let average_duration = total_duration.div(SAMPLE_COUNT);
37+
let average_duration = total_duration.div(samples);
3938
let total_time = current_time.elapsed();
4039

4140
println!(
4241
"Finished average time: {:?} - total time: {:?}",
43-
average_duration,total_time
42+
average_duration, total_time
4443
);
4544

46-
log_performance(total_time, average_duration, &query, log_file_name)
45+
log_performance(total_time, average_duration, &query, log_file_name, samples)
4746
.expect("Failed to log performance");
4847
}
4948

@@ -61,6 +60,7 @@ fn log_performance(
6160
current_avg: Duration,
6261
query: &str,
6362
log_file_name: &str,
63+
samples: u32,
6464
) -> io::Result<()> {
6565
let os_type = sys_info::os_type().unwrap_or_default();
6666
let cpu_count = sys_info::cpu_num().unwrap_or_default().to_string();
@@ -115,7 +115,7 @@ fn log_performance(
115115

116116
writeln!(
117117
log_file,
118-
"<{query}> - {SAMPLE_COUNT} SAMPLES\nCHROME_PATH: {}\nCHROME_ARGS: {}\nMACHINE: {}\nDATE: {}\nTotal Duration: {:?}\nAverage Duration: {:?}\n",
118+
"<{query}> - {samples} SAMPLES\nCHROME_PATH: {}\nCHROME_ARGS: {}\nMACHINE: {}\nDATE: {}\nTotal Duration: {:?}\nAverage Duration: {:?}\n",
119119
chrome_path,
120120
chrome_args,
121121
format!("{}/v{}cpu", os_type, cpu_count),

‎benches/run_concurrent.rs

+10-10
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,9 @@ use std::{
1212
use tokio::task::JoinSet;
1313

1414
const LOG_DIR: &str = "logs_concurrent";
15-
const SAMPLE_COUNT: u32 = 10;
1615

1716
/// Run the benchmarks concurrently to the env BENCH_URL.
18-
pub async fn run(log_file_name: &str) {
17+
pub async fn run(log_file_name: &str, samples: u32) {
1918
ensure_log_directory_exists(LOG_DIR).expect("Failed to create log directory");
2019
let query = env::var("BENCH_URL").unwrap_or_else(|_| "http://spider.cloud".into());
2120
let q1 = query.clone();
@@ -24,18 +23,18 @@ pub async fn run(log_file_name: &str) {
2423
let query = Arc::new(query);
2524
let current_time = Instant::now();
2625

27-
for i in 0..SAMPLE_COUNT {
28-
println!("Running sample {} of {}", i + 1, SAMPLE_COUNT);
26+
for i in 0..samples {
27+
println!("Running sample {} of {}", i + 1, samples);
2928
let query = query.clone();
3029

3130
set.spawn(async move {
3231
let start_time = Instant::now();
3332
let result = navigate_extract_and_close(&query).await;
3433
let duration = start_time.elapsed();
3534
if let Err(e) = result {
36-
eprintln!("Error running test {}: {:?}", i + 1, e);
35+
eprintln!("Error running concurrent test {}: {:?}", i + 1, e);
3736
} else {
38-
println!("Sample {} took: {:?}", i + 1, duration);
37+
println!("Sample concurrent {} took: {:?}", i + 1, duration);
3938
}
4039
duration
4140
});
@@ -45,15 +44,15 @@ pub async fn run(log_file_name: &str) {
4544
total_duration += res.unwrap_or_default();
4645
}
4746

48-
let average_duration = total_duration.div(SAMPLE_COUNT);
47+
let average_duration = total_duration.div(samples);
4948
let total_time = current_time.elapsed();
5049

5150
println!(
52-
"Finished average time: {:?} - total time: {:?}",
51+
"Finished concurrent average time: {:?} - total time: {:?}",
5352
average_duration, total_time
5453
);
5554

56-
log_performance(total_time, average_duration, &q1, log_file_name)
55+
log_performance(total_time, average_duration, &q1, log_file_name, samples)
5756
.expect("Failed to log performance");
5857
}
5958

@@ -71,6 +70,7 @@ fn log_performance(
7170
current_avg: Duration,
7271
query: &str,
7372
log_file_name: &str,
73+
samples: u32,
7474
) -> io::Result<()> {
7575
let os_type = sys_info::os_type().unwrap_or_default();
7676
let cpu_count = sys_info::cpu_num().unwrap_or_default().to_string();
@@ -125,7 +125,7 @@ fn log_performance(
125125

126126
writeln!(
127127
log_file,
128-
"<{query}> - {SAMPLE_COUNT} SAMPLES\nCHROME_PATH: {}\nCHROME_ARGS: {}\nMACHINE: {}\nDATE: {}\nTotal Duration: {:?}\nAverage Duration: {:?}\n",
128+
"<{query}> - {samples} SAMPLES\nCHROME_PATH: {}\nCHROME_ARGS: {}\nMACHINE: {}\nDATE: {}\nTotal Duration: {:?}\nAverage Duration: {:?}\n",
129129
chrome_path,
130130
chrome_args,
131131
format!("{}/v{}cpu", os_type, cpu_count),

‎headless_browser_lib/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ openssl = { version = "0.10", features = ["vendored"] }
2222
openssl-sys = { version = "0.9", features = ["vendored"] }
2323
cached = { version = "0", features = ["async_tokio_rt_multi_thread"]}
2424
tracing = "0.1"
25+
rand = "0.9"
2526

2627
[features]
2728
testing = []

‎headless_browser_lib/src/conf.rs

+7-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use std::collections::HashSet;
22
use std::sync::atomic::{AtomicBool, AtomicU64};
33

44
/// The performance arg count.
5-
pub(crate) const PERF_ARGS: usize = 92;
5+
pub(crate) const PERF_ARGS: usize = 98;
66

77
#[cfg(any(test, feature = "testing"))]
88
lazy_static::lazy_static! {
@@ -192,6 +192,7 @@ lazy_static::lazy_static! {
192192
"--use-angle=swiftshader",
193193
"--disable-low-res-tiling",
194194
"--disable-speech-api",
195+
"--disable-oobe-chromevox-hint-timer-for-testing",
195196
"--disable-smooth-scrolling",
196197
"--disable-default-apps",
197198
"--disable-prompt-on-repost",
@@ -227,6 +228,7 @@ lazy_static::lazy_static! {
227228
"--no-subproc-heap-profiling",
228229
"--no-pre-read-main-dll",
229230
"--disable-stack-profiler",
231+
"--disable-libassistant-logfile",
230232
"--crash-on-hang-threads",
231233
"--restore-last-session",
232234
"--ip-protection-proxy-opt-out",
@@ -248,6 +250,10 @@ lazy_static::lazy_static! {
248250
"--disable-vulkan-fallback-to-gl-for-testing",
249251
"--disable-vulkan-surface",
250252
"--disable-webrtc",
253+
"--disable-oopr-debug-crash-dump",
254+
"--disable-pnacl-crash-throttling",
255+
"--disable-renderer-accessibility",
256+
"--renderer-process-limit=0",
251257
"--disable-blink-features=AutomationControlled",
252258
"--disable-ipc-flooding-protection", // we do not need to throttle navigation for https://github.com/spider-rs/spider/commit/9ff5bbd7a2656b8edb84b62843b72ae9d09af079#diff-75ce697faf0d37c3dff4a3a19e7524798b3cb5487f8f54beb5d04c4d48e34234R446.
253259
// --deterministic-mode 10-20% drop in perf

‎headless_browser_lib/src/lib.rs

+7-7
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ use conf::{
1212
DEFAULT_PORT, DEFAULT_PORT_SERVER, ENDPOINT, HOST_NAME, IS_HEALTHY, LAST_CACHE,
1313
LIGHTPANDA_ARGS, LIGHT_PANDA, TARGET_REPLACEMENT,
1414
};
15-
1615
use core::sync::atomic::Ordering;
1716
use http_body_util::Full;
1817
use hyper::{
@@ -63,7 +62,7 @@ async fn connect_with_retries(address: &str) -> Option<TcpStream> {
6362
connection_failed = true;
6463
}
6564
// empty prevent connections retrying
66-
if attempts >= 8 && CHROME_INSTANCES.lock().await.is_empty() {
65+
if attempts >= 10 && CHROME_INSTANCES.lock().await.is_empty() {
6766
tracing::warn!("ConnectionRefused: {}. Attempt {} of 8", e, attempts);
6867
return None;
6968
}
@@ -79,12 +78,13 @@ async fn connect_with_retries(address: &str) -> Option<TcpStream> {
7978
return None;
8079
}
8180

82-
sleep(Duration::from_millis(if connection_failed {
83-
150
81+
let rng = rand::random_range(if connection_failed {
82+
80..=150
8483
} else {
85-
250
86-
}))
87-
.await;
84+
150..=250
85+
});
86+
87+
sleep(Duration::from_millis(rng)).await;
8888
}
8989
}
9090

0 commit comments

Comments
 (0)
Please sign in to comment.