-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathCore.js
100 lines (80 loc) · 2.59 KB
/
Core.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker';
import fs from 'fs/promises'; ;
puppeteer.use(StealthPlugin());
puppeteer.use(AdblockerPlugin());
async function StealthBrowser(additional = {}, additionalArgs = []) {
const browser = await puppeteer.launch({
...additional,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-gpu',
...additionalArgs,
],
ignoreDefaultArgs: ['--enable-automation'],
devtools: true,
ignoreHTTPSErrors: true,
});
browser.StealthPage = async () => {
const page = await browser.newPage();
await page.setRequestInterception(true);
// Add plugins
await page.evaluateOnNewDocument(() => {
Object.defineProperty(navigator, 'webdriver', {
get: () => false,
});
delete window.chrome.runtime;
delete window.chrome;
});
// Save cookies to a file
page.saveCookies = async (path = './cookies.json') => {
const cookies = await page.cookies();
await fs.writeFile(path, JSON.stringify(cookies));
};
// Load cookies from a file
page.loadCookies = async (path = './cookies.json') => {
const cookies = JSON.parse(await fs.readFile(path));
await page.setCookie(...cookies);
};
// Wait for HTML to be fully rendered
page.waitTillHTMLRendered = async (timeout = 30000) => {
const checkDurationMsecs = 1000;
const maxChecks = timeout / checkDurationMsecs;
let lastHTMLSize = 0;
let checkCounts = 1;
let countStableSizeIterations = 0;
const minStableSizeIterations = 3;
while (checkCounts++ <= maxChecks) {
const html = await page.content();
const currentHTMLSize = html.length;
const bodyHTMLSize = await page.evaluate(
() => document.body.innerHTML.length
);
console.log(
'last: ',
lastHTMLSize,
' <> curr: ',
currentHTMLSize,
' body html size: ',
bodyHTMLSize
);
if (lastHTMLSize !== 0 && currentHTMLSize === lastHTMLSize) {
countStableSizeIterations++;
} else {
countStableSizeIterations = 0;
}
if (countStableSizeIterations >= minStableSizeIterations) {
console.log('Page rendered fully..');
break;
}
lastHTMLSize = currentHTMLSize;
await page.waitForTimeout(checkDurationMsecs);
}
};
return page;
};
return browser;
}
export default StealthBrowser;