Skip to content
This repository has been archived by the owner on Feb 25, 2024. It is now read-only.

Commit

Permalink
Fix progress
Browse files Browse the repository at this point in the history
  • Loading branch information
DeltaEvo committed May 28, 2017
1 parent b94d5e0 commit b60b3ea
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 6 deletions.
2 changes: 1 addition & 1 deletion example/print.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ scraper.getProxies(500).then(stream => {

stream.on('progress', progress => {
console.log(
`Progress ${progress.percentage.toFixed(2)}% (${progress.length - progress.remaining}/${progress.length}) (Source: ${progress.source})`
`Progress ${progress.percentage.toFixed(2)}% (${progress.tested}/${progress.length}) (Source: ${progress.source})`
)
})

Expand Down
12 changes: 8 additions & 4 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ export default class ProxyScraper {
}

testProxies(timeout, proxies) {
log('Testing %d proxies with %d timeout', proxies.length, timeout)
const stream = new ReadableStream({ objectMode: true })
const proxiesCount = proxies.length
const queue = proxies.slice(0) //Clone it
let testedProxies = 0
stream._read = () => {
for (const worker of this._workers) {
let done = false
Expand Down Expand Up @@ -61,16 +63,18 @@ export default class ProxyScraper {
log('Content missmatch %o for proxy %o', e, proxy)
})
.then(() => {
testedProxies++
if(testedProxies === proxiesCount)
stream.push(null)
stream.emit('progress', {
length: proxiesCount,
remaining: queue.length,
percentage: (1 - queue.length / proxiesCount) * 100,
tested: testedProxies,
remaining: proxiesCount - testedProxies,
percentage: (testedProxies / proxiesCount) * 100,
source: proxy.source
})
if (!done) run()
})
} else {
stream.push(null)
}
}
run()
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "proxy-scraper",
"version": "2.1.4",
"version": "2.1.5",
"description": "Scrap proxis from the web",
"main": "lib/index.js",
"files": [
Expand Down

0 comments on commit b60b3ea

Please sign in to comment.