-
-
Notifications
You must be signed in to change notification settings - Fork 1k
Closed
Description
When I made multiple visits to the same url and the output returned the result, I found that if I printed each message, 6000 visits would take about 30 seconds, but printing only part of it took much longer than 30 seconds. Some sites do, some don't.
import httpx
import json
import asyncio
async def send_req(client:httpx.AsyncClient,url:str,sem:asyncio.Semaphore):
try:
async with sem:
resp=await client.get(url=url)
resp_json=json.loads(resp.text)
if resp_json['test']=='test':
print(url)
else:
print(url+' done')
except:
pass
async def run_tasks():
sem=asyncio.Semaphore(value=500)
limit=httpx.Limits(max_connections=None)
client=httpx.AsyncClient(verify=False,timeout=None,limits=limit)
tasks=[]
for i in range(6000):
url=f'https://xxxx.xxxx/xxxx?xxx={i}'
tasks.append(asyncio.ensure_future(send_req(client=client,url=url,sem=sem)))
await asyncio.gather(*tasks)
if __name__=="__main__":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
asyncio.run(run_tasks())For example, it takes about 30 seconds to print the unqualified information in send_req, but it takes much more than 30 seconds to print only the qualified information。
import httpx
import json
import asyncio
async def send_req(client:httpx.AsyncClient,url:str,sem:asyncio.Semaphore):
try:
async with sem:
resp=await client.get(url=url)
resp_json=json.loads(resp.text)
if resp_json['test']=='test':
print(url)
except:
pass
async def run_tasks():
sem=asyncio.Semaphore(value=500)
limit=httpx.Limits(max_connections=None)
client=httpx.AsyncClient(verify=False,timeout=None,limits=limit)
tasks=[]
for i in range(6000):
url=f'https://xxxx.xxxx/xxxx?xxx={i}'
tasks.append(asyncio.ensure_future(send_req(client=client,url=url,sem=sem)))
await asyncio.gather(*tasks)
if __name__=="__main__":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
asyncio.run(run_tasks()
but it takes much more than 30 seconds to print only the qualified information
Metadata
Metadata
Assignees
Labels
No labels