59 lines
1.8 KiB
Python
59 lines
1.8 KiB
Python
import aiohttp
|
|
import asyncio
|
|
|
|
|
|
def _body_preview(body: str, limit: int = 256) -> str:
|
|
if not body:
|
|
return ""
|
|
preview = body[:limit].strip()
|
|
if len(body) > limit:
|
|
return f"{preview}..."
|
|
return preview
|
|
|
|
|
|
async def fetch(session, url, httpc_params: dict = {}):
|
|
async with session.get(url, **httpc_params.get("connection", {})) as response:
|
|
body = await response.text()
|
|
if response.status != 200: # pragma: no cover
|
|
raise RuntimeError(
|
|
"HTTP GET failed for "
|
|
f"{url}: status={response.status} "
|
|
f"content_type={response.headers.get('Content-Type', '')} "
|
|
f"body_preview={_body_preview(body)}"
|
|
)
|
|
return body
|
|
|
|
|
|
async def fetch_all(session, urls, httpc_params):
|
|
tasks = []
|
|
for url in urls:
|
|
task = asyncio.create_task(fetch(session, url, httpc_params))
|
|
tasks.append(task)
|
|
results = await asyncio.gather(*tasks)
|
|
return results
|
|
|
|
|
|
async def http_get(urls, httpc_params: dict = {}):
|
|
_con = aiohttp.TCPConnector(**httpc_params.get("connection", {}))
|
|
async with aiohttp.ClientSession(
|
|
connector=_con,
|
|
**httpc_params.get("session", {})
|
|
) as session:
|
|
text = await fetch_all(session, urls, httpc_params)
|
|
return text
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
httpc_params = {
|
|
"connection": {"ssl": True},
|
|
"session": {"timeout": aiohttp.ClientTimeout(total=4)},
|
|
}
|
|
urls = [
|
|
"http://127.0.0.1:8001/.well-known/openid-federation",
|
|
"http://127.0.0.1:8000/.well-known/openid-federation",
|
|
"http://asdasd.it",
|
|
"http://127.0.0.1:8001/.well-known/openid-federation",
|
|
"http://google.it",
|
|
]
|
|
asyncio.run(http_get(urls, httpc_params=httpc_params))
|