import aiohttp import asyncio def _body_preview(body: str, limit: int = 256) -> str: if not body: return "" preview = body[:limit].strip() if len(body) > limit: return f"{preview}..." return preview async def fetch(session, url, httpc_params: dict = {}): async with session.get(url, **httpc_params.get("connection", {})) as response: body = await response.text() if response.status != 200: # pragma: no cover raise RuntimeError( "HTTP GET failed for " f"{url}: status={response.status} " f"content_type={response.headers.get('Content-Type', '')} " f"body_preview={_body_preview(body)}" ) return body async def fetch_all(session, urls, httpc_params): tasks = [] for url in urls: task = asyncio.create_task(fetch(session, url, httpc_params)) tasks.append(task) results = await asyncio.gather(*tasks) return results async def http_get(urls, httpc_params: dict = {}): _con = aiohttp.TCPConnector(**httpc_params.get("connection", {})) async with aiohttp.ClientSession( connector=_con, **httpc_params.get("session", {}) ) as session: text = await fetch_all(session, urls, httpc_params) return text if __name__ == "__main__": # pragma: no cover httpc_params = { "connection": {"ssl": True}, "session": {"timeout": aiohttp.ClientTimeout(total=4)}, } urls = [ "http://127.0.0.1:8001/.well-known/openid-federation", "http://127.0.0.1:8000/.well-known/openid-federation", "http://asdasd.it", "http://127.0.0.1:8001/.well-known/openid-federation", "http://google.it", ] asyncio.run(http_get(urls, httpc_params=httpc_params))