feat: troppa roba davvero
This commit is contained in:
gabdevele
@@ -1,152 +0,0 @@
|
||||
from models.example import AddonExampleDocument
|
||||
|
||||
DOCS_EXAMPLES_URL = "https://docs.mitmproxy.org/stable/addons/examples/"
|
||||
|
||||
BUILTIN_EXAMPLES = [
|
||||
AddonExampleDocument(
|
||||
id="http-tag-traffic",
|
||||
name="Tag proxied traffic",
|
||||
description=(
|
||||
"Adds a visible response header on every HTTP reply so you can tell "
|
||||
"immediately whether the request crossed the proxy."
|
||||
),
|
||||
category="http",
|
||||
source_url=DOCS_EXAMPLES_URL,
|
||||
built_in=True,
|
||||
code="""from mitmproxy import http
|
||||
|
||||
|
||||
def response(flow: http.HTTPFlow) -> None:
|
||||
flow.response.headers["x-theplug"] = "enabled"
|
||||
""",
|
||||
),
|
||||
AddonExampleDocument(
|
||||
id="http-force-query-flag",
|
||||
name="Force request flag",
|
||||
description=(
|
||||
"Adds or overwrites a request query flag before the flow is sent "
|
||||
"upstream. Useful when you need a stable marker on every request."
|
||||
),
|
||||
category="http",
|
||||
source_url=DOCS_EXAMPLES_URL,
|
||||
built_in=True,
|
||||
code="""from mitmproxy import http
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
flow.request.query["debug"] = "1"
|
||||
""",
|
||||
),
|
||||
AddonExampleDocument(
|
||||
id="http-local-health-response",
|
||||
name="Serve local health response",
|
||||
description=(
|
||||
"Answers one health endpoint directly from the proxy without "
|
||||
"touching the upstream service. Useful for smoke tests and checks."
|
||||
),
|
||||
category="http",
|
||||
source_url=DOCS_EXAMPLES_URL,
|
||||
built_in=True,
|
||||
code="""from mitmproxy import http
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
if flow.request.path == "/healthz":
|
||||
flow.response = http.Response.make(
|
||||
200,
|
||||
b"ok",
|
||||
{"content-type": "text/plain"},
|
||||
)
|
||||
""",
|
||||
),
|
||||
AddonExampleDocument(
|
||||
id="http-rewrite-health-path",
|
||||
name="Rewrite one request path",
|
||||
description=(
|
||||
"Rewrites a single incoming path before forwarding it upstream. "
|
||||
"Useful when the public route and upstream route do not match."
|
||||
),
|
||||
category="http",
|
||||
source_url=DOCS_EXAMPLES_URL,
|
||||
built_in=True,
|
||||
code="""from mitmproxy import http
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
if flow.request.path == "/healthz":
|
||||
flow.request.path = "/status"
|
||||
""",
|
||||
),
|
||||
AddonExampleDocument(
|
||||
id="websocket-rewrite",
|
||||
name="WebSocket rewrite",
|
||||
description=(
|
||||
"Rewrites matching bytes inside live WebSocket frames to flip a "
|
||||
"runtime marker while the flow is passing through the proxy."
|
||||
),
|
||||
category="websocket",
|
||||
source_url=DOCS_EXAMPLES_URL,
|
||||
built_in=True,
|
||||
code="""from mitmproxy import http
|
||||
|
||||
|
||||
def websocket_message(flow: http.HTTPFlow) -> None:
|
||||
if flow.websocket is None:
|
||||
return
|
||||
message = flow.websocket.messages[-1]
|
||||
message.content = message.content.replace(b"debug=false", b"debug=true")
|
||||
""",
|
||||
),
|
||||
AddonExampleDocument(
|
||||
id="tcp-log-messages",
|
||||
name="TCP log messages",
|
||||
description=(
|
||||
"Logs direction and byte size for every TCP frame so you can see "
|
||||
"what is crossing the route without modifying the payload."
|
||||
),
|
||||
category="tcp",
|
||||
source_url=DOCS_EXAMPLES_URL,
|
||||
built_in=True,
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import tcp
|
||||
|
||||
|
||||
def tcp_message(flow: tcp.TCPFlow) -> None:
|
||||
message = flow.messages[-1]
|
||||
direction = "client" if message.from_client else "server"
|
||||
logging.info("%s sent %d bytes", direction, len(message.content))
|
||||
""",
|
||||
),
|
||||
AddonExampleDocument(
|
||||
id="dns-spoof-a-record",
|
||||
name="DNS spoof A record",
|
||||
description=(
|
||||
"Overrides one DNS A record while leaving all other queries alone. "
|
||||
"Useful for selective redirection during local testing."
|
||||
),
|
||||
category="dns",
|
||||
source_url=DOCS_EXAMPLES_URL,
|
||||
built_in=True,
|
||||
code="""import ipaddress
|
||||
|
||||
from mitmproxy import dns
|
||||
|
||||
|
||||
def dns_request(flow: dns.DNSFlow) -> None:
|
||||
question = flow.request.question
|
||||
if question and question.name == "target.local" and question.type == dns.types.A:
|
||||
flow.response = flow.request.succeed(
|
||||
[
|
||||
dns.ResourceRecord(
|
||||
name="target.local",
|
||||
type=dns.types.A,
|
||||
class_=dns.classes.IN,
|
||||
ttl=60,
|
||||
data=ipaddress.ip_address("127.0.0.1").packed,
|
||||
)
|
||||
]
|
||||
)
|
||||
""",
|
||||
),
|
||||
]
|
||||
@@ -1,8 +1,11 @@
|
||||
import asyncio
|
||||
import ssl
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from models.proxy import HealthCheckDocument, ProxyDocument
|
||||
from app.models.health import HealthCheckEntry, HealthSnapshot
|
||||
from app.models.proxy import HealthCheckDocument, ProxyDocument
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -13,33 +16,48 @@ class Target:
|
||||
protocol: str
|
||||
|
||||
|
||||
async def check_proxy(proxy: ProxyDocument) -> dict[str, object]:
|
||||
async def check_proxy(proxy: ProxyDocument) -> HealthSnapshot:
|
||||
timestamp = datetime.now(UTC)
|
||||
if not proxy.healthcheck.enabled:
|
||||
return {"enabled": False, "overall": "disabled", "checks": []}
|
||||
return HealthSnapshot(
|
||||
proxy_port=proxy.proxy_port,
|
||||
timestamp=timestamp,
|
||||
enabled=False,
|
||||
overall="disabled",
|
||||
checks=[],
|
||||
)
|
||||
|
||||
upstream_port = proxy.service.upstream_port or proxy.service.port
|
||||
checks = await asyncio.gather(
|
||||
check_target(
|
||||
Target(
|
||||
name="upstream",
|
||||
host=proxy.service.host,
|
||||
port=upstream_port,
|
||||
protocol=health_protocol(proxy),
|
||||
checks = list(
|
||||
await asyncio.gather(
|
||||
check_target(
|
||||
Target(
|
||||
name="upstream",
|
||||
host=proxy.service.host,
|
||||
port=upstream_port,
|
||||
protocol=health_protocol(proxy),
|
||||
),
|
||||
proxy.healthcheck,
|
||||
),
|
||||
proxy.healthcheck,
|
||||
),
|
||||
check_target(
|
||||
Target(
|
||||
name="proxy",
|
||||
host=listen_host_for_check(proxy.listen_host),
|
||||
port=proxy.proxy_port,
|
||||
protocol=health_protocol(proxy),
|
||||
check_target(
|
||||
Target(
|
||||
name="proxy",
|
||||
host=listen_host_for_check(proxy.listen_host),
|
||||
port=proxy.proxy_port,
|
||||
protocol=health_protocol(proxy),
|
||||
),
|
||||
proxy.healthcheck,
|
||||
),
|
||||
proxy.healthcheck,
|
||||
),
|
||||
)
|
||||
)
|
||||
overall = "healthy" if all(check.healthy for check in checks) else "unhealthy"
|
||||
return HealthSnapshot(
|
||||
proxy_port=proxy.proxy_port,
|
||||
timestamp=timestamp,
|
||||
enabled=True,
|
||||
overall=overall,
|
||||
checks=checks,
|
||||
)
|
||||
overall = "healthy" if all(check["healthy"] for check in checks) else "unhealthy"
|
||||
return {"enabled": True, "overall": overall, "checks": checks}
|
||||
|
||||
|
||||
def health_protocol(proxy: ProxyDocument) -> str:
|
||||
@@ -55,7 +73,7 @@ def listen_host_for_check(host: str) -> str:
|
||||
async def check_target(
|
||||
target: Target,
|
||||
config: HealthCheckDocument,
|
||||
) -> dict[str, object]:
|
||||
) -> HealthCheckEntry:
|
||||
try:
|
||||
match target.protocol:
|
||||
case "http":
|
||||
@@ -81,7 +99,8 @@ async def check_http(
|
||||
config: HealthCheckDocument,
|
||||
*,
|
||||
use_tls: bool,
|
||||
) -> dict[str, object]:
|
||||
) -> HealthCheckEntry:
|
||||
started = time.perf_counter()
|
||||
reader, writer = await open_connection(target, config, use_tls=use_tls)
|
||||
try:
|
||||
path = config.path if config.path.startswith("/") else f"/{config.path}"
|
||||
@@ -101,7 +120,13 @@ async def check_http(
|
||||
status = int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else None
|
||||
expected = config.expected_status
|
||||
healthy = status == expected if expected is not None else status is not None
|
||||
return result(target, healthy, None, {"status": status})
|
||||
return result(
|
||||
target,
|
||||
healthy,
|
||||
None,
|
||||
{"status": status},
|
||||
benchmark_ms=elapsed_ms(started) if config.benchmark_enabled else None,
|
||||
)
|
||||
|
||||
|
||||
async def check_tcp(
|
||||
@@ -109,14 +134,20 @@ async def check_tcp(
|
||||
config: HealthCheckDocument,
|
||||
*,
|
||||
use_tls: bool,
|
||||
) -> dict[str, object]:
|
||||
) -> HealthCheckEntry:
|
||||
started = time.perf_counter()
|
||||
_reader, writer = await open_connection(target, config, use_tls=use_tls)
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return result(target, True)
|
||||
return result(
|
||||
target,
|
||||
True,
|
||||
benchmark_ms=elapsed_ms(started) if config.benchmark_enabled else None,
|
||||
)
|
||||
|
||||
|
||||
async def check_udp(target: Target, config: HealthCheckDocument) -> dict[str, object]:
|
||||
async def check_udp(target: Target, config: HealthCheckDocument) -> HealthCheckEntry:
|
||||
started = time.perf_counter()
|
||||
loop = asyncio.get_running_loop()
|
||||
transport, protocol = await loop.create_datagram_endpoint(
|
||||
lambda: DatagramProbe(config.expect_response),
|
||||
@@ -128,7 +159,13 @@ async def check_udp(target: Target, config: HealthCheckDocument) -> dict[str, ob
|
||||
if not config.expect_response:
|
||||
return result(target, True, None, {"mode": "send-only"})
|
||||
data = await asyncio.wait_for(protocol.response, config.timeout_seconds)
|
||||
return result(target, True, None, {"bytes": len(data)})
|
||||
return result(
|
||||
target,
|
||||
True,
|
||||
None,
|
||||
{"bytes": len(data)},
|
||||
benchmark_ms=elapsed_ms(started) if config.benchmark_enabled else None,
|
||||
)
|
||||
finally:
|
||||
transport.close()
|
||||
|
||||
@@ -159,7 +196,7 @@ class DatagramProbe(asyncio.DatagramProtocol):
|
||||
if not expect_response:
|
||||
self.response.set_result(b"")
|
||||
|
||||
def datagram_received(self, data, _addr):
|
||||
def datagram_received(self, data, addr):
|
||||
if not self.response.done():
|
||||
self.response.set_result(data)
|
||||
|
||||
@@ -173,13 +210,19 @@ def result(
|
||||
healthy: bool,
|
||||
error: str | None = None,
|
||||
details: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"target": target.name,
|
||||
"host": target.host,
|
||||
"port": target.port,
|
||||
"protocol": target.protocol,
|
||||
"healthy": healthy,
|
||||
"error": error,
|
||||
"details": details or {},
|
||||
}
|
||||
benchmark_ms: float | None = None,
|
||||
) -> HealthCheckEntry:
|
||||
return HealthCheckEntry(
|
||||
target=target.name,
|
||||
host=target.host,
|
||||
port=target.port,
|
||||
protocol=target.protocol,
|
||||
healthy=healthy,
|
||||
error=error,
|
||||
duration_ms=benchmark_ms,
|
||||
details=details or {},
|
||||
)
|
||||
|
||||
|
||||
def elapsed_ms(started: float) -> float:
|
||||
return round((time.perf_counter() - started) * 1000, 2)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models.health import HealthSnapshot
|
||||
from app.models.proxy import ProxyDocument
|
||||
from app.repositories.health import record_health_sample
|
||||
from app.repositories.proxy import list_proxies
|
||||
from app.services.mitmproxy.health import check_proxy
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HealthMonitor:
|
||||
def __init__(self):
|
||||
self.task: asyncio.Task[None] | None = None
|
||||
self.subscribers: dict[int, set[asyncio.Queue[HealthSnapshot]]] = {}
|
||||
|
||||
async def start(self) -> None:
|
||||
if self.task is None or self.task.done():
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self.task is None:
|
||||
return
|
||||
self.task.cancel()
|
||||
await asyncio.gather(self.task, return_exceptions=True)
|
||||
self.task = None
|
||||
|
||||
async def run(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
next_cycle = loop.time()
|
||||
while True:
|
||||
delay = next_cycle - loop.time()
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
cycle_started = loop.time()
|
||||
try:
|
||||
await self.run_cycle()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Health monitor cycle failed")
|
||||
next_cycle = loop.time() + min(
|
||||
1.0,
|
||||
float(settings.HEALTHCHECK_INTERVAL_SECONDS),
|
||||
)
|
||||
continue
|
||||
|
||||
next_cycle = cycle_started + settings.HEALTHCHECK_INTERVAL_SECONDS
|
||||
|
||||
async def run_cycle(self) -> None:
|
||||
proxies = [
|
||||
proxy
|
||||
for proxy in await list_proxies()
|
||||
if proxy.status == "running" and proxy.healthcheck.enabled
|
||||
]
|
||||
async with asyncio.TaskGroup() as group:
|
||||
for proxy in proxies:
|
||||
group.create_task(self.run_check(proxy))
|
||||
|
||||
async def run_check(self, proxy: ProxyDocument) -> None:
|
||||
try:
|
||||
snapshot = await check_proxy(proxy)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Health check failed for proxy %s",
|
||||
proxy.proxy_port,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
await record_health_sample(snapshot)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Health snapshot persistence failed for proxy %s",
|
||||
proxy.proxy_port,
|
||||
)
|
||||
|
||||
await self.publish(proxy.proxy_port, snapshot)
|
||||
|
||||
async def publish(self, proxy_port: int, snapshot: HealthSnapshot) -> None:
|
||||
for queue in list(self.subscribers.get(proxy_port, set())):
|
||||
try:
|
||||
queue.put_nowait(snapshot)
|
||||
except asyncio.QueueFull:
|
||||
logger.debug(
|
||||
"Dropped health update for slow subscriber on proxy %s",
|
||||
proxy_port,
|
||||
)
|
||||
continue
|
||||
|
||||
def subscribe(self, proxy_port: int) -> asyncio.Queue[HealthSnapshot]:
|
||||
queue: asyncio.Queue[HealthSnapshot] = asyncio.Queue(maxsize=8)
|
||||
listeners = self.subscribers.setdefault(proxy_port, set())
|
||||
listeners.add(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(
|
||||
self,
|
||||
proxy_port: int,
|
||||
queue: asyncio.Queue[HealthSnapshot],
|
||||
) -> None:
|
||||
listeners = self.subscribers.get(proxy_port)
|
||||
if listeners is None:
|
||||
return
|
||||
listeners.discard(queue)
|
||||
if not listeners:
|
||||
self.subscribers.pop(proxy_port, None)
|
||||
|
||||
|
||||
health_monitor = HealthMonitor()
|
||||
@@ -5,10 +5,15 @@ import signal
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from integrations import docker
|
||||
from integrations.docker.ports import get_random_port
|
||||
from models.proxy import ProxyDocument
|
||||
from services.mitmproxy.options import MitmOptionError, validate_options
|
||||
|
||||
from app.integrations.docker import change_port
|
||||
from app.integrations.docker.ports import get_random_port
|
||||
from app.models.proxy import ProxyDocument
|
||||
from app.services.mitmproxy.options import MitmOptionError, validate_options
|
||||
from app.services.mitmproxy.runtime_validation import (
|
||||
ReservedProxyPortError,
|
||||
ensure_proxy_port_not_reserved,
|
||||
)
|
||||
|
||||
|
||||
class MitmdumpError(Exception):
|
||||
@@ -22,6 +27,7 @@ class MitmdumpManager:
|
||||
self.processes: dict[int, asyncio.subprocess.Process] = {}
|
||||
self.attached_pids: dict[int, int] = {}
|
||||
self.locks: dict[int, asyncio.Lock] = {}
|
||||
self.upstream_port_attempts = 8
|
||||
|
||||
async def start(self, proxy: ProxyDocument) -> dict[str, object]:
|
||||
async with self.lock_for(proxy.proxy_port):
|
||||
@@ -31,7 +37,30 @@ class MitmdumpManager:
|
||||
async with self.lock_for(proxy.proxy_port):
|
||||
return await self.stop_unlocked(proxy)
|
||||
|
||||
def cleanup_runtime_artifacts(self, proxy: ProxyDocument) -> None:
|
||||
script_path = (
|
||||
Path(proxy.script_path)
|
||||
if proxy.script_path
|
||||
else self.script_path_for(proxy)
|
||||
)
|
||||
log_path = Path(proxy.log_path) if proxy.log_path else self.log_path_for(proxy)
|
||||
config_path = (
|
||||
Path(proxy.config_path)
|
||||
if proxy.config_path
|
||||
else self.config_path_for(proxy)
|
||||
)
|
||||
|
||||
self.remove_file(script_path)
|
||||
self.remove_file(log_path)
|
||||
self.remove_file(config_path)
|
||||
self.remove_dir(self.conf_dir_for(proxy))
|
||||
|
||||
async def start_unlocked(self, proxy: ProxyDocument) -> dict[str, object]:
|
||||
try:
|
||||
ensure_proxy_port_not_reserved(proxy.proxy_port)
|
||||
except ReservedProxyPortError as error:
|
||||
raise MitmdumpError(str(error)) from error
|
||||
|
||||
if await self.status(proxy) == "running":
|
||||
raise MitmdumpError("Proxy is already running")
|
||||
|
||||
@@ -44,9 +73,9 @@ class MitmdumpManager:
|
||||
except MitmOptionError as error:
|
||||
raise MitmdumpError(str(error)) from error
|
||||
upstream_port = await self.prepare_upstream(proxy)
|
||||
script_path = self.write_script(proxy)
|
||||
config_path = self.write_config(proxy)
|
||||
log_path = self.log_path(proxy)
|
||||
script_path = await self.write_script(proxy)
|
||||
config_path = await self.write_config(proxy)
|
||||
log_path = await self.log_path(proxy)
|
||||
command = self.build_command(
|
||||
command_path,
|
||||
proxy,
|
||||
@@ -64,7 +93,10 @@ class MitmdumpManager:
|
||||
failed = await self.exited_early(process)
|
||||
if failed:
|
||||
await self.restore_upstream(proxy, upstream_port)
|
||||
reason = self.read_log_tail(log_path) or "mitmdump exited during startup"
|
||||
reason = (
|
||||
await self.read_log_tail(log_path)
|
||||
or "mitmdump exited during startup"
|
||||
)
|
||||
raise MitmdumpError(reason)
|
||||
|
||||
self.processes[proxy.proxy_port] = process
|
||||
@@ -174,18 +206,22 @@ class MitmdumpManager:
|
||||
if proxy.service.name is None or proxy.service.folder is None:
|
||||
return proxy.service.port
|
||||
|
||||
upstream_port = get_random_port()
|
||||
changed = await docker.change_port(
|
||||
proxy.service.name,
|
||||
Path(proxy.service.folder),
|
||||
proxy.service.port,
|
||||
upstream_port,
|
||||
host_to="127.0.0.1",
|
||||
rewrite_host=True,
|
||||
)
|
||||
if not changed:
|
||||
raise MitmdumpError("Unable to move Docker service port")
|
||||
return upstream_port
|
||||
attempted_ports: set[int] = {proxy.proxy_port, proxy.service.port}
|
||||
for _attempt in range(self.upstream_port_attempts):
|
||||
upstream_port = self.pick_upstream_port(attempted_ports)
|
||||
attempted_ports.add(upstream_port)
|
||||
changed = await change_port(
|
||||
proxy.service.name,
|
||||
Path(proxy.service.folder),
|
||||
proxy.service.port,
|
||||
upstream_port,
|
||||
host_to="127.0.0.1",
|
||||
rewrite_host=True,
|
||||
)
|
||||
if changed:
|
||||
return upstream_port
|
||||
|
||||
raise MitmdumpError("Unable to move Docker service port after retries")
|
||||
|
||||
async def restore_upstream(
|
||||
self,
|
||||
@@ -199,7 +235,7 @@ class MitmdumpManager:
|
||||
if upstream_port == proxy.service.port:
|
||||
return True
|
||||
|
||||
return await docker.change_port(
|
||||
return await change_port(
|
||||
proxy.service.name,
|
||||
Path(proxy.service.folder),
|
||||
upstream_port,
|
||||
@@ -257,22 +293,29 @@ class MitmdumpManager:
|
||||
return False
|
||||
return proxy.mode in {"regular", "socks5", "dns", "transparent"}
|
||||
|
||||
def write_script(self, proxy: ProxyDocument) -> Path:
|
||||
def pick_upstream_port(self, excluded_ports: set[int]) -> int:
|
||||
while True:
|
||||
candidate = get_random_port()
|
||||
if candidate not in excluded_ports:
|
||||
return candidate
|
||||
|
||||
async def write_script(self, proxy: ProxyDocument) -> Path:
|
||||
path = self.script_path_for(proxy)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(proxy.code, encoding="utf-8")
|
||||
await asyncio.to_thread(path.parent.mkdir, parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(path.write_text, proxy.code, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def write_config(self, proxy: ProxyDocument) -> Path:
|
||||
async def write_config(self, proxy: ProxyDocument) -> Path:
|
||||
path = self.config_path_for(proxy)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(path.parent.mkdir, parents=True, exist_ok=True)
|
||||
options = {"block_global": False, **proxy.mitm_options}
|
||||
path.write_text(yaml.safe_dump(options, sort_keys=True), encoding="utf-8")
|
||||
config_text = yaml.safe_dump(options, sort_keys=True)
|
||||
await asyncio.to_thread(path.write_text, config_text, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def log_path(self, proxy: ProxyDocument) -> Path:
|
||||
async def log_path(self, proxy: ProxyDocument) -> Path:
|
||||
path = self.log_path_for(proxy)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(path.parent.mkdir, parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
async def spawn(self, command: list[str], log_path: Path):
|
||||
@@ -385,12 +428,30 @@ class MitmdumpManager:
|
||||
def log_path_for(self, proxy: ProxyDocument) -> Path:
|
||||
return self.work_dir / "logs" / f"{proxy.proxy_port}.log"
|
||||
|
||||
def read_log_tail(self, path: Path, limit: int = 4000) -> str:
|
||||
if not path.exists():
|
||||
async def read_log_tail(self, path: Path, limit: int = 4000) -> str:
|
||||
if not await asyncio.to_thread(path.exists):
|
||||
return ""
|
||||
content = path.read_text(encoding="utf-8", errors="replace")
|
||||
content = await asyncio.to_thread(
|
||||
path.read_text,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
return content[-limit:].strip()
|
||||
|
||||
def remove_file(self, path: Path) -> None:
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
def remove_dir(self, path: Path) -> None:
|
||||
try:
|
||||
shutil.rmtree(path)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except OSError:
|
||||
return
|
||||
|
||||
def lock_for(self, proxy_port: int) -> asyncio.Lock:
|
||||
lock = self.locks.get(proxy_port)
|
||||
if lock is None:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from repositories.proxy import list_proxies, update_proxy_runtime
|
||||
from services.mitmproxy.manager import manager
|
||||
from app.repositories.proxy import list_proxies, update_proxy_runtime
|
||||
from app.services.mitmproxy.manager import manager
|
||||
|
||||
|
||||
async def reconcile_runtime() -> list[tuple[int, dict[str, object]]]:
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
class ReservedProxyPortError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def backend_port() -> int:
|
||||
return settings.API_PORT
|
||||
|
||||
|
||||
def reserved_proxy_port_message(proxy_port: int) -> str:
|
||||
return (
|
||||
f"Proxy port {proxy_port} is reserved for the backend API "
|
||||
"and cannot be used by mitmdump."
|
||||
)
|
||||
|
||||
|
||||
def ensure_proxy_port_not_reserved(proxy_port: int) -> None:
|
||||
if proxy_port == backend_port():
|
||||
raise ReservedProxyPortError(reserved_proxy_port_message(proxy_port))
|
||||
@@ -0,0 +1,524 @@
|
||||
from app.models.template import ProxyTemplateDocument
|
||||
|
||||
DOCS_TEMPLATES_URL = "https://docs.mitmproxy.org/stable/addons/examples/"
|
||||
|
||||
|
||||
def builtin_template(
|
||||
*,
|
||||
id: str,
|
||||
name: str,
|
||||
protocol: str,
|
||||
family: str,
|
||||
tags: list[str],
|
||||
action: str,
|
||||
summary: str,
|
||||
behavior: str,
|
||||
code: str,
|
||||
) -> ProxyTemplateDocument:
|
||||
category = protocol if protocol in {"http", "tcp"} else family
|
||||
return ProxyTemplateDocument(
|
||||
id=id,
|
||||
name=name,
|
||||
description=summary,
|
||||
category=category,
|
||||
protocol=protocol,
|
||||
family=family,
|
||||
tags=tags,
|
||||
action=action,
|
||||
summary=summary,
|
||||
behavior=behavior,
|
||||
source_url=DOCS_TEMPLATES_URL,
|
||||
built_in=True,
|
||||
code=code,
|
||||
)
|
||||
|
||||
|
||||
BUILTIN_TEMPLATES = [
|
||||
builtin_template(
|
||||
id="http-block-path-traversal",
|
||||
name="Block path traversal",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["path-traversal", "web", "block"],
|
||||
action="block",
|
||||
summary="Stops common path traversal probes in path and query.",
|
||||
behavior=(
|
||||
"Logs the matched traversal token and immediately drops the client "
|
||||
"connection before the request reaches the upstream service."
|
||||
),
|
||||
code="""import logging
|
||||
from urllib.parse import unquote
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
TRAVERSAL_MARKERS = (
|
||||
"../",
|
||||
"..\\\\",
|
||||
"%2e%2e",
|
||||
"%252e%252e",
|
||||
"..%2f",
|
||||
"..%5c",
|
||||
)
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
raw_target = (
|
||||
f"{flow.request.path}?{flow.request.query}"
|
||||
if flow.request.query
|
||||
else flow.request.path
|
||||
)
|
||||
normalized = unquote(unquote(raw_target)).lower()
|
||||
for marker in TRAVERSAL_MARKERS:
|
||||
if marker in normalized:
|
||||
logging.warning(
|
||||
"[http-block-path-traversal] blocked %s %s from %s",
|
||||
flow.request.method,
|
||||
flow.request.pretty_url,
|
||||
flow.client_conn.peername,
|
||||
)
|
||||
flow.kill()
|
||||
return
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="http-block-blacklisted-substrings",
|
||||
name="Block blacklisted substrings",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["blacklist-string", "web", "block"],
|
||||
action="block",
|
||||
summary="Blocks requests containing forbidden strings in path, query, or body.",
|
||||
behavior=(
|
||||
"Scans URL and request body for blacklisted markers, then serves a "
|
||||
"blank HTML page with 200 OK and logs the first hit."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
BLOCKED_STRINGS = (
|
||||
"flag{",
|
||||
"give_me_shell",
|
||||
"debug=1",
|
||||
"/etc/passwd",
|
||||
)
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
body = flow.request.get_text(strict=False)
|
||||
haystack = "\\n".join(
|
||||
[flow.request.path, str(flow.request.query), body]
|
||||
).lower()
|
||||
for marker in BLOCKED_STRINGS:
|
||||
if marker.lower() in haystack:
|
||||
logging.warning(
|
||||
"[http-block-blacklisted-substrings] blocked marker=%r url=%s",
|
||||
marker,
|
||||
flow.request.pretty_url,
|
||||
)
|
||||
flow.response = http.Response.make(
|
||||
200,
|
||||
b"",
|
||||
{"content-type": "text/html; charset=utf-8"},
|
||||
)
|
||||
return
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="http-block-large-body",
|
||||
name="Block large request bodies",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["length-limit", "body-size", "web", "block"],
|
||||
action="block",
|
||||
summary="Rejects oversized HTTP request bodies before they hit the service.",
|
||||
behavior=(
|
||||
"Checks the buffered request body length, returns 413 Payload Too "
|
||||
"Large, and logs the observed size."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
MAX_BODY_BYTES = 8192
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
size = len(flow.request.raw_content or b"")
|
||||
if size > MAX_BODY_BYTES:
|
||||
logging.warning(
|
||||
"[http-block-large-body] blocked %s bytes on %s",
|
||||
size,
|
||||
flow.request.pretty_url,
|
||||
)
|
||||
flow.response = http.Response.make(
|
||||
413,
|
||||
b"",
|
||||
{"content-type": "text/plain"},
|
||||
)
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="http-block-suspicious-methods",
|
||||
name="Block suspicious HTTP methods",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["method-filter", "web", "block"],
|
||||
action="block",
|
||||
summary="Denies uncommon or dangerous HTTP methods.",
|
||||
behavior=(
|
||||
"Returns 405 Method Not Allowed for methods outside the allowlist "
|
||||
"and logs the blocked method and target URL."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
ALLOWED_METHODS = {"GET", "POST", "HEAD"}
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
if flow.request.method not in ALLOWED_METHODS:
|
||||
logging.warning(
|
||||
"[http-block-suspicious-methods] blocked method=%s url=%s",
|
||||
flow.request.method,
|
||||
flow.request.pretty_url,
|
||||
)
|
||||
flow.response = http.Response.make(
|
||||
405,
|
||||
b"",
|
||||
{"allow": "GET, POST, HEAD"},
|
||||
)
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="http-block-admin-paths",
|
||||
name="Block admin and debug paths",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["admin-path", "debug-path", "web", "block"],
|
||||
action="block",
|
||||
summary="Protects common admin, console, and debug endpoints.",
|
||||
behavior=(
|
||||
"Matches the request path against a small denylist and returns a "
|
||||
"blank 404 page before the request is forwarded."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
BLOCKED_PREFIXES = (
|
||||
"/admin",
|
||||
"/debug",
|
||||
"/console",
|
||||
"/actuator",
|
||||
)
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
if flow.request.path.startswith(BLOCKED_PREFIXES):
|
||||
logging.warning(
|
||||
"[http-block-admin-paths] blocked path=%s",
|
||||
flow.request.path,
|
||||
)
|
||||
flow.response = http.Response.make(
|
||||
404,
|
||||
b"",
|
||||
{"content-type": "text/html; charset=utf-8"},
|
||||
)
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="http-sanitize-headers-basic",
|
||||
name="Sanitize risky forwarding headers",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["rewrite", "headers", "web"],
|
||||
action="rewrite",
|
||||
summary=(
|
||||
"Removes a few risky forwarding headers before the request is "
|
||||
"sent upstream."
|
||||
),
|
||||
behavior=(
|
||||
"Deletes selected override or forwarding headers and logs what was "
|
||||
"rewritten without blocking the request."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
STRIP_HEADERS = (
|
||||
"x-forwarded-host",
|
||||
"x-http-method-override",
|
||||
"x-original-url",
|
||||
"x-rewrite-url",
|
||||
)
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
removed = []
|
||||
for header in STRIP_HEADERS:
|
||||
if header in flow.request.headers:
|
||||
removed.append(header)
|
||||
del flow.request.headers[header]
|
||||
if removed:
|
||||
logging.info(
|
||||
"[http-sanitize-headers-basic] stripped headers=%s url=%s",
|
||||
",".join(removed),
|
||||
flow.request.pretty_url,
|
||||
)
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="http-block-command-injection-markers",
|
||||
name="Block command injection markers",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["command-injection", "web", "block"],
|
||||
action="block",
|
||||
summary=(
|
||||
"Rejects obvious shell metacharacters and command-substitution "
|
||||
"markers."
|
||||
),
|
||||
behavior=(
|
||||
"Scans URL and body for shell-style injection markers, logs the "
|
||||
"first hit, and drops the client connection."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
BLOCKED_MARKERS = (";", "&&", "||", "$(", "`", "|sh", "|bash")
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
body = flow.request.get_text(strict=False)
|
||||
haystack = "\\n".join(
|
||||
[flow.request.pretty_url, body]
|
||||
).lower()
|
||||
for marker in BLOCKED_MARKERS:
|
||||
if marker in haystack:
|
||||
logging.warning(
|
||||
"[http-block-command-injection-markers] blocked marker=%r url=%s",
|
||||
marker,
|
||||
flow.request.pretty_url,
|
||||
)
|
||||
flow.kill()
|
||||
return
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="http-block-sqli-markers",
|
||||
name="Block obvious SQLi markers",
|
||||
protocol="http",
|
||||
family="web-hardening",
|
||||
tags=["sqli", "web", "block"],
|
||||
action="block",
|
||||
summary="Stops common SQL injection probes before they reach the app.",
|
||||
behavior=(
|
||||
"Matches a short list of high-signal SQLi markers in path, query, "
|
||||
"and body, then serves a blank HTML page and logs the hit."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import http
|
||||
|
||||
BLOCKED_MARKERS = (
|
||||
"' or 1=1",
|
||||
'" or 1=1',
|
||||
"union select",
|
||||
"sleep(",
|
||||
"information_schema",
|
||||
"--",
|
||||
)
|
||||
|
||||
|
||||
def request(flow: http.HTTPFlow) -> None:
|
||||
body = flow.request.get_text(strict=False)
|
||||
haystack = "\\n".join(
|
||||
[flow.request.path, str(flow.request.query), body]
|
||||
).lower()
|
||||
for marker in BLOCKED_MARKERS:
|
||||
if marker in haystack:
|
||||
logging.warning(
|
||||
"[http-block-sqli-markers] blocked marker=%r url=%s",
|
||||
marker,
|
||||
flow.request.pretty_url,
|
||||
)
|
||||
flow.response = http.Response.make(
|
||||
200,
|
||||
b"",
|
||||
{"content-type": "text/html; charset=utf-8"},
|
||||
)
|
||||
return
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="tcp-block-repeated-byte-threshold",
|
||||
name="Block repeated-byte flood",
|
||||
protocol="tcp",
|
||||
family="tcp-hardening",
|
||||
tags=["bof", "repeated-byte", "tcp", "block"],
|
||||
action="block",
|
||||
summary="Closes connections that send suspiciously long runs of the same byte.",
|
||||
behavior=(
|
||||
"Measures the longest repeated-byte run in each client message, logs "
|
||||
"the byte and run length, and closes the TCP flow on threshold hit."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import tcp
|
||||
|
||||
MAX_REPEAT = 128
|
||||
|
||||
|
||||
def longest_run(data: bytes) -> tuple[int, int]:
|
||||
best_byte = -1
|
||||
best_run = 0
|
||||
current_byte = -1
|
||||
current_run = 0
|
||||
for byte in data:
|
||||
if byte == current_byte:
|
||||
current_run += 1
|
||||
else:
|
||||
current_byte = byte
|
||||
current_run = 1
|
||||
if current_run > best_run:
|
||||
best_byte = byte
|
||||
best_run = current_run
|
||||
return best_byte, best_run
|
||||
|
||||
|
||||
def tcp_message(flow: tcp.TCPFlow) -> None:
|
||||
message = flow.messages[-1]
|
||||
if not message.from_client:
|
||||
return
|
||||
repeated_byte, run_length = longest_run(message.content)
|
||||
if run_length > MAX_REPEAT:
|
||||
logging.warning(
|
||||
"[tcp-block-repeated-byte-threshold] blocked byte=0x%02x run=%d",
|
||||
repeated_byte,
|
||||
run_length,
|
||||
)
|
||||
message.drop()
|
||||
flow.kill()
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="tcp-block-message-length-threshold",
|
||||
name="Block oversized TCP messages",
|
||||
protocol="tcp",
|
||||
family="tcp-hardening",
|
||||
tags=["length-limit", "tcp", "block"],
|
||||
action="block",
|
||||
summary="Cuts off client messages that exceed a fixed size threshold.",
|
||||
behavior=(
|
||||
"Checks every client TCP message size, logs the offending length, "
|
||||
"drops the message, and kills the flow when the threshold is hit."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import tcp
|
||||
|
||||
MAX_MESSAGE_BYTES = 4096
|
||||
|
||||
|
||||
def tcp_message(flow: tcp.TCPFlow) -> None:
|
||||
message = flow.messages[-1]
|
||||
if not message.from_client:
|
||||
return
|
||||
size = len(message.content)
|
||||
if size > MAX_MESSAGE_BYTES:
|
||||
logging.warning(
|
||||
"[tcp-block-message-length-threshold] blocked message size=%d",
|
||||
size,
|
||||
)
|
||||
message.drop()
|
||||
flow.kill()
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="tcp-block-blacklisted-byte-sequences",
|
||||
name="Block blacklisted TCP byte sequences",
|
||||
protocol="tcp",
|
||||
family="tcp-hardening",
|
||||
tags=["blacklist-bytes", "tcp", "block"],
|
||||
action="block",
|
||||
summary=(
|
||||
"Blocks payloads containing forbidden byte sequences or marker "
|
||||
"strings."
|
||||
),
|
||||
behavior=(
|
||||
"Looks for dangerous byte patterns in client messages, then drops "
|
||||
"the message and closes the flow."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import tcp
|
||||
|
||||
BLOCKED_SEQUENCES = (
|
||||
b"/bin/sh",
|
||||
b"cat flag",
|
||||
b"AAAA%p%p",
|
||||
b"\\x90\\x90\\x90\\x90",
|
||||
)
|
||||
|
||||
|
||||
def tcp_message(flow: tcp.TCPFlow) -> None:
|
||||
message = flow.messages[-1]
|
||||
if not message.from_client:
|
||||
return
|
||||
for sequence in BLOCKED_SEQUENCES:
|
||||
if sequence in message.content:
|
||||
logging.warning(
|
||||
"[tcp-block-blacklisted-byte-sequences] blocked sequence=%r",
|
||||
sequence,
|
||||
)
|
||||
message.drop()
|
||||
flow.kill()
|
||||
return
|
||||
""",
|
||||
),
|
||||
builtin_template(
|
||||
id="tcp-block-non-printable-threshold",
|
||||
name="Block binary-noise threshold",
|
||||
protocol="tcp",
|
||||
family="tcp-hardening",
|
||||
tags=["non-printable", "binary-noise", "tcp", "block"],
|
||||
action="block",
|
||||
summary=(
|
||||
"Blocks client messages with an excessive amount of non-printable "
|
||||
"bytes."
|
||||
),
|
||||
behavior=(
|
||||
"Treats unusually binary-heavy client payloads as suspicious, logs "
|
||||
"the ratio, and closes the flow when the threshold is exceeded."
|
||||
),
|
||||
code="""import logging
|
||||
|
||||
from mitmproxy import tcp
|
||||
|
||||
NON_PRINTABLE_RATIO = 0.35
|
||||
|
||||
|
||||
def tcp_message(flow: tcp.TCPFlow) -> None:
|
||||
message = flow.messages[-1]
|
||||
if not message.from_client or not message.content:
|
||||
return
|
||||
non_printable = sum(
|
||||
1
|
||||
for byte in message.content
|
||||
if byte < 0x20 and byte not in {0x09, 0x0a, 0x0d}
|
||||
)
|
||||
ratio = non_printable / len(message.content)
|
||||
if ratio > NON_PRINTABLE_RATIO:
|
||||
logging.warning(
|
||||
"[tcp-block-non-printable-threshold] blocked ratio=%.2f size=%d",
|
||||
ratio,
|
||||
len(message.content),
|
||||
)
|
||||
message.drop()
|
||||
flow.kill()
|
||||
""",
|
||||
),
|
||||
]
|
||||
Reference in New Issue
Block a user