reloia/thePlug
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove Redis dependency and related code

Browse Source
This commit is contained in:
gabdevele
2026-05-05 15:01:14 +02:00 Unverified
parent a523c2faa3
commit d977a953cc
21 changed files with 504 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.env.example
-1
View File
@@ -4,7 +4,6 @@ FRONTEND_PORT=5173
MONGO_INITDB_ROOT_PASSWORD=pass
MONGODB_URI=mongodb://theplug:pass@localhost:27017/theplug?authSource=admin
MONGODB_DB_NAME=theplug
REDIS_URL=redis://localhost:6379
# Use a random value in real environments.
JWT_SECRET_KEY=change-me
JWT_ALGORITHM=HS256
backend/Dockerfile
-1
View File
@@ -17,7 +17,6 @@ RUN pip install --no-cache-dir --upgrade pip && \
"pwdlib[argon2,bcrypt]>=0.3.0" \
"pyyaml>=6.0.3" \
"pydantic-settings>=2.13.1" \
"redis>=7.4.0" \
"ruff>=0.15.10"
COPY app ./src
backend/app/api/proxy/router.py
+147 -25
View File
@@ -1,8 +1,13 @@
import asyncio
from pathlib import Path
from core.config import settings
from core.security import require_authenticated_user
from fastapi import APIRouter, Depends, HTTPException, Request, status
from core.security import (
InvalidTokenError,
authenticate_token,
require_authenticated_user,
)
from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, status
from integrations.docker import get_all_services_in_dir
from integrations.docker.discovery import service_scan_root
from models.example import AddonExampleCreateInput
@@ -33,6 +38,7 @@ from services.mitmproxy.options import (
validate_options,
)
from services.mitmproxy.validator import ProxyCodeValidationError, validate_proxy_code
from starlette.websockets import WebSocketDisconnect
SUPPORTED_MODES = {
"regular",
@@ -60,8 +66,8 @@ SUPPORTED_PROTOCOLS = {
router = APIRouter(
prefix="/proxy",
tags=["Proxy"],
dependencies=[Depends(require_authenticated_user)],
)
auth_dependencies = [Depends(require_authenticated_user)]
async def _validate_code_or_raise(code: str) -> None:
@@ -117,12 +123,71 @@ async def get_proxy_or_404(proxy_port: int):
return proxy
@router.get("")
async def authenticate_proxy_socket(websocket: WebSocket) -> None:
token = websocket.query_params.get("token")
if token is None:
authorization = websocket.headers.get("authorization", "")
scheme, _, value = authorization.partition(" ")
if scheme.lower() == "bearer" and value:
token = value
if not token:
raise InvalidTokenError
await authenticate_token(token)
def proxy_log_path(proxy) -> Path:
if proxy.log_path:
return Path(proxy.log_path)
return manager.log_path_for(proxy)
def read_log_snapshot(path: Path, limit: int = 8000) -> str:
if not path.is_file():
return ""
return path.read_bytes()[-limit:].decode("utf-8", errors="replace")
async def watch_socket_disconnect(websocket: WebSocket) -> None:
while True:
await websocket.receive_text()
async def stream_proxy_logs(websocket: WebSocket, path: Path) -> None:
offset = path.stat().st_size if path.is_file() else 0
while True:
if not path.exists():
await asyncio.sleep(0.4)
continue
try:
size = path.stat().st_size
except OSError:
await asyncio.sleep(0.4)
continue
if size < offset:
offset = size
if size > offset:
with path.open("rb") as handle:
handle.seek(offset)
chunk = handle.read(size - offset).decode("utf-8", errors="replace")
offset = size
if chunk:
await websocket.send_json({"type": "append", "log": chunk})
await asyncio.sleep(0.4)
@router.get("", dependencies=auth_dependencies)
async def list_proxies_handler():
return await list_proxy_documents()
@router.get("/services")
@router.get("/services", dependencies=auth_dependencies)
async def list_services(request: Request, scan_root: str | None = None):
_ = settings
root = Path(scan_root).expanduser() if scan_root else service_scan_root()
@@ -131,17 +196,21 @@ async def list_services(request: Request, scan_root: str | None = None):
return {"scan_root": str(root), "services": services}
@router.get("/mitmproxy/options")
@router.get("/mitmproxy/options", dependencies=auth_dependencies)
async def list_mitmproxy_options():
return public_options(await load_options())
@router.get("/mitmproxy/examples")
@router.get("/mitmproxy/examples", dependencies=auth_dependencies)
async def list_mitmproxy_examples():
return [*BUILTIN_EXAMPLES, *await list_custom_examples()]
@router.post("/mitmproxy/examples", status_code=status.HTTP_201_CREATED)
@router.post(
"/mitmproxy/examples",
status_code=status.HTTP_201_CREATED,
dependencies=auth_dependencies,
)
async def create_mitmproxy_example(payload: AddonExampleCreateInput):
await _validate_code_or_raise(payload.code)
try:
@@ -156,6 +225,7 @@ async def create_mitmproxy_example(payload: AddonExampleCreateInput):
@router.delete(
"/mitmproxy/examples/{example_id}",
status_code=status.HTTP_204_NO_CONTENT,
dependencies=auth_dependencies,
)
async def delete_mitmproxy_example(example_id: str):
if any(example.id == example_id for example in BUILTIN_EXAMPLES):
@@ -172,12 +242,12 @@ async def delete_mitmproxy_example(example_id: str):
)
@router.get("/{proxy_port}")
@router.get("/{proxy_port}", dependencies=auth_dependencies)
async def get_proxy(proxy_port: int):
return await get_proxy_or_404(proxy_port)
@router.post("", status_code=status.HTTP_201_CREATED)
@router.post("", status_code=status.HTTP_201_CREATED, dependencies=auth_dependencies)
async def create_proxy_handler(payload: ProxyCreateInput):
existing_proxy = await get_proxy_by_port(payload.proxy_port)
if existing_proxy is not None:
@@ -195,7 +265,7 @@ async def create_proxy_handler(payload: ProxyCreateInput):
return await create_proxy(payload)
@router.put("/{proxy_port}")
@router.put("/{proxy_port}", dependencies=auth_dependencies)
async def edit_proxy(proxy_port: int, payload: ProxyUpdateInput):
if payload.code is not None:
await _validate_code_or_raise(payload.code)
@@ -217,7 +287,7 @@ async def edit_proxy(proxy_port: int, payload: ProxyUpdateInput):
return updated_proxy
@router.post("/{proxy_port}/start")
@router.post("/{proxy_port}/start", dependencies=auth_dependencies)
async def start_proxy(proxy_port: int):
proxy = await get_proxy_or_404(proxy_port)
@@ -233,14 +303,14 @@ async def start_proxy(proxy_port: int):
return await update_proxy_runtime(proxy_port, runtime)
@router.post("/{proxy_port}/stop")
@router.post("/{proxy_port}/stop", dependencies=auth_dependencies)
async def stop_proxy(proxy_port: int):
proxy = await get_proxy_or_404(proxy_port)
runtime = await manager.stop(proxy)
return await update_proxy_runtime(proxy_port, runtime)
@router.post("/{proxy_port}/restart")
@router.post("/{proxy_port}/restart", dependencies=auth_dependencies)
async def restart_proxy(proxy_port: int):
proxy = await get_proxy_or_404(proxy_port)
stopped = await manager.stop(proxy)
@@ -256,7 +326,7 @@ async def restart_proxy(proxy_port: int):
return await update_proxy_runtime(proxy_port, runtime)
@router.get("/{proxy_port}/status")
@router.get("/{proxy_port}/status", dependencies=auth_dependencies)
async def proxy_status(proxy_port: int):
proxy = await get_proxy_or_404(proxy_port)
runtime_status = await manager.status(proxy)
@@ -268,29 +338,81 @@ async def proxy_status(proxy_port: int):
return {"proxy_port": proxy.proxy_port, "status": runtime_status, "pid": proxy.pid}
@router.get("/{proxy_port}/health")
@router.get("/{proxy_port}/health", dependencies=auth_dependencies)
async def proxy_health(proxy_port: int):
proxy = await get_proxy_or_404(proxy_port)
return await check_proxy(proxy)
@router.get("/{proxy_port}/logs")
@router.get("/{proxy_port}/logs", dependencies=auth_dependencies)
async def proxy_logs(proxy_port: int):
proxy = await get_proxy_or_404(proxy_port)
if proxy.log_path is None:
return {"proxy_port": proxy.proxy_port, "log": ""}
path = Path(proxy.log_path)
if not path.is_file():
return {"proxy_port": proxy.proxy_port, "log": ""}
path = proxy_log_path(proxy)
return {
"proxy_port": proxy.proxy_port,
"log": path.read_text(encoding="utf-8", errors="replace")[-8000:],
"log": read_log_snapshot(path),
}
@router.delete("/{proxy_port}", status_code=status.HTTP_204_NO_CONTENT)
@router.websocket("/{proxy_port}/logs/ws")
async def proxy_logs_socket(websocket: WebSocket, proxy_port: int):
disconnect_task: asyncio.Task[None] | None = None
stream_task: asyncio.Task[None] | None = None
try:
try:
await authenticate_proxy_socket(websocket)
except InvalidTokenError:
await websocket.close(code=4401)
return
proxy = await get_proxy_by_port(proxy_port)
if proxy is None:
await websocket.close(code=4404)
return
path = proxy_log_path(proxy)
await websocket.accept()
await websocket.send_json({"type": "snapshot", "log": read_log_snapshot(path)})
disconnect_task = asyncio.create_task(watch_socket_disconnect(websocket))
stream_task = asyncio.create_task(stream_proxy_logs(websocket, path))
done, pending = await asyncio.wait(
{disconnect_task, stream_task},
return_when=asyncio.FIRST_EXCEPTION,
)
for task in pending:
task.cancel()
if pending:
await asyncio.gather(*pending, return_exceptions=True)
for task in done:
try:
task.result()
except asyncio.CancelledError:
return
except WebSocketDisconnect:
return
except RuntimeError:
await websocket.close(code=1011, reason="log stream failed")
return
except asyncio.CancelledError:
return
finally:
for task in (disconnect_task, stream_task):
if task is None or task.done():
continue
task.cancel()
@router.delete(
"/{proxy_port}",
status_code=status.HTTP_204_NO_CONTENT,
dependencies=auth_dependencies,
)
async def remove_proxy(proxy_port: int):
proxy = await get_proxy_by_port(proxy_port)
if proxy is None:
backend/app/core/config.py
-1
View File
@@ -12,7 +12,6 @@ class Settings(BaseSettings):
MONGODB_URI: str
MONGODB_DB_NAME: str
REDIS_URL: str
JWT_SECRET_KEY: str
JWT_ALGORITHM: str = "HS256"
JWT_EXPIRE_MINUTES: int = Field(default=720, gt=0)
backend/app/core/redis.py
-22
View File
@@ -1,22 +0,0 @@
import redis.asyncio as aioredis
from core.config import settings
_redis_pool: aioredis.Redis | None = None
async def init_redis() -> None:
global _redis_pool
_redis_pool = aioredis.from_url(
settings.REDIS_URL, encoding="utf-8", decode_responses=True
)
async def close_redis() -> None:
if _redis_pool:
await _redis_pool.aclose()
async def get_redis() -> aioredis.Redis:
if _redis_pool is None:
raise RuntimeError("Redis pool has not been initialized")
return _redis_pool
backend/app/main.py
-3
View File
@@ -7,7 +7,6 @@ from api.editor.router import router as editor_router
from api.proxy.router import router as proxy_router
from core.config import settings
from core.mongodb import close_mongodb, init_mongodb
from core.redis import close_redis, init_redis
from core.security import hash_password
from fastapi import FastAPI
from integrations.docker import get_all_services_in_dir
@@ -21,7 +20,6 @@ from services.mitmproxy.recovery import reconcile_runtime
@asynccontextmanager
async def lifespan(_: FastAPI):
await init_mongodb()
await init_redis()
await ensure_auth_indexes()
await ensure_single_account(
username=settings.AUTH_USERNAME,
@@ -56,7 +54,6 @@ async def lifespan(_: FastAPI):
try:
yield
finally:
await close_redis()
await close_mongodb()
backend/pyproject.toml
-1
View File
@@ -13,7 +13,6 @@ dependencies = [
"pydantic>=2.12.5",
"pydantic-settings>=2.13.1",
"pyyaml>=6.0.3",
"redis>=7.4.0",
"ruff>=0.15.10",
]
backend/tests/api/test_proxy.py
+86
View File
@@ -1,10 +1,12 @@
import importlib
import pytest
from core.security import get_auth_context, require_authenticated_user
from models.proxy import ProxyCreateInput, ProxyDocument
from models.service import ServiceEndpointDocument
from services.mitmproxy.examples import BUILTIN_EXAMPLES
from services.mitmproxy.validator import ProxyCodeValidationError
from starlette.websockets import WebSocketDisconnect
proxy_router_module = importlib.import_module("api.proxy.router")
@@ -15,12 +17,14 @@ def make_proxy_document(
name: str = "svc",
code: str = "proxy-code",
port: int = 9000,
log_path: str | None = None,
) -> ProxyDocument:
return ProxyDocument(
proxy_port=proxy_port,
name=name,
code=code,
service=ServiceEndpointDocument(port=port),
log_path=log_path,
)
@@ -292,6 +296,45 @@ def test_update_proxy_returns_422_when_code_validation_fails(
assert update_called is False
def test_update_proxy_persists_code_changes(client, monkeypatch: object):
captured: dict[str, object] = {}
updated_code = "def response(flow):\n flow.response.text = 'ok'\n"
async def fake_validate_proxy_code(code: str) -> None:
captured["validated"] = code
async def fake_update_proxy(
proxy_port: int,
payload: object,
) -> ProxyDocument | None:
captured["proxy_port"] = proxy_port
captured["code"] = payload.code
return make_proxy_document(
proxy_port=proxy_port,
code=payload.code,
)
monkeypatch.setattr(
proxy_router_module,
"validate_proxy_code",
fake_validate_proxy_code,
)
monkeypatch.setattr(proxy_router_module, "update_proxy", fake_update_proxy)
response = client.put(
"/proxy/8123",
json={"code": updated_code},
)
assert response.status_code == 200
assert response.json()["code"] == updated_code
assert captured == {
"validated": updated_code,
"proxy_port": 8123,
"code": updated_code,
}
def test_start_proxy_updates_runtime_state(client, monkeypatch: object):
proxy = make_proxy_document(proxy_port=4040)
captured: dict[str, object] = {}
@@ -384,6 +427,49 @@ def test_start_proxy_records_runtime_error(client, monkeypatch: object):
assert response.json()["last_error"] == "mitmdump executable not found"
def test_proxy_logs_socket_rejects_missing_token(client):
with pytest.raises(WebSocketDisconnect) as error:
with client.websocket_connect("/proxy/4040/logs/ws"):
pass
assert error.value.code == 4401
def test_proxy_logs_socket_streams_snapshot_and_append(client, monkeypatch, tmp_path):
log_path = tmp_path / "proxy.log"
log_path.write_text("line one\n", encoding="utf-8")
proxy = make_proxy_document(proxy_port=4040, log_path=str(log_path))
async def fake_authenticate(_: object) -> None:
return None
async def fake_get_proxy_by_port(proxy_port: int) -> ProxyDocument | None:
assert proxy_port == 4040
return proxy
monkeypatch.setattr(
proxy_router_module,
"authenticate_proxy_socket",
fake_authenticate,
)
monkeypatch.setattr(
proxy_router_module,
"get_proxy_by_port",
fake_get_proxy_by_port,
)
with client.websocket_connect("/proxy/4040/logs/ws?token=test-token") as socket:
snapshot = socket.receive_json()
assert snapshot == {"type": "snapshot", "log": "line one\n"}
with log_path.open("a", encoding="utf-8") as handle:
handle.write("line two\n")
handle.flush()
appended = socket.receive_json()
assert appended == {"type": "append", "log": "line two\n"}
def test_stop_proxy_updates_runtime_state(client, monkeypatch: object):
proxy = make_proxy_document(proxy_port=4040).model_copy(
update={"status": "running", "pid": 1234}
backend/tests/conftest.py
-1
View File
@@ -10,7 +10,6 @@ os.environ.setdefault(
"mongodb://theplug:pass@localhost:27017/theplug?authSource=admin",
)
os.environ.setdefault("MONGODB_DB_NAME", "theplug")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379")
os.environ.setdefault("JWT_SECRET_KEY", "test-secret-key")
os.environ.setdefault("JWT_ALGORITHM", "HS256")
os.environ.setdefault("JWT_EXPIRE_MINUTES", "720")
backend/uv.lock
Generated
-11
View File
@@ -1124,15 +1124,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
]
[[package]]
name = "redis"
version = "7.4.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/3759b1d0d72b7c92f0d70ffd9dc962b7b7b5ee74e135f9d7d8ab06b8a318/redis-7.4.0.tar.gz", hash = "sha256:64a6ea7bf567ad43c964d2c30d82853f8df927c5c9017766c55a1d1ed95d18ad", size = 4943913, upload-time = "2026-03-24T09:14:37.53Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" },
]
[[package]]
name = "rich"
version = "15.0.0"
@@ -1304,7 +1295,6 @@ dependencies = [
{ name = "pydantic" },
{ name = "pydantic-settings" },
{ name = "pyyaml" },
{ name = "redis" },
{ name = "ruff" },
]
@@ -1324,7 +1314,6 @@ requires-dist = [
{ name = "pydantic", specifier = ">=2.12.5" },
{ name = "pydantic-settings", specifier = ">=2.13.1" },
{ name = "pyyaml", specifier = ">=6.0.3" },
{ name = "redis", specifier = ">=7.4.0" },
{ name = "ruff", specifier = ">=0.15.10" },
]